Compare commits

...

35 Commits

Author SHA1 Message Date
yusing
306cb7a20e fix(access_logger): fix stdout and path not working at the same time 2025-11-01 12:07:22 +08:00
yusing
e3915210aa fix(time): data race in DefaultTimeNow 2025-11-01 02:18:24 +08:00
yusing
e8fb202ea9 fix(docker): fix wildcard not working correctly with #N ref aliases 2025-11-01 02:10:09 +08:00
yusing
082b2f5da2 refactor(websocket): use only the first error and fix race condition 2025-11-01 01:28:12 +08:00
yusing
e670acb4b8 fix(access_logger): nil panic when stdout only, improve concurrency safety 2025-11-01 01:17:55 +08:00
yusing
77e486f4fe refactor(route): ensure validation and start only starts once, and lock error before finishing 2025-10-31 18:10:09 +08:00
yusing
3ccaba3163 fix(validation): prioritize pointer method for custom validation in serialization 2025-10-31 18:06:41 +08:00
yusing
705923960c feat(fileserver): add rules support for fileservers 2025-10-31 17:32:37 +08:00
yusing
ca737c8979 fix(modify-html): re-enable modifying HTML with chunked encoding 2025-10-31 17:30:23 +08:00
yusing
b6b5d4dbd7 fix(auth): handle nil defaultAuth to prevent nil panic before auth intializes 2025-10-31 17:15:03 +08:00
yusing
b2919fbaf6 feat(rules): supress some errors in rule execution 2025-10-31 17:13:09 +08:00
yusing
722c40d103 chore(examples): update example configurations with comments for certificate paths and lite variant 2025-10-30 11:45:06 +08:00
yusing
860d9c71b6 fix(pool,io): overlap memory on buffer splitting; hook in HookReadCloser should run after Close 2025-10-29 22:48:28 +08:00
yusing
e354d901c4 fix(monitor): safer approach to avoid nil panic in edge cases 2025-10-29 00:19:17 +08:00
yusing
921a8fb935 fix(monitor): handle missing container state in Docker health check 2025-10-28 23:49:59 +08:00
yusing
975354cdc1 chore(compose): comment out user for lite variant in example configuration 2025-10-28 23:15:54 +08:00
yusing
7d38bfd2d2 build: drop old image name support 2025-10-28 22:05:35 +08:00
yusing
5506cafa26 fix(rules): pages not loading correct for lite webui variant 2025-10-28 22:00:54 +08:00
yusing
9fd5bff81a fix(oidc): fix Webui OIDC loop 2025-10-28 21:54:46 +08:00
yusing
38041ca5b8 fix(pool): handle buffer capacity check in GetSized
- Added a check to ensure the buffer's capacity is sufficient before reusing it.
- Included a FIXME comment to address an unexpected condition in buffer allocation.
2025-10-28 21:48:27 +08:00
yusing
61be88c1d3 chore: upgrade dependencies 2025-10-28 21:40:23 +08:00
yusing
cb4dcb962e fix(http): nil panic in goutils http/intercept.go 2025-10-28 21:37:19 +08:00
yusing
1797a222cd fix(middlewares): correctly bypass middlewares with response rules 2025-10-28 20:44:46 +08:00
yusing
098fb7e62d fix(compose): update rootless compose example 2025-10-28 17:03:20 +08:00
yusing
d4dfec8293 refactor(http): proper ResponseWriter and headers handling across files 2025-10-28 14:43:10 +08:00
yusing
f29b69ff3b refactor(rules): remove Flush method and replace with http.NewResponseController in ResponseModifier 2025-10-27 17:46:23 +08:00
yusing
5e00e1c437 fix(middleware): correct and simplify HTML modification / buffer management logic, correct Accept-Encoding header 2025-10-27 15:08:29 +08:00
yusing
39c8cc2820 fix(auth): nil panic by handling in TryRefreshToken 2025-10-27 14:25:05 +08:00
yusing
56232dbd0e fix(monitor): nil panic in DockerHealthMonitor 2025-10-27 12:46:22 +08:00
yusing
baf774f927 fix(middleware): properly release buffer on error and not to reuse content for bytes.Buffer 2025-10-26 23:16:38 +08:00
yusing
a3c82209c6 refactor(api): disable caching completely 2025-10-26 21:33:58 +08:00
yusing
386d946bd2 feat(rules): support variables for error comand 2025-10-26 20:25:46 +08:00
yusing
ee9bf31d30 chore(compose): add comments for lite variant uid/gid configuration in example 2025-10-26 19:46:59 +08:00
yusing
2c87eebee3 chore(compose): remove host network_mode from example 2025-10-26 19:27:56 +08:00
yusing
5be784d567 chore(env): remove frontend port configuration from example files 2025-10-26 19:26:53 +08:00
47 changed files with 941 additions and 316 deletions

View File

@@ -63,9 +63,6 @@ GODOXY_METRICS_DISABLE_DISK=false
GODOXY_METRICS_DISABLE_NETWORK=false
GODOXY_METRICS_DISABLE_SENSORS=false
# Frontend listening port
GODOXY_FRONTEND_PORT=3000
# Frontend aliases (subdomains / FQDNs, e.g. godoxy, godoxy.domain.com)
GODOXY_FRONTEND_ALIASES=godoxy

View File

@@ -10,7 +10,6 @@ jobs:
uses: ./.github/workflows/docker-image.yml
with:
image_name: ${{ github.repository_owner }}/godoxy
old_image_name: ${{ github.repository_owner }}/go-proxy
tag: latest
target: main
build-prod-agent:

View File

@@ -9,9 +9,6 @@ on:
image_name:
required: true
type: string
old_image_name:
required: false
type: string
target:
required: true
type: string
@@ -156,17 +153,6 @@ jobs:
docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
$(printf '${{ env.REGISTRY }}/${{ inputs.image_name }}@sha256:%s ' *)
- name: Old image name
if: inputs.old_image_name != ''
run: |
docker buildx imagetools create -t ${{ env.REGISTRY }}/${{ inputs.old_image_name }}:${{ steps.meta.outputs.version }}\
${{ env.REGISTRY }}/${{ inputs.image_name }}:${{ steps.meta.outputs.version }}
- name: Inspect image
run: |
docker buildx imagetools inspect ${{ env.REGISTRY }}/${{ inputs.image_name }}:${{ steps.meta.outputs.version }}
- name: Inspect image (old)
if: inputs.old_image_name != ''
run: |
docker buildx imagetools inspect ${{ env.REGISTRY }}/${{ inputs.old_image_name }}:${{ steps.meta.outputs.version }}

View File

@@ -13,14 +13,14 @@ replace github.com/yusing/goutils => ../goutils
exclude github.com/containerd/nerdctl/mod/tigron v0.0.0
require (
github.com/bytedance/sonic v1.14.1
github.com/bytedance/sonic v1.14.2
github.com/gin-gonic/gin v1.11.0
github.com/gorilla/websocket v1.5.3
github.com/puzpuzpuz/xsync/v4 v4.2.0
github.com/rs/zerolog v1.34.0
github.com/stretchr/testify v1.11.1
github.com/valyala/fasthttp v1.68.0
github.com/yusing/godoxy v0.19.2
github.com/yusing/godoxy v0.20.2
github.com/yusing/godoxy/socketproxy v0.0.0-00010101000000-000000000000
github.com/yusing/goutils v0.7.0
)
@@ -32,7 +32,7 @@ require (
github.com/andybalholm/brotli v1.2.0 // indirect
github.com/andybalholm/cascadia v1.3.3 // indirect
github.com/bytedance/gopkg v0.1.3 // indirect
github.com/bytedance/sonic/loader v0.3.0 // indirect
github.com/bytedance/sonic/loader v0.4.0 // indirect
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
github.com/cloudwego/base64x v0.1.6 // indirect
github.com/containerd/errdefs v1.0.0 // indirect
@@ -87,7 +87,7 @@ require (
github.com/tklauser/go-sysconf v0.3.15 // indirect
github.com/tklauser/numcpus v0.10.0 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.3.0 // indirect
github.com/ugorji/go/codec v1.3.1 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/vincent-petithory/dataurl v1.0.0 // indirect
github.com/yusing/ds v0.3.1 // indirect

View File

@@ -12,10 +12,10 @@ github.com/buger/goterm v1.0.4 h1:Z9YvGmOih81P0FbVtEYTFF6YsSgxSUKEhf/f9bTMXbY=
github.com/buger/goterm v1.0.4/go.mod h1:HiFWV3xnkolgrBV3mY8m0X0Pumt4zg4QhbdOzQtB8tE=
github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M=
github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM=
github.com/bytedance/sonic v1.14.1 h1:FBMC0zVz5XUmE4z9wF4Jey0An5FueFvOsTKKKtwIl7w=
github.com/bytedance/sonic v1.14.1/go.mod h1:gi6uhQLMbTdeP0muCnrjHLeCUPyb70ujhnNlhOylAFc=
github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA=
github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI=
github.com/bytedance/sonic v1.14.2 h1:k1twIoe97C1DtYUo+fZQy865IuHia4PR5RPiuGPPIIE=
github.com/bytedance/sonic v1.14.2/go.mod h1:T80iDELeHiHKSc0C9tubFygiuXoGzrkjKzX2quAx980=
github.com/bytedance/sonic/loader v0.4.0 h1:olZ7lEqcxtZygCK9EKYKADnpQoYkRQxaeY2NYzevs+o=
github.com/bytedance/sonic/loader v0.4.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo=
github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M=
@@ -189,10 +189,12 @@ github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/tklauser/go-sysconf v0.3.15 h1:VE89k0criAymJ/Os65CSn1IXaol+1wrsFHEB8Ol49K4=
@@ -201,8 +203,8 @@ github.com/tklauser/numcpus v0.10.0 h1:18njr6LDBk1zuna922MgdjQuJFjrdppsZG60sHGfj
github.com/tklauser/numcpus v0.10.0/go.mod h1:BiTKazU708GQTYF4mB+cmlpT2Is1gLk7XVuEeem8LsQ=
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
github.com/ugorji/go/codec v1.3.0 h1:Qd2W2sQawAfG8XSvzwhBeoGq71zXOC/Q1E9y/wUcsUA=
github.com/ugorji/go/codec v1.3.0/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4=
github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY=
github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.68.0 h1:v12Nx16iepr8r9ySOwqI+5RBJ/DqTxhOy1HrHoDFnok=
@@ -335,8 +337,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
google.golang.org/genproto v0.0.0-20250908214217-97024824d090 h1:ywCL7vA2n3vVHyf+bx1ZV/knaTPRI8GIeKY0MEhEeOc=
google.golang.org/genproto/googleapis/api v0.0.0-20250929231259-57b25ae835d4 h1:8XJ4pajGwOlasW+L13MnEGA8W4115jJySQtVfS2/IBU=
google.golang.org/genproto/googleapis/api v0.0.0-20250929231259-57b25ae835d4/go.mod h1:NnuHhy+bxcg30o7FnVAZbXsPHUDQ9qKWAQKCD7VxFtk=
google.golang.org/genproto/googleapis/rpc v0.0.0-20251014184007-4626949a642f h1:1FTH6cpXFsENbPR5Bu8NQddPSaUUE6NA2XdZdDSAJK4=
google.golang.org/genproto/googleapis/rpc v0.0.0-20251014184007-4626949a642f/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8 h1:M1rk8KBnUsBDg1oPGHNCxG4vc1f49epmTO7xscSajMk=
google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A=
google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c=
google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=

View File

@@ -22,26 +22,28 @@ services:
- ${SOCKET_PROXY_LISTEN_ADDR:-127.0.0.1:2375}:2375
frontend:
image: ghcr.io/yusing/godoxy-frontend:${TAG:-latest}
# lite variant
# image: ghcr.io/yusing/godoxy-frontend:${TAG:-latest}-lite
container_name: godoxy-frontend
restart: unless-stopped
network_mode: host # do not change this
env_file: .env
# comment out `user` for lite variant
user: ${GODOXY_UID:-1000}:${GODOXY_GID:-1000}
read_only: true
tmpfs:
- /app/.next/cache # next image caching
# for lite variant, do not change uid/gid
# - /var/cache/nginx:uid=101,gid=101
# - /run:uid=101,gid=101
security_opt:
- no-new-privileges:true
cap_drop:
- all
depends_on:
- app
environment:
HOSTNAME: 127.0.0.1
PORT: ${GODOXY_FRONTEND_PORT:-3000}
labels:
proxy.aliases: ${GODOXY_FRONTEND_ALIASES:-godoxy}
proxy.#1.port: ${GODOXY_FRONTEND_PORT:-3000}
# proxy.#1.middlewares.cidr_whitelist: |
# status: 403
# message: IP not allowed
@@ -74,10 +76,9 @@ services:
- ./error_pages:/app/error_pages:ro
- ./data:/app/data
# To use autocert, certs will be stored in "./certs".
# You can also use a docker volume to store it
# This path stores certs obtained from autocert and agent TLS client certs
- ./certs:/app/certs
# remove "./certs:/app/certs" and uncomment below to use existing certificate
# mount existing certificate
# - /path/to/certs/cert.crt:/app/certs/cert.crt
# - /path/to/certs/priv.key:/app/certs/priv.key

View File

@@ -4,6 +4,8 @@
# autocert:
# provider: local
# cert_path: /path/to/cert.crt # default: /app/certs/cert.crt
# key_path: /path/to/priv.key # default: /app/certs/priv.key
# 2. cloudflare
# autocert:

10
go.mod
View File

@@ -46,8 +46,8 @@ require (
github.com/spf13/afero v1.15.0
github.com/stretchr/testify v1.11.1
github.com/yusing/ds v0.3.1
github.com/yusing/godoxy/agent v0.0.0-20251025144347-1ec2872f3d4c
github.com/yusing/godoxy/internal/dnsproviders v0.0.0-20251025144347-1ec2872f3d4c
github.com/yusing/godoxy/agent v0.0.0-20251028124446-1797a222cd18
github.com/yusing/godoxy/internal/dnsproviders v0.0.0-20251028124446-1797a222cd18
github.com/yusing/goutils v0.7.0
)
@@ -136,7 +136,7 @@ require (
)
require (
github.com/bytedance/sonic v1.14.1
github.com/bytedance/sonic v1.14.2
github.com/shirou/gopsutil/v4 v4.25.9
github.com/valyala/fasthttp v1.68.0
github.com/yusing/gointernals v0.1.16
@@ -146,7 +146,7 @@ require (
github.com/akamai/AkamaiOPEN-edgegrid-golang/v11 v11.1.0 // indirect
github.com/andybalholm/brotli v1.2.0 // indirect
github.com/bytedance/gopkg v0.1.3 // indirect
github.com/bytedance/sonic/loader v0.3.0 // indirect
github.com/bytedance/sonic/loader v0.4.0 // indirect
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
github.com/cloudwego/base64x v0.1.6 // indirect
github.com/containerd/errdefs v1.0.0 // indirect
@@ -174,7 +174,7 @@ require (
github.com/tklauser/go-sysconf v0.3.15 // indirect
github.com/tklauser/numcpus v0.10.0 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.3.0 // indirect
github.com/ugorji/go/codec v1.3.1 // indirect
github.com/ulikunitz/xz v0.5.14 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/vultr/govultr/v3 v3.24.0 // indirect

16
go.sum
View File

@@ -50,10 +50,10 @@ github.com/buger/goterm v1.0.4 h1:Z9YvGmOih81P0FbVtEYTFF6YsSgxSUKEhf/f9bTMXbY=
github.com/buger/goterm v1.0.4/go.mod h1:HiFWV3xnkolgrBV3mY8m0X0Pumt4zg4QhbdOzQtB8tE=
github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M=
github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM=
github.com/bytedance/sonic v1.14.1 h1:FBMC0zVz5XUmE4z9wF4Jey0An5FueFvOsTKKKtwIl7w=
github.com/bytedance/sonic v1.14.1/go.mod h1:gi6uhQLMbTdeP0muCnrjHLeCUPyb70ujhnNlhOylAFc=
github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA=
github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI=
github.com/bytedance/sonic v1.14.2 h1:k1twIoe97C1DtYUo+fZQy865IuHia4PR5RPiuGPPIIE=
github.com/bytedance/sonic v1.14.2/go.mod h1:T80iDELeHiHKSc0C9tubFygiuXoGzrkjKzX2quAx980=
github.com/bytedance/sonic/loader v0.4.0 h1:olZ7lEqcxtZygCK9EKYKADnpQoYkRQxaeY2NYzevs+o=
github.com/bytedance/sonic/loader v0.4.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo=
github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M=
@@ -288,6 +288,7 @@ github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4=
github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -295,7 +296,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/tklauser/go-sysconf v0.3.15 h1:VE89k0criAymJ/Os65CSn1IXaol+1wrsFHEB8Ol49K4=
@@ -304,8 +306,8 @@ github.com/tklauser/numcpus v0.10.0 h1:18njr6LDBk1zuna922MgdjQuJFjrdppsZG60sHGfj
github.com/tklauser/numcpus v0.10.0/go.mod h1:BiTKazU708GQTYF4mB+cmlpT2Is1gLk7XVuEeem8LsQ=
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
github.com/ugorji/go/codec v1.3.0 h1:Qd2W2sQawAfG8XSvzwhBeoGq71zXOC/Q1E9y/wUcsUA=
github.com/ugorji/go/codec v1.3.0/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4=
github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY=
github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4=
github.com/ulikunitz/xz v0.5.14 h1:uv/0Bq533iFdnMHZdRBTOlaNMdb1+ZxXIlHDZHIHcvg=
github.com/ulikunitz/xz v0.5.14/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=

Submodule goutils updated: c0955732e9...4f51e03aa8

View File

@@ -55,7 +55,7 @@ type config struct {
logAllowed bool
// will be nil if Log is nil
logger *accesslog.AccessLogger
logger accesslog.AccessLogger
// will never tick if Notify.To is empty
notifyTicker *time.Ticker

View File

@@ -2,10 +2,10 @@ package api
import (
"net/http"
"strconv"
"time"
"reflect"
"github.com/gin-gonic/gin"
"github.com/gin-gonic/gin/codec/json"
"github.com/gorilla/websocket"
"github.com/rs/zerolog/log"
apiV1 "github.com/yusing/godoxy/internal/api/v1"
@@ -45,6 +45,9 @@ func NewHandler() *gin.Engine {
r := gin.New()
r.Use(ErrorHandler())
r.Use(ErrorLoggingMiddleware())
r.Use(NoCache())
log.Debug().Msg("gin codec json.API: " + reflect.TypeOf(json.API).Name())
r.GET("/api/v1/version", apiV1.Version)
@@ -69,7 +72,7 @@ func NewHandler() *gin.Engine {
}
{
// enable cache for favicon
v1.GET("/favicon", apiV1.FavIcon).Use(Cache(time.Hour * 24))
v1.GET("/favicon", apiV1.FavIcon)
v1.GET("/health", apiV1.Health)
v1.GET("/icons", apiV1.Icons)
v1.POST("/reload", apiV1.Reload)
@@ -140,15 +143,13 @@ func NewHandler() *gin.Engine {
}
}
// disable cache by default
r.Use(NoCache())
return r
}
func NoCache() gin.HandlerFunc {
return func(c *gin.Context) {
// skip cache if Cache-Control header is set or if caching is explicitly enabled
if !c.GetBool("cache_enabled") && c.Writer.Header().Get("Cache-Control") == "" {
// skip cache if Cache-Control header is set
if c.Writer.Header().Get("Cache-Control") == "" {
c.Header("Cache-Control", "no-cache, no-store, must-revalidate")
c.Header("Pragma", "no-cache")
c.Header("Expires", "0")
@@ -157,20 +158,6 @@ func NoCache() gin.HandlerFunc {
}
}
func Cache(duration time.Duration) gin.HandlerFunc {
return func(c *gin.Context) {
// Signal to NoCache middleware that caching is intended
c.Set("cache_enabled", true)
// skip cache if Cache-Control header is set
if c.Writer.Header().Get("Cache-Control") == "" {
c.Header("Cache-Control", "public, max-age="+strconv.FormatFloat(duration.Seconds(), 'f', 0, 64)+", immutable")
c.Header("Pragma", "public")
c.Header("Expires", time.Now().Add(duration).Format(time.RFC1123))
}
c.Next()
}
}
func AuthMiddleware() gin.HandlerFunc {
return func(c *gin.Context) {
err := auth.GetDefaultAuth().CheckToken(c.Request)

View File

@@ -51,6 +51,10 @@ func ProceedNext(w http.ResponseWriter, r *http.Request) {
}
func AuthCheckHandler(w http.ResponseWriter, r *http.Request) {
if defaultAuth == nil {
w.WriteHeader(http.StatusServiceUnavailable)
return
}
err := defaultAuth.CheckToken(r)
if err != nil {
defaultAuth.LoginHandler(w, r)
@@ -60,6 +64,10 @@ func AuthCheckHandler(w http.ResponseWriter, r *http.Request) {
}
func AuthOrProceed(w http.ResponseWriter, r *http.Request) (proceed bool) {
if defaultAuth == nil {
w.WriteHeader(http.StatusServiceUnavailable)
return false
}
err := defaultAuth.CheckToken(r)
if err != nil {
defaultAuth.LoginHandler(w, r)

View File

@@ -151,7 +151,11 @@ func (auth *OIDCProvider) TryRefreshToken(ctx context.Context, sessionJWT string
// verify the session cookie
claims, valid, err := auth.parseSessionJWT(sessionJWT)
if err != nil {
return nil, fmt.Errorf("session: %s - %w: %w", claims.SessionID, ErrInvalidSessionToken, err)
var sessionID sessionID
if claims != nil {
sessionID = claims.SessionID
}
return nil, fmt.Errorf("session: %s - %w: %w", sessionID, ErrInvalidSessionToken, err)
}
if !valid {
return nil, ErrInvalidSessionToken

View File

@@ -6,7 +6,7 @@ replace github.com/yusing/godoxy => ../..
require (
github.com/go-acme/lego/v4 v4.27.0
github.com/yusing/godoxy v0.19.2
github.com/yusing/godoxy v0.20.2
)
require (
@@ -23,8 +23,8 @@ require (
github.com/akamai/AkamaiOPEN-edgegrid-golang/v11 v11.1.0 // indirect
github.com/benbjohnson/clock v1.3.5 // indirect
github.com/bytedance/gopkg v0.1.3 // indirect
github.com/bytedance/sonic v1.14.1 // indirect
github.com/bytedance/sonic/loader v0.3.0 // indirect
github.com/bytedance/sonic v1.14.2 // indirect
github.com/bytedance/sonic/loader v0.4.0 // indirect
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
github.com/cloudwego/base64x v0.1.6 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect

View File

@@ -36,10 +36,10 @@ github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz
github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M=
github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM=
github.com/bytedance/sonic v1.14.1 h1:FBMC0zVz5XUmE4z9wF4Jey0An5FueFvOsTKKKtwIl7w=
github.com/bytedance/sonic v1.14.1/go.mod h1:gi6uhQLMbTdeP0muCnrjHLeCUPyb70ujhnNlhOylAFc=
github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA=
github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI=
github.com/bytedance/sonic v1.14.2 h1:k1twIoe97C1DtYUo+fZQy865IuHia4PR5RPiuGPPIIE=
github.com/bytedance/sonic v1.14.2/go.mod h1:T80iDELeHiHKSc0C9tubFygiuXoGzrkjKzX2quAx980=
github.com/bytedance/sonic/loader v0.4.0 h1:olZ7lEqcxtZygCK9EKYKADnpQoYkRQxaeY2NYzevs+o=
github.com/bytedance/sonic/loader v0.4.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo=
github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M=
@@ -165,13 +165,15 @@ github.com/sony/gobreaker v1.0.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJ
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4=
github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=

View File

@@ -87,8 +87,8 @@ func ExpandWildcard(labels map[string]string, aliases ...string) {
wildcardLabels[parts[2]] = value
continue
}
// explicit alias label remember the alias
if _, ok := aliasSet[alias]; !ok {
// explicit alias label remember the alias (but not reference aliases like #1, #2)
if _, ok := aliasSet[alias]; !ok && !strings.HasPrefix(alias, "#") {
aliasSet[alias] = len(aliasSet)
}
}
@@ -100,10 +100,9 @@ func ExpandWildcard(labels map[string]string, aliases ...string) {
// expand collected wildcard labels for every alias
for suffix, v := range wildcardLabels {
for alias, i := range aliasSet {
// for FQDN aliases, use numeric index instead of the alias name
if strings.Contains(alias, ".") {
alias = fmt.Sprintf("#%d", i+1)
}
// use numeric index instead of the alias name
alias = fmt.Sprintf("#%d", i+1)
key := fmt.Sprintf("%s.%s.%s", NSProxy, alias, suffix)
if suffix == "" { // this should not happen (root wildcard handled earlier) but keep safe
key = fmt.Sprintf("%s.%s", NSProxy, alias)

View File

@@ -67,6 +67,21 @@ healthcheck:
}, labels)
}
func TestWildcardWithRefAliases(t *testing.T) {
labels := map[string]string{
"proxy.#1.host": "localhost",
"proxy.#1.port": "5555",
"proxy.*.middlewares.request.hide_headers": "X-Header1,X-Header2",
}
docker.ExpandWildcard(labels, "a.example.com", "b.example.com")
require.Equal(t, map[string]string{
"proxy.#1.host": "localhost",
"proxy.#1.port": "5555",
"proxy.#1.middlewares.request.hide_headers": "X-Header1,X-Header2",
"proxy.#2.middlewares.request.hide_headers": "X-Header1,X-Header2",
}, labels)
}
func BenchmarkParseLabels(b *testing.B) {
for b.Loop() {
_, _ = docker.ParseLabels(map[string]string{

View File

@@ -19,7 +19,7 @@ import (
type Entrypoint struct {
middleware *middleware.Middleware
notFoundHandler http.Handler
accessLogger *accesslog.AccessLogger
accessLogger accesslog.AccessLogger
findRouteFunc func(host string) types.HTTPRoute
}

View File

@@ -7,6 +7,7 @@ import (
"sync/atomic"
"time"
"github.com/puzpuzpuz/xsync/v4"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
maxmind "github.com/yusing/godoxy/internal/maxmind/types"
@@ -19,15 +20,24 @@ import (
)
type (
AccessLogger struct {
AccessLogger interface {
Log(req *http.Request, res *http.Response)
LogError(req *http.Request, err error)
LogACL(info *maxmind.IPInfo, blocked bool)
Config() *Config
Flush()
Close() error
}
accessLogger struct {
task *task.Task
cfg *Config
rawWriter io.Writer
closer io.Closer
supportRotate supportRotate
writer *ioutils.BufferedWriter
writeLock sync.Mutex
writer BufferedWriter
supportRotate SupportRotate
writeLock *sync.Mutex
closed bool
writeCount int64
@@ -41,8 +51,9 @@ type (
ACLFormatter
}
WriterWithName interface {
Writer interface {
io.WriteCloser
ShouldBeBuffered() bool
Name() string // file name or path
}
@@ -52,6 +63,10 @@ type (
Name() string
}
AccessLogRotater interface {
Rotate(result *RotateResult) (rotated bool, err error)
}
RequestFormatter interface {
// AppendRequestLog appends a log line to line with or without a trailing newline
AppendRequestLog(line []byte, req *http.Request, res *http.Response) []byte
@@ -62,6 +77,8 @@ type (
}
)
var writerLocks = xsync.NewMap[string, *sync.Mutex]()
const (
InitialBufferSize = 4 * kilobyte
MaxBufferSize = 8 * megabyte
@@ -78,41 +95,43 @@ const (
var bytesPool = synk.GetUnsizedBytesPool()
func NewAccessLogger(parent task.Parent, cfg AnyConfig) (*AccessLogger, error) {
io, err := cfg.IO()
func NewAccessLogger(parent task.Parent, cfg AnyConfig) (AccessLogger, error) {
writers, err := cfg.Writers()
if err != nil {
return nil, err
}
return NewAccessLoggerWithIO(parent, io, cfg), nil
return NewMultiAccessLogger(parent, cfg, writers), nil
}
func NewMockAccessLogger(parent task.Parent, cfg *RequestLoggerConfig) *AccessLogger {
return NewAccessLoggerWithIO(parent, NewMockFile(), cfg)
func NewMockAccessLogger(parent task.Parent, cfg *RequestLoggerConfig) AccessLogger {
return NewAccessLoggerWithIO(parent, NewMockFile(true), cfg)
}
func NewAccessLoggerWithIO(parent task.Parent, writer WriterWithName, anyCfg AnyConfig) *AccessLogger {
func NewAccessLoggerWithIO(parent task.Parent, writer Writer, anyCfg AnyConfig) AccessLogger {
cfg := anyCfg.ToConfig()
if cfg.RotateInterval == 0 {
cfg.RotateInterval = defaultRotateInterval
}
l := &AccessLogger{
l := &accessLogger{
task: parent.Subtask("accesslog."+writer.Name(), true),
cfg: cfg,
rawWriter: writer,
bufSize: InitialBufferSize,
errRateLimiter: rate.NewLimiter(rate.Every(errRateLimit), errBurst),
logger: log.With().Str("file", writer.Name()).Logger(),
}
if writer != nil {
l.writeLock, _ = writerLocks.LoadOrStore(writer.Name(), &sync.Mutex{})
if writer.ShouldBeBuffered() {
l.writer = ioutils.NewBufferedWriter(writer, InitialBufferSize)
if supportRotate, ok := writer.(SupportRotate); ok {
l.supportRotate = supportRotate
}
if closer, ok := writer.(io.Closer); ok {
l.closer = closer
}
} else {
l.writer = NewUnbufferedWriter(writer)
}
if supportRotate, ok := writer.(SupportRotate); ok {
l.supportRotate = supportRotate
}
if cfg.req != nil {
@@ -131,17 +150,15 @@ func NewAccessLoggerWithIO(parent task.Parent, writer WriterWithName, anyCfg Any
l.ACLFormatter = ACLLogFormatter{}
}
if l.writer != nil {
go l.start()
} // otherwise stdout only
go l.start()
return l
}
func (l *AccessLogger) Config() *Config {
func (l *accessLogger) Config() *Config {
return l.cfg
}
func (l *AccessLogger) shouldLog(req *http.Request, res *http.Response) bool {
func (l *accessLogger) shouldLog(req *http.Request, res *http.Response) bool {
if !l.cfg.req.Filters.StatusCodes.CheckKeep(req, res) ||
!l.cfg.req.Filters.Method.CheckKeep(req, res) ||
!l.cfg.req.Filters.Headers.CheckKeep(req, res) ||
@@ -151,7 +168,7 @@ func (l *AccessLogger) shouldLog(req *http.Request, res *http.Response) bool {
return true
}
func (l *AccessLogger) Log(req *http.Request, res *http.Response) {
func (l *accessLogger) Log(req *http.Request, res *http.Response) {
if !l.shouldLog(req, res) {
return
}
@@ -165,11 +182,11 @@ func (l *AccessLogger) Log(req *http.Request, res *http.Response) {
bytesPool.Put(line)
}
func (l *AccessLogger) LogError(req *http.Request, err error) {
func (l *accessLogger) LogError(req *http.Request, err error) {
l.Log(req, &http.Response{StatusCode: http.StatusInternalServerError, Status: err.Error()})
}
func (l *AccessLogger) LogACL(info *maxmind.IPInfo, blocked bool) {
func (l *accessLogger) LogACL(info *maxmind.IPInfo, blocked bool) {
line := bytesPool.Get()
line = l.AppendACLLog(line, info, blocked)
if line[len(line)-1] != '\n' {
@@ -179,16 +196,16 @@ func (l *AccessLogger) LogACL(info *maxmind.IPInfo, blocked bool) {
bytesPool.Put(line)
}
func (l *AccessLogger) ShouldRotate() bool {
func (l *accessLogger) ShouldRotate() bool {
return l.supportRotate != nil && l.cfg.Retention.IsValid()
}
func (l *AccessLogger) Rotate(result *RotateResult) (rotated bool, err error) {
func (l *accessLogger) Rotate(result *RotateResult) (rotated bool, err error) {
if !l.ShouldRotate() {
return false, nil
}
l.writer.Flush()
l.Flush()
l.writeLock.Lock()
defer l.writeLock.Unlock()
@@ -196,7 +213,7 @@ func (l *AccessLogger) Rotate(result *RotateResult) (rotated bool, err error) {
return
}
func (l *AccessLogger) handleErr(err error) {
func (l *accessLogger) handleErr(err error) {
if l.errRateLimiter.Allow() {
gperr.LogError("failed to write access log", err, &l.logger)
} else {
@@ -205,7 +222,7 @@ func (l *AccessLogger) handleErr(err error) {
}
}
func (l *AccessLogger) start() {
func (l *accessLogger) start() {
defer func() {
l.Flush()
l.Close()
@@ -241,52 +258,42 @@ func (l *AccessLogger) start() {
}
}
func (l *AccessLogger) Close() error {
func (l *accessLogger) Close() error {
l.writeLock.Lock()
defer l.writeLock.Unlock()
if l.closed {
return nil
}
if l.closer != nil {
l.closer.Close()
}
l.writer.Release()
l.writer.Flush()
l.closed = true
return nil
return l.writer.Close()
}
func (l *AccessLogger) Flush() {
func (l *accessLogger) Flush() {
l.writeLock.Lock()
defer l.writeLock.Unlock()
if l.closed {
return
}
if err := l.writer.Flush(); err != nil {
l.writer.Flush()
}
func (l *accessLogger) write(data []byte) {
l.writeLock.Lock()
defer l.writeLock.Unlock()
if l.closed {
return
}
n, err := l.writer.Write(data)
if err != nil {
l.handleErr(err)
} else if n < len(data) {
l.handleErr(gperr.Errorf("%w, writing %d bytes, only %d written", io.ErrShortWrite, len(data), n))
}
atomic.AddInt64(&l.writeCount, int64(n))
}
func (l *AccessLogger) write(data []byte) {
if l.writer != nil {
l.writeLock.Lock()
defer l.writeLock.Unlock()
if l.closed {
return
}
n, err := l.writer.Write(data)
if err != nil {
l.handleErr(err)
} else if n < len(data) {
l.handleErr(gperr.Errorf("%w, writing %d bytes, only %d written", io.ErrShortWrite, len(data), n))
}
atomic.AddInt64(&l.writeCount, int64(n))
}
if l.cfg.Stdout {
log.Logger.Write(data) // write to stdout immediately
}
}
func (l *AccessLogger) adjustBuffer() {
func (l *accessLogger) adjustBuffer() {
wps := int(atomic.SwapInt64(&l.writeCount, 0)) / int(bufferAdjustInterval.Seconds())
origBufSize := l.bufSize
newBufSize := origBufSize

View File

@@ -58,7 +58,7 @@ func fmtLog(cfg *RequestLoggerConfig) (ts string, line string) {
t := time.Now()
logger := NewMockAccessLogger(testTask, cfg)
utils.MockTimeNow(t)
buf = logger.AppendRequestLog(buf, req, resp)
buf = logger.(RequestFormatter).AppendRequestLog(buf, req, resp)
return t.Format(LogTimeFormat), string(buf)
}

View File

@@ -61,7 +61,7 @@ func TestBackScanner(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Setup mock file
mockFile := NewMockFile()
mockFile := NewMockFile(false)
_, err := mockFile.Write([]byte(tt.input))
if err != nil {
t.Fatalf("failed to write to mock file: %v", err)
@@ -103,7 +103,7 @@ func TestBackScannerWithVaryingChunkSizes(t *testing.T) {
for _, chunkSize := range chunkSizes {
t.Run(fmt.Sprintf("chunk_size_%d", chunkSize), func(t *testing.T) {
mockFile := NewMockFile()
mockFile := NewMockFile(false)
_, err := mockFile.Write([]byte(input))
if err != nil {
t.Fatalf("failed to write to mock file: %v", err)
@@ -149,7 +149,7 @@ func logEntry() []byte {
res := httptest.NewRecorder()
// server the request
srv.Config.Handler.ServeHTTP(res, req)
b := accesslog.AppendRequestLog(nil, req, res.Result())
b := accesslog.(RequestFormatter).AppendRequestLog(nil, req, res.Result())
if b[len(b)-1] != '\n' {
b = append(b, '\n')
}
@@ -197,7 +197,7 @@ func TestReset(t *testing.T) {
// 100000 log entries.
func BenchmarkBackScanner(b *testing.B) {
mockFile := NewMockFile()
mockFile := NewMockFile(false)
line := logEntry()
for range 100000 {
_, _ = mockFile.Write(line)

View File

@@ -32,7 +32,7 @@ type (
}
AnyConfig interface {
ToConfig() *Config
IO() (WriterWithName, error)
Writers() ([]Writer, error)
}
Format string
@@ -65,17 +65,20 @@ func (cfg *ConfigBase) Validate() gperr.Error {
return nil
}
// IO returns a writer for the config.
// If only stdout is enabled, it returns nil, nil.
func (cfg *ConfigBase) IO() (WriterWithName, error) {
// Writers returns a list of writers for the config.
func (cfg *ConfigBase) Writers() ([]Writer, error) {
writers := make([]Writer, 0, 2)
if cfg.Path != "" {
io, err := NewFileIO(cfg.Path)
if err != nil {
return nil, err
}
return io, nil
writers = append(writers, io)
}
return nil, nil
if cfg.Stdout {
writers = append(writers, NewStdout())
}
return writers, nil
}
func (cfg *ACLLoggerConfig) ToConfig() *Config {

View File

@@ -29,12 +29,19 @@ var (
// NewFileIO creates a new file writer with cleaned path.
//
// If the file is already opened, it will be returned.
func NewFileIO(path string) (WriterWithName, error) {
func NewFileIO(path string) (Writer, error) {
openedFilesMu.Lock()
defer openedFilesMu.Unlock()
var file *File
path = filepath.Clean(path)
var err error
// make it absolute path, so that we can use it as key of `openedFiles` and shared lock
path, err = filepath.Abs(path)
if err != nil {
return nil, fmt.Errorf("access log path error: %w", err)
}
if opened, ok := openedFiles[path]; ok {
opened.refCount.Add()
return opened, nil
@@ -54,8 +61,13 @@ func NewFileIO(path string) (WriterWithName, error) {
return file, nil
}
// Name returns the absolute path of the file.
func (f *File) Name() string {
return f.f.Name()
return f.path
}
func (f *File) ShouldBeBuffered() bool {
return true
}
func (f *File) Write(p []byte) (n int, err error) {

View File

@@ -1,89 +1,96 @@
package accesslog
import (
"fmt"
"math/rand/v2"
"net/http"
"os"
"runtime"
"sync"
"testing"
"github.com/stretchr/testify/assert"
"github.com/yusing/goutils/task"
expect "github.com/yusing/goutils/testing"
)
func TestConcurrentFileLoggersShareSameAccessLogIO(t *testing.T) {
var wg sync.WaitGroup
cfg := DefaultRequestLoggerConfig()
cfg.Path = "test.log"
loggerCount := 10
accessLogIOs := make([]WriterWithName, loggerCount)
loggerCount := runtime.GOMAXPROCS(0)
accessLogIOs := make([]Writer, loggerCount)
// make test log file
file, err := os.Create(cfg.Path)
expect.NoError(t, err)
assert.NoError(t, err)
file.Close()
t.Cleanup(func() {
expect.NoError(t, os.Remove(cfg.Path))
assert.NoError(t, os.Remove(cfg.Path))
})
var wg sync.WaitGroup
for i := range loggerCount {
wg.Add(1)
go func(index int) {
defer wg.Done()
wg.Go(func() {
file, err := NewFileIO(cfg.Path)
expect.NoError(t, err)
accessLogIOs[index] = file
}(i)
assert.NoError(t, err)
accessLogIOs[i] = file
})
}
wg.Wait()
firstIO := accessLogIOs[0]
for _, io := range accessLogIOs {
expect.Equal(t, io, firstIO)
assert.Equal(t, firstIO, io)
}
}
func TestConcurrentAccessLoggerLogAndFlush(t *testing.T) {
file := NewMockFile()
for _, buffered := range []bool{false, true} {
t.Run(fmt.Sprintf("buffered=%t", buffered), func(t *testing.T) {
file := NewMockFile(buffered)
cfg := DefaultRequestLoggerConfig()
parent := task.RootTask("test", false)
cfg := DefaultRequestLoggerConfig()
parent := task.RootTask("test", false)
loggerCount := 5
logCountPerLogger := 10
loggers := make([]*AccessLogger, loggerCount)
loggerCount := runtime.GOMAXPROCS(0)
logCountPerLogger := 10
loggers := make([]AccessLogger, loggerCount)
for i := range loggerCount {
loggers[i] = NewAccessLoggerWithIO(parent, file, cfg)
for i := range loggerCount {
loggers[i] = NewAccessLoggerWithIO(parent, file, cfg)
}
req, _ := http.NewRequest(http.MethodGet, "http://example.com", nil)
resp := &http.Response{StatusCode: http.StatusOK}
var wg sync.WaitGroup
for _, logger := range loggers {
wg.Go(func() {
concurrentLog(logger, req, resp, logCountPerLogger)
})
}
wg.Wait()
for _, logger := range loggers {
logger.Close()
}
expected := loggerCount * logCountPerLogger
actual := file.NumLines()
assert.Equal(t, expected, actual)
})
}
var wg sync.WaitGroup
req, _ := http.NewRequest(http.MethodGet, "http://example.com", nil)
resp := &http.Response{StatusCode: http.StatusOK}
wg.Add(len(loggers))
for _, logger := range loggers {
go func(l *AccessLogger) {
defer wg.Done()
parallelLog(l, req, resp, logCountPerLogger)
l.Flush()
}(logger)
}
wg.Wait()
expected := loggerCount * logCountPerLogger
actual := file.NumLines()
expect.Equal(t, actual, expected)
}
func parallelLog(logger *AccessLogger, req *http.Request, resp *http.Response, n int) {
func concurrentLog(logger AccessLogger, req *http.Request, resp *http.Response, n int) {
var wg sync.WaitGroup
for range n {
wg.Go(func() {
logger.Log(req, resp)
if rand.IntN(2) == 0 {
logger.Flush()
}
})
}
wg.Wait()

View File

@@ -7,26 +7,27 @@ import (
"github.com/spf13/afero"
)
type noLock struct{}
func (noLock) Lock() {}
func (noLock) Unlock() {}
type MockFile struct {
afero.File
noLock
buffered bool
}
var _ SupportRotate = (*MockFile)(nil)
func NewMockFile() *MockFile {
func NewMockFile(buffered bool) *MockFile {
f, _ := afero.TempFile(afero.NewMemMapFs(), "", "")
f.Seek(0, io.SeekEnd)
return &MockFile{
File: f,
File: f,
buffered: buffered,
}
}
func (m *MockFile) ShouldBeBuffered() bool {
return m.buffered
}
func (m *MockFile) Len() int64 {
filesize, _ := m.Seek(0, io.SeekEnd)
_, _ = m.Seek(0, io.SeekStart)
@@ -60,3 +61,7 @@ func (m *MockFile) MustSize() int64 {
size, _ := m.Size()
return size
}
func (m *MockFile) Close() error {
return nil
}

View File

@@ -0,0 +1,63 @@
package accesslog
import (
"net/http"
maxmind "github.com/yusing/godoxy/internal/maxmind/types"
"github.com/yusing/goutils/task"
)
type MultiAccessLogger struct {
accessLoggers []AccessLogger
}
// NewMultiAccessLogger creates a new AccessLogger that writes to multiple writers.
//
// If there is only one writer, it will return a single AccessLogger.
// Otherwise, it will return a MultiAccessLogger that writes to all the writers.
func NewMultiAccessLogger(parent task.Parent, cfg AnyConfig, writers []Writer) AccessLogger {
if len(writers) == 1 {
return NewAccessLoggerWithIO(parent, writers[0], cfg)
}
accessLoggers := make([]AccessLogger, len(writers))
for i, writer := range writers {
accessLoggers[i] = NewAccessLoggerWithIO(parent, writer, cfg)
}
return &MultiAccessLogger{accessLoggers}
}
func (m *MultiAccessLogger) Config() *Config {
return m.accessLoggers[0].Config()
}
func (m *MultiAccessLogger) Log(req *http.Request, res *http.Response) {
for _, accessLogger := range m.accessLoggers {
accessLogger.Log(req, res)
}
}
func (m *MultiAccessLogger) LogError(req *http.Request, err error) {
for _, accessLogger := range m.accessLoggers {
accessLogger.LogError(req, err)
}
}
func (m *MultiAccessLogger) LogACL(info *maxmind.IPInfo, blocked bool) {
for _, accessLogger := range m.accessLoggers {
accessLogger.LogACL(info, blocked)
}
}
func (m *MultiAccessLogger) Flush() {
for _, accessLogger := range m.accessLoggers {
accessLogger.Flush()
}
}
func (m *MultiAccessLogger) Close() error {
for _, accessLogger := range m.accessLoggers {
accessLogger.Close()
}
return nil
}

View File

@@ -0,0 +1,261 @@
package accesslog
import (
"errors"
"net"
"net/http"
"net/url"
"testing"
maxmind "github.com/yusing/godoxy/internal/maxmind/types"
"github.com/yusing/goutils/task"
expect "github.com/yusing/goutils/testing"
)
func TestNewMultiAccessLogger(t *testing.T) {
testTask := task.RootTask("test", false)
cfg := DefaultRequestLoggerConfig()
writers := []Writer{
NewMockFile(true),
NewMockFile(true),
}
logger := NewMultiAccessLogger(testTask, cfg, writers)
expect.NotNil(t, logger)
}
func TestMultiAccessLoggerConfig(t *testing.T) {
testTask := task.RootTask("test", false)
cfg := DefaultRequestLoggerConfig()
cfg.Format = FormatCommon
writers := []Writer{
NewMockFile(true),
NewMockFile(true),
}
logger := NewMultiAccessLogger(testTask, cfg, writers)
retrievedCfg := logger.Config()
expect.Equal(t, retrievedCfg.req.Format, FormatCommon)
}
func TestMultiAccessLoggerLog(t *testing.T) {
testTask := task.RootTask("test", false)
cfg := DefaultRequestLoggerConfig()
cfg.Format = FormatCommon
writer1 := NewMockFile(true)
writer2 := NewMockFile(true)
writers := []Writer{writer1, writer2}
logger := NewMultiAccessLogger(testTask, cfg, writers)
testURL, _ := url.Parse("http://example.com/test")
req := &http.Request{
RemoteAddr: "192.168.1.1",
Method: http.MethodGet,
Proto: "HTTP/1.1",
Host: "example.com",
URL: testURL,
Header: http.Header{
"User-Agent": []string{"test-agent"},
},
}
resp := &http.Response{
StatusCode: http.StatusOK,
ContentLength: 100,
}
logger.Log(req, resp)
logger.Flush()
expect.Equal(t, writer1.NumLines(), 1)
expect.Equal(t, writer2.NumLines(), 1)
}
func TestMultiAccessLoggerLogError(t *testing.T) {
testTask := task.RootTask("test", false)
cfg := DefaultRequestLoggerConfig()
writer1 := NewMockFile(true)
writer2 := NewMockFile(true)
writers := []Writer{writer1, writer2}
logger := NewMultiAccessLogger(testTask, cfg, writers)
testURL, _ := url.Parse("http://example.com/test")
req := &http.Request{
RemoteAddr: "192.168.1.1",
Method: http.MethodGet,
URL: testURL,
}
testErr := errors.New("test error")
logger.LogError(req, testErr)
logger.Flush()
expect.Equal(t, writer1.NumLines(), 1)
expect.Equal(t, writer2.NumLines(), 1)
}
func TestMultiAccessLoggerLogACL(t *testing.T) {
testTask := task.RootTask("test", false)
cfg := DefaultACLLoggerConfig()
cfg.LogAllowed = true
writer1 := NewMockFile(true)
writer2 := NewMockFile(true)
writers := []Writer{writer1, writer2}
logger := NewMultiAccessLogger(testTask, cfg, writers)
info := &maxmind.IPInfo{
IP: net.ParseIP("192.168.1.1"),
Str: "192.168.1.1",
}
logger.LogACL(info, false)
logger.Flush()
expect.Equal(t, writer1.NumLines(), 1)
expect.Equal(t, writer2.NumLines(), 1)
}
func TestMultiAccessLoggerFlush(t *testing.T) {
testTask := task.RootTask("test", false)
cfg := DefaultRequestLoggerConfig()
writer1 := NewMockFile(true)
writer2 := NewMockFile(true)
writers := []Writer{writer1, writer2}
logger := NewMultiAccessLogger(testTask, cfg, writers)
testURL, _ := url.Parse("http://example.com/test")
req := &http.Request{
RemoteAddr: "192.168.1.1",
Method: http.MethodGet,
URL: testURL,
}
resp := &http.Response{
StatusCode: http.StatusOK,
}
logger.Log(req, resp)
logger.Flush()
expect.Equal(t, writer1.NumLines(), 1)
expect.Equal(t, writer2.NumLines(), 1)
}
func TestMultiAccessLoggerClose(t *testing.T) {
testTask := task.RootTask("test", false)
cfg := DefaultRequestLoggerConfig()
writer1 := NewMockFile(true)
writer2 := NewMockFile(true)
writers := []Writer{writer1, writer2}
logger := NewMultiAccessLogger(testTask, cfg, writers)
err := logger.Close()
expect.Nil(t, err)
}
func TestMultiAccessLoggerMultipleLogs(t *testing.T) {
testTask := task.RootTask("test", false)
cfg := DefaultRequestLoggerConfig()
writer1 := NewMockFile(true)
writer2 := NewMockFile(true)
writers := []Writer{writer1, writer2}
logger := NewMultiAccessLogger(testTask, cfg, writers)
testURL, _ := url.Parse("http://example.com/test")
for range 3 {
req := &http.Request{
RemoteAddr: "192.168.1.1",
Method: http.MethodGet,
URL: testURL,
}
resp := &http.Response{
StatusCode: http.StatusOK,
}
logger.Log(req, resp)
}
logger.Flush()
expect.Equal(t, writer1.NumLines(), 3)
expect.Equal(t, writer2.NumLines(), 3)
}
func TestMultiAccessLoggerSingleWriter(t *testing.T) {
testTask := task.RootTask("test", false)
cfg := DefaultRequestLoggerConfig()
writer := NewMockFile(true)
writers := []Writer{writer}
logger := NewMultiAccessLogger(testTask, cfg, writers)
expect.NotNil(t, logger)
testURL, _ := url.Parse("http://example.com/test")
req := &http.Request{
RemoteAddr: "192.168.1.1",
Method: http.MethodGet,
URL: testURL,
}
resp := &http.Response{
StatusCode: http.StatusOK,
}
logger.Log(req, resp)
logger.Flush()
expect.Equal(t, writer.NumLines(), 1)
}
func TestMultiAccessLoggerMixedOperations(t *testing.T) {
testTask := task.RootTask("test", false)
cfg := DefaultRequestLoggerConfig()
writer1 := NewMockFile(true)
writer2 := NewMockFile(true)
writers := []Writer{writer1, writer2}
logger := NewMultiAccessLogger(testTask, cfg, writers)
testURL, _ := url.Parse("http://example.com/test")
req := &http.Request{
RemoteAddr: "192.168.1.1",
Method: http.MethodGet,
URL: testURL,
}
resp := &http.Response{
StatusCode: http.StatusOK,
}
logger.Log(req, resp)
logger.Flush()
info := &maxmind.IPInfo{
IP: net.ParseIP("192.168.1.1"),
Str: "192.168.1.1",
}
cfg2 := DefaultACLLoggerConfig()
cfg2.LogAllowed = true
aclLogger := NewMultiAccessLogger(testTask, cfg2, writers)
aclLogger.LogACL(info, false)
logger.Flush()
expect.Equal(t, writer1.NumLines(), 1)
expect.Equal(t, writer2.NumLines(), 1)
}

View File

@@ -55,7 +55,7 @@ func TestParseLogTime(t *testing.T) {
func TestRotateKeepLast(t *testing.T) {
for _, format := range ReqLoggerFormats {
t.Run(string(format)+" keep last", func(t *testing.T) {
file := NewMockFile()
file := NewMockFile(true)
utils.MockTimeNow(testTime)
logger := NewAccessLoggerWithIO(task.RootTask("test", false), file, &RequestLoggerConfig{
Format: format,
@@ -77,7 +77,7 @@ func TestRotateKeepLast(t *testing.T) {
logger.Config().Retention = retention
var result RotateResult
rotated, err := logger.Rotate(&result)
rotated, err := logger.(AccessLogRotater).Rotate(&result)
expect.NoError(t, err)
expect.Equal(t, rotated, true)
expect.Equal(t, file.NumLines(), int(retention.Last))
@@ -86,7 +86,7 @@ func TestRotateKeepLast(t *testing.T) {
})
t.Run(string(format)+" keep days", func(t *testing.T) {
file := NewMockFile()
file := NewMockFile(true)
logger := NewAccessLoggerWithIO(task.RootTask("test", false), file, &RequestLoggerConfig{
Format: format,
})
@@ -107,7 +107,7 @@ func TestRotateKeepLast(t *testing.T) {
utils.MockTimeNow(testTime)
var result RotateResult
rotated, err := logger.Rotate(&result)
rotated, err := logger.(AccessLogRotater).Rotate(&result)
expect.NoError(t, err)
expect.Equal(t, rotated, true)
expect.Equal(t, file.NumLines(), int(retention.Days))
@@ -132,7 +132,7 @@ func TestRotateKeepLast(t *testing.T) {
func TestRotateKeepFileSize(t *testing.T) {
for _, format := range ReqLoggerFormats {
t.Run(string(format)+" keep size no rotation", func(t *testing.T) {
file := NewMockFile()
file := NewMockFile(true)
logger := NewAccessLoggerWithIO(task.RootTask("test", false), file, &RequestLoggerConfig{
Format: format,
})
@@ -153,7 +153,7 @@ func TestRotateKeepFileSize(t *testing.T) {
utils.MockTimeNow(testTime)
var result RotateResult
rotated, err := logger.Rotate(&result)
rotated, err := logger.(AccessLogRotater).Rotate(&result)
expect.NoError(t, err)
// file should be untouched as 100KB > 10 lines * bytes per line
@@ -164,7 +164,7 @@ func TestRotateKeepFileSize(t *testing.T) {
}
t.Run("keep size with rotation", func(t *testing.T) {
file := NewMockFile()
file := NewMockFile(true)
logger := NewAccessLoggerWithIO(task.RootTask("test", false), file, &RequestLoggerConfig{
Format: FormatJSON,
})
@@ -185,7 +185,7 @@ func TestRotateKeepFileSize(t *testing.T) {
utils.MockTimeNow(testTime)
var result RotateResult
rotated, err := logger.Rotate(&result)
rotated, err := logger.(AccessLogRotater).Rotate(&result)
expect.NoError(t, err)
expect.Equal(t, rotated, true)
expect.Equal(t, result.NumBytesKeep, int64(retention.KeepSize))
@@ -198,7 +198,7 @@ func TestRotateKeepFileSize(t *testing.T) {
func TestRotateSkipInvalidTime(t *testing.T) {
for _, format := range ReqLoggerFormats {
t.Run(string(format), func(t *testing.T) {
file := NewMockFile()
file := NewMockFile(true)
logger := NewAccessLoggerWithIO(task.RootTask("test", false), file, &RequestLoggerConfig{
Format: format,
})
@@ -221,7 +221,7 @@ func TestRotateSkipInvalidTime(t *testing.T) {
logger.Config().Retention = retention
var result RotateResult
rotated, err := logger.Rotate(&result)
rotated, err := logger.(AccessLogRotater).Rotate(&result)
expect.NoError(t, err)
expect.Equal(t, rotated, true)
// should read one invalid line after every valid line
@@ -240,7 +240,7 @@ func BenchmarkRotate(b *testing.B) {
}
for _, retention := range tests {
b.Run(fmt.Sprintf("retention_%s", retention.String()), func(b *testing.B) {
file := NewMockFile()
file := NewMockFile(true)
logger := NewAccessLoggerWithIO(task.RootTask("test", false), file, &RequestLoggerConfig{
ConfigBase: ConfigBase{
Retention: retention,
@@ -256,11 +256,11 @@ func BenchmarkRotate(b *testing.B) {
b.ResetTimer()
for b.Loop() {
b.StopTimer()
file = NewMockFile()
file = NewMockFile(true)
_, _ = file.Write(content)
b.StartTimer()
var result RotateResult
_, _ = logger.Rotate(&result)
_, _ = logger.(AccessLogRotater).Rotate(&result)
}
})
}
@@ -274,7 +274,7 @@ func BenchmarkRotateWithInvalidTime(b *testing.B) {
}
for _, retention := range tests {
b.Run(fmt.Sprintf("retention_%s", retention.String()), func(b *testing.B) {
file := NewMockFile()
file := NewMockFile(true)
logger := NewAccessLoggerWithIO(task.RootTask("test", false), file, &RequestLoggerConfig{
ConfigBase: ConfigBase{
Retention: retention,
@@ -293,11 +293,11 @@ func BenchmarkRotateWithInvalidTime(b *testing.B) {
b.ResetTimer()
for b.Loop() {
b.StopTimer()
file = NewMockFile()
file = NewMockFile(true)
_, _ = file.Write(content)
b.StartTimer()
var result RotateResult
_, _ = logger.Rotate(&result)
_, _ = logger.(AccessLogRotater).Rotate(&result)
}
})
}

View File

@@ -0,0 +1,32 @@
package accesslog
import (
"os"
"github.com/rs/zerolog"
"github.com/yusing/godoxy/internal/logging"
)
type Stdout struct {
logger zerolog.Logger
}
func NewStdout() Writer {
return &Stdout{logger: logging.NewLoggerWithFixedLevel(zerolog.InfoLevel, os.Stdout)}
}
func (s Stdout) Name() string {
return "stdout"
}
func (s Stdout) ShouldBeBuffered() bool {
return false
}
func (s Stdout) Write(p []byte) (n int, err error) {
return s.logger.Write(p)
}
func (s Stdout) Close() error {
return nil
}

View File

@@ -0,0 +1,47 @@
package accesslog
import (
"io"
)
type BufferedWriter interface {
io.Writer
io.Closer
Flush() error
Resize(size int) error
}
type unbufferedWriter struct {
w io.Writer
}
func NewUnbufferedWriter(w io.Writer) BufferedWriter {
return unbufferedWriter{w: w}
}
func (w unbufferedWriter) Write(p []byte) (n int, err error) {
return w.w.Write(p)
}
func (w unbufferedWriter) Close() error {
if closer, ok := w.w.(io.Closer); ok {
return closer.Close()
}
return nil
}
func (w unbufferedWriter) Flush() error {
if flusher, ok := w.w.(interface{ Flush() }); ok {
flusher.Flush()
} else if errFlusher, ok := w.w.(interface{ FlushError() error }); ok {
return errFlusher.FlushError()
} else if errFlusher2, ok := w.w.(interface{ Flush() error }); ok {
return errFlusher2.Flush()
}
return nil
}
func (w unbufferedWriter) Resize(size int) error {
// No-op for unbuffered writer
return nil
}

View File

@@ -30,8 +30,8 @@ func (c *checkBypass) before(w http.ResponseWriter, r *http.Request) (proceedNex
return c.modReq.before(w, r)
}
func (c *checkBypass) modifyResponse(w http.ResponseWriter, resp *http.Response) error {
if c.modRes == nil || c.bypass.ShouldBypass(w, resp.Request) {
func (c *checkBypass) modifyResponse(resp *http.Response) error {
if c.modRes == nil || c.bypass.ShouldBypass(rules.ResponseAsRW(resp), resp.Request) {
return nil
}
return c.modRes.modifyResponse(resp)

View File

@@ -138,6 +138,82 @@ func TestReverseProxyBypass(t *testing.T) {
}
}
func TestBypassResponse(t *testing.T) {
t.Run("req_rules", func(t *testing.T) {
mr, err := ModifyResponse.New(map[string]any{
"bypass": []string{"path glob(/test/*) | path /api"},
"set_headers": map[string]string{
"Test-Header": "test-value",
},
})
expect.NoError(t, err)
tests := []struct {
name string
path string
expectBypass bool
}{
{"bypass", "/test/123", true},
{"bypass2", "/test/123/456", true},
{"bypass3", "/api", true},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
req := httptest.NewRequest("GET", "http://example.com"+test.path, nil)
resp := &http.Response{
StatusCode: http.StatusOK,
Body: io.NopCloser(strings.NewReader("test")),
Request: req,
Header: make(http.Header),
}
mErr := mr.ModifyResponse(resp)
expect.NoError(t, mErr)
if test.expectBypass {
expect.Equal(t, resp.Header.Get("Test-Header"), "")
} else {
expect.Equal(t, resp.Header.Get("Test-Header"), "test-value")
}
})
}
})
t.Run("res_rules", func(t *testing.T) {
mr, err := ModifyResponse.New(map[string]any{
"bypass": []string{"status 200"},
"set_headers": map[string]string{
"Test-Header": "test-value",
},
})
expect.NoError(t, err)
tests := []struct {
name string
statusCode int
expectBypass bool
}{
{"bypass", 200, true},
{"no_bypass", 201, false},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
resp := &http.Response{
StatusCode: test.statusCode,
Body: io.NopCloser(strings.NewReader("test")),
Header: make(http.Header),
}
mErr := mr.ModifyResponse(resp)
expect.NoError(t, mErr)
if test.expectBypass {
expect.Equal(t, resp.Header.Get("Test-Header"), "")
} else {
expect.Equal(t, resp.Header.Get("Test-Header"), "test-value")
}
})
}
})
}
func TestEntrypointBypassRoute(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("test"))

View File

@@ -15,20 +15,15 @@ import (
)
type modifyHTML struct {
Target string // css selector
HTML string // html to inject
Replace bool // replace the target element with the new html instead of appending it
bytesPool synk.UnsizedBytesPool
Target string // css selector
HTML string // html to inject
Replace bool // replace the target element with the new html instead of appending it
}
var ModifyHTML = NewMiddleware[modifyHTML]()
func (m *modifyHTML) setup() {
m.bytesPool = synk.GetUnsizedBytesPool()
}
func (m *modifyHTML) before(_ http.ResponseWriter, req *http.Request) bool {
req.Header.Set("Accept-Encoding", "")
req.Header.Set("Accept-Encoding", "identity")
return true
}
@@ -50,15 +45,27 @@ func (m *modifyHTML) modifyResponse(resp *http.Response) error {
return nil
}
// Skip modification for streaming/chunked responses to avoid blocking reads
// Unknown content length or any transfer encoding indicates streaming.
// if resp.ContentLength < 0 || len(resp.TransferEncoding) > 0 {
// log.Debug().Str("url", fullURL(resp.Request)).Strs("transfer-encoding", resp.TransferEncoding).Msg("skipping modification for streaming/chunked response")
// return nil
// }
// NOTE: do not put it in the defer, it will be used as resp.Body
content, release, err := httputils.ReadAllBody(resp)
resp.Body.Close()
if err != nil {
log.Err(err).Str("url", fullURL(resp.Request)).Msg("failed to read response body")
resp.Body.Close()
// Fail open: do not abort the response. Return an empty body safely.
resp.ContentLength = 0
resp.Header.Set("Content-Length", "0")
resp.Header.Del("Transfer-Encoding")
resp.Header.Del("Trailer")
resp.Header.Del("Content-Encoding")
resp.Body = eofReader{}
return err
return nil
}
resp.Body.Close()
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(content))
if err != nil {
@@ -83,20 +90,27 @@ func (m *modifyHTML) modifyResponse(resp *http.Response) error {
ele.First().AppendHtml(m.HTML)
}
buf := bytes.NewBuffer(content[:0])
pool := synk.GetUnsizedBytesPool()
buf := pool.GetBuffer()
err = buildHTML(doc, buf)
if err != nil {
pool.PutBuffer(buf)
log.Err(err).Str("url", fullURL(resp.Request)).Msg("failed to build html")
// invalid html, restore the original body
resp.Body = readerWithRelease(content, release)
return err
}
release(content)
resp.ContentLength = int64(buf.Len())
resp.Header.Set("Content-Length", strconv.Itoa(buf.Len()))
resp.Header.Del("Transfer-Encoding")
resp.Header.Del("Trailer")
resp.Header.Del("Content-Encoding")
resp.Header.Set("Content-Type", "text/html; charset=utf-8")
resp.Body = readerWithRelease(buf.Bytes(), func(_ []byte) {
// release content, not buf.Bytes()
release(content)
pool.PutBuffer(buf)
})
return nil
}

View File

@@ -45,10 +45,6 @@ var (
var fontCSSTemplate = template.Must(template.New("fontCSS").Parse(fontCSS))
func (m *themed) setup() {
m.m.setup()
}
func (m *themed) before(w http.ResponseWriter, req *http.Request) bool {
return m.m.before(w, req)
}

View File

@@ -20,7 +20,7 @@ type (
middleware *middleware.Middleware
handler http.Handler
accessLogger *accesslog.AccessLogger
accessLogger accesslog.AccessLogger
}
)
@@ -86,6 +86,10 @@ func (s *FileServer) Start(parent task.Parent) gperr.Error {
}
}
if len(s.Rules) > 0 {
s.handler = s.Rules.BuildHandler(s.handler.ServeHTTP)
}
if s.UseHealthCheck() {
s.HealthMon = monitor.NewFileServerHealthMonitor(s.HealthCheck, s.Root)
if err := s.HealthMon.Start(s.task); err != nil {

View File

@@ -82,19 +82,39 @@ type (
impl types.Route
task *task.Task
isValidated bool
lastError gperr.Error
provider types.RouteProvider
// ensure err is read after validation or start
valErr lockedError
startErr lockedError
provider types.RouteProvider
agent *agent.AgentConfig
started chan struct{}
once sync.Once
started chan struct{}
onceStart sync.Once
onceValidate sync.Once
}
Routes map[string]*Route
Port = route.Port
)
type lockedError struct {
err gperr.Error
lock sync.Mutex
}
func (le *lockedError) Get() gperr.Error {
le.lock.Lock()
defer le.lock.Unlock()
return le.err
}
func (le *lockedError) Set(err gperr.Error) {
le.lock.Lock()
defer le.lock.Unlock()
le.err = err
}
const DefaultHost = "localhost"
func (r Routes) Contains(alias string) bool {
@@ -103,11 +123,13 @@ func (r Routes) Contains(alias string) bool {
}
func (r *Route) Validate() gperr.Error {
if r.isValidated {
return r.lastError
}
r.isValidated = true
r.onceValidate.Do(func() {
r.valErr.Set(r.validate())
})
return r.valErr.Get()
}
func (r *Route) validate() gperr.Error {
if r.Agent != "" {
if r.Container != nil {
return gperr.Errorf("specifying agent is not allowed for docker container routes")
@@ -250,7 +272,6 @@ func (r *Route) Validate() gperr.Error {
}
if errs.HasError() {
r.lastError = errs.Error()
return errs.Error()
}
@@ -266,7 +287,6 @@ func (r *Route) Validate() gperr.Error {
}
if err != nil {
r.lastError = err
return err
}
@@ -320,13 +340,10 @@ func (r *Route) Task() *task.Task {
}
func (r *Route) Start(parent task.Parent) gperr.Error {
if r.lastError != nil {
return r.lastError
}
r.once.Do(func() {
r.lastError = r.start(parent)
r.onceStart.Do(func() {
r.startErr.Set(r.start(parent))
})
return r.lastError
return r.startErr.Get()
}
func (r *Route) start(parent task.Parent) gperr.Error {
@@ -496,7 +513,7 @@ func (r *Route) IsZeroPort() bool {
}
func (r *Route) ShouldExclude() bool {
if r.lastError != nil {
if r.valErr.Get() != nil {
return true
}
if r.Excluded {
@@ -565,7 +582,7 @@ func (re ExcludedReason) MarshalJSON() ([]byte, error) {
// no need to unmarshal json because we don't store this
func (r *Route) findExcludedReason() ExcludedReason {
if r.lastError != nil {
if r.valErr.Get() != nil {
return ExcludedReasonError
}
if r.ExcludedReason != ExcludedReasonNone {

View File

@@ -188,15 +188,20 @@ var commands = map[string]struct {
if !httputils.IsStatusCodeValid(code) {
return nil, ErrInvalidArguments.Subject(codeStr)
}
return &Tuple[int, string]{code, text}, nil
textTmpl, err := validateTemplate(text, true)
if err != nil {
return nil, ErrInvalidArguments.With(err)
}
return &Tuple[int, templateString]{code, textTmpl}, nil
},
build: func(args any) CommandHandler {
code, text := args.(*Tuple[int, string]).Unpack()
code, textTmpl := args.(*Tuple[int, templateString]).Unpack()
return TerminatingCommand(func(w http.ResponseWriter, r *http.Request) error {
// error command should overwrite the response body
GetInitResponseModifier(w).ResetBody()
http.Error(w, text, code)
return nil
w.WriteHeader(code)
err := textTmpl.ExpandVars(w, r, w)
return err
})
},
},

View File

@@ -5,7 +5,8 @@
on: |
!path regex("(_next/static|_next/image|favicon.ico).*")
!path glob("/api/v1/auth/*")
!path regex("[A-Za-z0-9_-]+\.(svg|png|jpg|jpeg|gif|ico|webp|woff2?|eot|ttf|otf)(\?.+)?")
!path glob("/auth/*")
!path regex("[A-Za-z0-9_-]+\.(svg|png|jpg|jpeg|gif|ico|webp|woff2?|eot|ttf|otf|txt)(\?.+)?")
!path /api/v1/version
do: require_auth
- name: proxy to backend

View File

@@ -4,10 +4,12 @@ import (
"bufio"
"bytes"
"errors"
"io"
"net"
"net/http"
"strconv"
"github.com/rs/zerolog/log"
gperr "github.com/yusing/goutils/errs"
"github.com/yusing/goutils/synk"
)
@@ -43,6 +45,29 @@ func unwrapResponseModifier(w http.ResponseWriter) *ResponseModifier {
}
}
type responseAsRW struct {
resp *http.Response
}
func (r responseAsRW) WriteHeader(code int) {
log.Error().Msg("write header after response has been created")
}
func (r responseAsRW) Write(b []byte) (int, error) {
return 0, io.ErrClosedPipe
}
func (r responseAsRW) Header() http.Header {
return r.resp.Header
}
func ResponseAsRW(resp *http.Response) *ResponseModifier {
return &ResponseModifier{
statusCode: resp.StatusCode,
w: responseAsRW{resp},
}
}
// GetInitResponseModifier returns the response modifier for the given response writer.
// If the response writer is already wrapped, it will return the wrapped response modifier.
// Otherwise, it will return a new response modifier.
@@ -144,15 +169,6 @@ func (rm *ResponseModifier) Hijack() (net.Conn, *bufio.ReadWriter, error) {
return nil, nil, errors.New("hijack not supported")
}
func (rm *ResponseModifier) Flush() error {
if flusher, ok := rm.w.(http.Flusher); ok {
flusher.Flush()
} else if errFlusher, ok := rm.w.(interface{ Flush() error }); ok {
return errFlusher.Flush()
}
return nil
}
// FlushRelease flushes the response modifier and releases the resources
// it returns the number of bytes written and the aggregated error
// if there is any error (rule errors or write error), it will be returned
@@ -167,6 +183,8 @@ func (rm *ResponseModifier) FlushRelease() (int, error) {
// }
contentLength := rm.ContentLength()
h.Set("Content-Length", strconv.Itoa(rm.ContentLength()))
h.Del("Transfer-Encoding")
h.Del("Trailer")
rm.w.WriteHeader(rm.StatusCode())
if contentLength > 0 {
@@ -175,7 +193,7 @@ func (rm *ResponseModifier) FlushRelease() (int, error) {
if werr != nil {
rm.errs.Addf("write error: %w", werr)
}
if err := rm.Flush(); err != nil {
if err := http.NewResponseController(rm.w).Flush(); err != nil {
rm.errs.Addf("flush error: %w", err)
}
}

View File

@@ -6,7 +6,11 @@ import (
"net/http"
"github.com/bytedance/sonic"
gperr "github.com/yusing/goutils/errs"
"github.com/quic-go/quic-go/http3"
"github.com/rs/zerolog/log"
"golang.org/x/net/http2"
_ "unsafe"
)
type (
@@ -89,6 +93,11 @@ func (rules Rules) BuildHandler(up http.HandlerFunc) http.HandlerFunc {
if defaultRule.IsResponseRule() {
return func(w http.ResponseWriter, r *http.Request) {
rm := NewResponseModifier(w)
defer func() {
if _, err := rm.FlushRelease(); err != nil {
logError(err, r)
}
}()
w = rm
up(w, r)
err := defaultRule.Do.exec.Handle(w, r)
@@ -99,6 +108,11 @@ func (rules Rules) BuildHandler(up http.HandlerFunc) http.HandlerFunc {
}
return func(w http.ResponseWriter, r *http.Request) {
rm := NewResponseModifier(w)
defer func() {
if _, err := rm.FlushRelease(); err != nil {
logError(err, r)
}
}()
w = rm
err := defaultRule.Do.exec.Handle(w, r)
if err == nil {
@@ -128,7 +142,7 @@ func (rules Rules) BuildHandler(up http.HandlerFunc) http.HandlerFunc {
rm := NewResponseModifier(w)
defer func() {
if _, err := rm.FlushRelease(); err != nil {
gperr.LogError("error executing rules", err)
logError(err, r)
}
}()
@@ -252,3 +266,31 @@ func (rule *Rule) Check(w http.ResponseWriter, r *http.Request) bool {
func (rule *Rule) Handle(w http.ResponseWriter, r *http.Request) error {
return rule.Do.exec.Handle(w, r)
}
//go:linkname errStreamClosed golang.org/x/net/http2.errStreamClosed
var errStreamClosed error
func logError(err error, r *http.Request) {
if errors.Is(err, errStreamClosed) {
return
}
var h2Err http2.StreamError
if errors.As(err, &h2Err) {
// ignore these errors
switch h2Err.Code {
case http2.ErrCodeStreamClosed:
return
}
}
var h3Err *http3.Error
if errors.As(err, &h3Err) {
// ignore these errors
switch h3Err.ErrorCode {
case
http3.ErrCodeNoError,
http3.ErrCodeRequestCanceled:
return
}
}
log.Err(err).Str("method", r.Method).Str("url", r.Host+r.URL.Path).Msg("error executing rules")
}

View File

@@ -52,10 +52,7 @@ func ValidateVars(s string) error {
func ExpandVars(w *ResponseModifier, req *http.Request, src string, dstW io.Writer) error {
dst := ioutils.NewBufferedWriter(dstW, 1024)
defer func() {
dst.Flush()
dst.Release()
}()
defer dst.Close()
for i := 0; i < len(src); i++ {
ch := src[i]

View File

@@ -41,12 +41,17 @@ func ValidateWithCustomValidator(v reflect.Value) gperr.Error {
} else {
vt := v.Type()
if vt.PkgPath() != "" { // not a builtin type
// prioritize pointer method
if v.CanAddr() {
vAddr := v.Addr()
if vAddr.Type().Implements(validatorType) {
return vAddr.Interface().(CustomValidator).Validate()
}
}
// fallback to value method
if vt.Implements(validatorType) {
return v.Interface().(CustomValidator).Validate()
}
if v.CanAddr() {
return validateWithValidator(v.Addr())
}
}
}
return nil

View File

@@ -10,7 +10,7 @@ var (
TimeNow = DefaultTimeNow
shouldCallTimeNow atomic.Bool
timeNowTicker = time.NewTicker(shouldCallTimeNowInterval)
lastTimeNow = time.Now()
lastTimeNow = atomic.NewTime(time.Now())
)
const shouldCallTimeNowInterval = 100 * time.Millisecond
@@ -26,11 +26,13 @@ func MockTimeNow(t time.Time) {
//
// Returned value may have +-100ms error.
func DefaultTimeNow() time.Time {
if shouldCallTimeNow.Load() {
lastTimeNow = time.Now()
shouldCallTimeNow.Store(false)
swapped := shouldCallTimeNow.CompareAndSwap(false, true)
if swapped { // first call
now := time.Now()
lastTimeNow.Store(now)
return now
}
return lastTimeNow
return lastTimeNow.Load()
}
func init() {

View File

@@ -5,16 +5,18 @@ import (
"time"
)
var sink time.Time
func BenchmarkTimeNow(b *testing.B) {
b.Run("default", func(b *testing.B) {
for b.Loop() {
time.Now()
sink = time.Now()
}
})
b.Run("reduced_call", func(b *testing.B) {
for b.Loop() {
DefaultTimeNow()
sink = DefaultTimeNow()
}
})
}

View File

@@ -40,15 +40,15 @@ func (mon *DockerHealthMonitor) Start(parent task.Parent) gperr.Error {
if err != nil {
return err
}
// zero port
if mon.monitor.task == nil {
return nil
}
mon.client.InterceptHTTPClient(mon.interceptInspectResponse)
mon.monitor.task.OnFinished("close docker client", mon.client.Close)
return nil
}
type inspectState struct {
State *container.State
}
func (mon *DockerHealthMonitor) interceptInspectResponse(resp *http.Response) (intercepted bool, err error) {
if resp.StatusCode != http.StatusOK {
return false, nil
@@ -60,12 +60,13 @@ func (mon *DockerHealthMonitor) interceptInspectResponse(resp *http.Response) (i
return false, err
}
var state inspectState
var state container.State
err = sonic.Unmarshal(body, &state)
release(body)
if err != nil {
return false, err
}
return true, httputils.NewRequestInterceptedError(resp, state)
}
@@ -82,13 +83,20 @@ func (mon *DockerHealthMonitor) CheckHealth() (types.HealthCheckResult, error) {
_, err := mon.client.ContainerInspect(ctx, mon.containerID)
var interceptedErr *httputils.RequestInterceptedError
if err != nil && !httputils.AsRequestInterceptedError(err, &interceptedErr) {
if !httputils.AsRequestInterceptedError(err, &interceptedErr) {
mon.numDockerFailures++
log.Debug().Err(err).Str("container_id", mon.containerID).Msg("docker health check failed, using fallback")
return mon.fallback.CheckHealth()
}
state := interceptedErr.Data.(inspectState).State
if interceptedErr == nil || interceptedErr.Data == nil { // should not happen
log.Debug().Msgf("intercepted error is nil or data is nil, container_id: %s", mon.containerID)
mon.numDockerFailures++
log.Debug().Err(err).Str("container_id", mon.containerID).Msg("docker health check failed, using fallback")
return mon.fallback.CheckHealth()
}
state := interceptedErr.Data.(container.State)
status := state.Status
switch status {
case "dead", "exited", "paused", "restarting", "removing":

View File

@@ -35,12 +35,9 @@ services:
depends_on:
- app
environment:
HOSTNAME: 0.0.0.0
PORT: 3000
GODOXY_API_ADDR: app:8888
labels:
proxy.aliases: ${GODOXY_FRONTEND_ALIASES:-godoxy}
proxy.#1.port: 3000
networks:
- godoxy
app: