Compare commits

..

9 Commits

Author SHA1 Message Date
yusing
6576b7640a chore(deps: update dependencies 2026-02-24 18:14:39 +08:00
yusing
d4e552754e fix(Makefile): update WEBUI_DIR and DOCS_DIR to use absolute paths
Changed WEBUI_DIR to resolve to an absolute path using the current working directory,
and updated DOCS_DIR to be relative to WEBUI_DIR
2026-02-24 18:12:47 +08:00
yusing
9ca2983a52 docs(rules): document new dynamic variables and redaction 2026-02-24 18:11:35 +08:00
yusing
ed2ca236b0 feat(api): accept rule config block string in playground
Update playground request to take rules as a string and parse
either YAML list or DSL config, with tests and swagger updates.
2026-02-24 18:11:17 +08:00
yusing
0eba045104 feat(rules): add $redacted dynamic variable for masking sensitive values
Introduces a new `$redacted` dynamic variable that wraps its single
argument with `strutils.Redact`, allowing sensitive values (e.g.,
authorization headers, query parameters) to be masked in rule
expressions.

The variable accepts exactly one argument, which may itself be a nested
dynamic variable expression such as `$header(Authorization)` or
`$arg(token)`, enabling patterns like `$redacted($header(Authorization))`.

Adds corresponding tests covering plain string redaction, nested header
and query arg wrapping, and the error case when no argument is provided.
2026-02-24 15:17:28 +08:00
yusing
77f2779114 fix(rules): expand nested function args
Allow $func(...) expressions inside function arguments by
extracting nested calls and expanding them before evaluation.
2026-02-24 15:15:42 +08:00
yusing
743eb03b27 fix(scripts): correct repo root path in update-wiki
The repoRootAbs was resolving to the script directory instead of the
repository root. Fixed by resolving two levels up from import.meta.dir.

Also optimized writeImplDocToMdx to skip writes when content is unchanged
and removed unused return value from syncImplDocs.
2026-02-24 15:10:03 +08:00
Yuzerion
d2d686b4d1 feat(rules): introduce block DSL, phase-based execution (#203)
* chore(deps): update submodule goutils

* docs(http): remove default client from README.md

* refactor(rules): introduce block DSL, phase-based execution, and flow validation

- add block syntax parser/scanner with nested @blocks and elif/else support
- restructure rule execution into explicit pre/post phases with phase flags
- classify commands by phase and termination behavior
- enforce flow semantics (default rule handling, dead-rule detection)
- expand HTTP flow coverage with block + YAML parity tests and benches
- refresh rules README/spec and update playground/docs integration
- Default rules act as fallback handlers that execute only when no matching non-default rule exists in the pre phase
- IfElseBlockCommand now returns early when a condition matches with a nil Do block, instead of falling through to else blocks
- Add nil check for auth handler to allow requests when no auth is configured

* fix(rules): buffer log output before writing to stdout/stderr

* refactor(api/rules): remove IsResponseRule field from ParsedRule and related logic

* docs(rules): update examples to use block syntax
2026-02-24 10:44:47 +08:00
yusing
169358659a refactor(middleware): improve response body modification gating
Refactor response body modification to only allow text-like content types
(JSON, YAML, XML, etc.) instead of all HTML responses.

Body modification is now
blocked for binary content and transfer/content encoded responses, while status
code and headers can still be modified.

This prevents issues with compressed or streaming responses while
maintaining the ability to modify text-based API responses.
2026-02-23 23:46:41 +08:00
21 changed files with 790 additions and 382 deletions

View File

@@ -6,8 +6,8 @@ export GOOS = linux
REPO_URL ?= https://github.com/yusing/godoxy
WEBUI_DIR ?= ../godoxy-webui
DOCS_DIR ?= wiki
WEBUI_DIR ?= $(shell pwd)/../godoxy-webui
DOCS_DIR ?= ${WEBUI_DIR}/wiki
ifneq ($(BRANCH), compat)
GO_TAGS = sonic

View File

@@ -46,7 +46,7 @@ require (
github.com/docker/cli v29.2.1+incompatible // indirect
github.com/docker/go-connections v0.6.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/ebitengine/purego v0.9.1 // indirect
github.com/ebitengine/purego v0.10.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/gabriel-vasile/mimetype v1.4.13 // indirect
github.com/gin-contrib/sse v1.1.0 // indirect
@@ -91,8 +91,8 @@ require (
github.com/valyala/fasthttp v1.69.0 // indirect
github.com/yusing/ds v0.4.1 // indirect
github.com/yusing/gointernals v0.2.0 // indirect
github.com/yusing/goutils/http/reverseproxy v0.0.0-20260218062549-0b0fa3a059ec // indirect
github.com/yusing/goutils/http/websocket v0.0.0-20260218062549-0b0fa3a059ec // indirect
github.com/yusing/goutils/http/reverseproxy v0.0.0-20260223150038-3be815cb6e3b // indirect
github.com/yusing/goutils/http/websocket v0.0.0-20260223150038-3be815cb6e3b // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 // indirect

View File

@@ -43,8 +43,8 @@ github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pM
github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/ebitengine/purego v0.9.1 h1:a/k2f2HQU3Pi399RPW1MOaZyhKJL9w/xFpKAg4q1s0A=
github.com/ebitengine/purego v0.9.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU=
github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=

16
go.mod
View File

@@ -58,13 +58,13 @@ require (
github.com/stretchr/testify v1.11.1 // testing framework
github.com/valyala/fasthttp v1.69.0 // fast http for health check
github.com/yusing/ds v0.4.1 // data structures and algorithms
github.com/yusing/godoxy/agent v0.0.0-20260218101334-add7884a365e
github.com/yusing/godoxy/internal/dnsproviders v0.0.0-20260218101334-add7884a365e
github.com/yusing/godoxy/agent v0.0.0-20260224071728-0eba04510480
github.com/yusing/godoxy/internal/dnsproviders v0.0.0-20260224071728-0eba04510480
github.com/yusing/gointernals v0.2.0
github.com/yusing/goutils v0.7.0
github.com/yusing/goutils/http/reverseproxy v0.0.0-20260218062549-0b0fa3a059ec
github.com/yusing/goutils/http/websocket v0.0.0-20260218062549-0b0fa3a059ec
github.com/yusing/goutils/server v0.0.0-20260218062549-0b0fa3a059ec
github.com/yusing/goutils/http/reverseproxy v0.0.0-20260223150038-3be815cb6e3b
github.com/yusing/goutils/http/websocket v0.0.0-20260223150038-3be815cb6e3b
github.com/yusing/goutils/server v0.0.0-20260223150038-3be815cb6e3b
)
require (
@@ -88,7 +88,7 @@ require (
github.com/djherbis/times v1.6.0 // indirect
github.com/docker/go-connections v0.6.0
github.com/docker/go-units v0.5.0 // indirect
github.com/ebitengine/purego v0.9.1 // indirect
github.com/ebitengine/purego v0.10.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/gabriel-vasile/mimetype v1.4.13 // indirect
github.com/go-jose/go-jose/v4 v4.1.3 // indirect
@@ -142,8 +142,8 @@ require (
golang.org/x/sys v0.41.0 // indirect
golang.org/x/text v0.34.0 // indirect
golang.org/x/tools v0.42.0 // indirect
google.golang.org/api v0.267.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260217215200-42d3e9bedb6d // indirect
google.golang.org/api v0.268.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260223185530-2f722ef697dc // indirect
google.golang.org/grpc v1.79.1 // indirect
google.golang.org/protobuf v1.36.11 // indirect
gopkg.in/ini.v1 v1.67.1 // indirect

12
go.sum
View File

@@ -82,8 +82,8 @@ github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pM
github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/ebitengine/purego v0.9.1 h1:a/k2f2HQU3Pi399RPW1MOaZyhKJL9w/xFpKAg4q1s0A=
github.com/ebitengine/purego v0.9.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU=
github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
github.com/elliotwutingfeng/asciiset v0.0.0-20230602022725-51bbb787efab h1:h1UgjJdAAhj+uPL68n7XASS6bU+07ZX1WJvVS2eyoeY=
github.com/elliotwutingfeng/asciiset v0.0.0-20230602022725-51bbb787efab/go.mod h1:GLo/8fDswSAniFG+BFIaiSPcK610jyzgEhWYPQwuQdw=
github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
@@ -447,14 +447,14 @@ golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
google.golang.org/api v0.267.0 h1:w+vfWPMPYeRs8qH1aYYsFX68jMls5acWl/jocfLomwE=
google.golang.org/api v0.267.0/go.mod h1:Jzc0+ZfLnyvXma3UtaTl023TdhZu6OMBP9tJ+0EmFD0=
google.golang.org/api v0.268.0 h1:hgA3aS4lt9rpF5RCCkX0Q2l7DvHgvlb53y4T4u6iKkA=
google.golang.org/api v0.268.0/go.mod h1:HXMyMH496wz+dAJwD/GkAPLd3ZL33Kh0zEG32eNvy9w=
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 h1:VQZ/yAbAtjkHgH80teYd2em3xtIkkHd7ZhqfH2N9CsM=
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409/go.mod h1:rxKD3IEILWEu3P44seeNOAwZN4SaoKaQ/2eTg4mM6EM=
google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 h1:merA0rdPeUV3YIIfHHcH4qBkiQAc1nfCKSI7lB4cV2M=
google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409/go.mod h1:fl8J1IvUjCilwZzQowmw2b7HQB2eAuYBabMXzWurF+I=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260217215200-42d3e9bedb6d h1:t/LOSXPJ9R0B6fnZNyALBRfZBH0Uy0gT+uR+SJ6syqQ=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260217215200-42d3e9bedb6d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260223185530-2f722ef697dc h1:51Wupg8spF+5FC6D+iMKbOddFjMckETnNnEiZ+HX37s=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260223185530-2f722ef697dc/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
google.golang.org/grpc v1.79.1 h1:zGhSi45ODB9/p3VAawt9a+O/MULLl9dpizzNNpq7flY=
google.golang.org/grpc v1.79.1/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=

View File

@@ -5125,10 +5125,7 @@
"$ref": "#/definitions/MockResponse"
},
"rules": {
"type": "array",
"items": {
"$ref": "#/definitions/routeApi.RawRule"
},
"type": "string",
"x-nullable": false,
"x-omitempty": false
}
@@ -6926,28 +6923,6 @@
"x-nullable": false,
"x-omitempty": false
},
"routeApi.RawRule": {
"type": "object",
"properties": {
"do": {
"type": "string",
"x-nullable": false,
"x-omitempty": false
},
"name": {
"type": "string",
"x-nullable": false,
"x-omitempty": false
},
"on": {
"type": "string",
"x-nullable": false,
"x-omitempty": false
}
},
"x-nullable": false,
"x-omitempty": false
},
"routeApi.RoutesByProvider": {
"type": "object",
"additionalProperties": {

View File

@@ -905,9 +905,7 @@ definitions:
mockResponse:
$ref: '#/definitions/MockResponse'
rules:
items:
$ref: '#/definitions/routeApi.RawRule'
type: array
type: string
required:
- rules
type: object
@@ -1858,15 +1856,6 @@ definitions:
uptime:
type: string
type: object
routeApi.RawRule:
properties:
do:
type: string
name:
type: string
"on":
type: string
type: object
routeApi.RoutesByProvider:
additionalProperties:
items:

View File

@@ -9,6 +9,7 @@ import (
"strings"
"github.com/gin-gonic/gin"
"github.com/goccy/go-yaml"
"github.com/yusing/godoxy/internal/common"
"github.com/yusing/godoxy/internal/route/rules"
apitypes "github.com/yusing/goutils/apitypes"
@@ -23,7 +24,7 @@ type RawRule struct {
}
type PlaygroundRequest struct {
Rules []RawRule `json:"rules" binding:"required"`
Rules string `json:"rules" binding:"required"`
MockRequest MockRequest `json:"mockRequest"`
MockResponse MockResponse `json:"mockResponse"`
} // @name PlaygroundRequest
@@ -255,7 +256,35 @@ func handlerWithRecover(w http.ResponseWriter, r *http.Request, h http.HandlerFu
h(w, r)
}
func parseRules(rawRules []RawRule) ([]ParsedRule, rules.Rules, error) {
func parseRules(config string) ([]ParsedRule, rules.Rules, error) {
config = strings.TrimSpace(config)
if config == "" {
return []ParsedRule{}, nil, nil
}
var rawRules []RawRule
if err := yaml.Unmarshal([]byte(config), &rawRules); err == nil && len(rawRules) > 0 {
return parseRawRules(rawRules)
}
var rulesList rules.Rules
if err := rulesList.Parse(config); err != nil {
return nil, nil, err
}
parsedRules := make([]ParsedRule, 0, len(rulesList))
for _, rule := range rulesList {
parsedRules = append(parsedRules, ParsedRule{
Name: rule.Name,
On: rule.On.String(),
Do: rule.Do.String(),
})
}
return parsedRules, rulesList, nil
}
func parseRawRules(rawRules []RawRule) ([]ParsedRule, rules.Rules, error) {
parsedRules := make([]ParsedRule, 0, len(rawRules))
rulesList := make(rules.Rules, 0, len(rawRules))

View File

@@ -22,13 +22,10 @@ func TestPlayground(t *testing.T) {
{
name: "simple path matching rule",
request: PlaygroundRequest{
Rules: []RawRule{
{
Name: "test rule",
On: "path /api",
Do: "pass",
},
},
Rules: `- name: test rule
on: path /api
do: pass
`,
MockRequest: MockRequest{
Method: "GET",
Path: "/api",
@@ -53,13 +50,10 @@ func TestPlayground(t *testing.T) {
{
name: "header matching rule",
request: PlaygroundRequest{
Rules: []RawRule{
{
Name: "check user agent",
On: "header User-Agent Chrome",
Do: "error 403 Forbidden",
},
},
Rules: `- name: check user agent
on: header User-Agent Chrome
do: error 403 Forbidden
`,
MockRequest: MockRequest{
Method: "GET",
Path: "/",
@@ -90,13 +84,10 @@ func TestPlayground(t *testing.T) {
{
name: "invalid rule syntax",
request: PlaygroundRequest{
Rules: []RawRule{
{
Name: "bad rule",
On: "invalid_checker something",
Do: "pass",
},
},
Rules: `- name: bad rule
on: invalid_checker something
do: pass
`,
MockRequest: MockRequest{
Method: "GET",
Path: "/",
@@ -115,13 +106,10 @@ func TestPlayground(t *testing.T) {
{
name: "rewrite path rule",
request: PlaygroundRequest{
Rules: []RawRule{
{
Name: "rewrite rule",
On: "path glob(/api/*)",
Do: "rewrite /api/ /v1/",
},
},
Rules: `- name: rewrite rule
on: path glob(/api/*)
do: rewrite /api/ /v1/
`,
MockRequest: MockRequest{
Method: "GET",
Path: "/api/users",
@@ -148,13 +136,10 @@ func TestPlayground(t *testing.T) {
{
name: "method matching rule",
request: PlaygroundRequest{
Rules: []RawRule{
{
Name: "block POST",
On: "method POST",
Do: `error "405" "Method Not Allowed"`,
},
},
Rules: `- name: block POST
on: method POST
do: error "405" "Method Not Allowed"
`,
MockRequest: MockRequest{
Method: "POST",
Path: "/api",
@@ -173,6 +158,63 @@ func TestPlayground(t *testing.T) {
}
},
},
{
name: "block syntax default rule",
request: PlaygroundRequest{
Rules: `default {
pass
}`,
MockRequest: MockRequest{
Method: "GET",
Path: "/",
},
},
wantStatusCode: http.StatusOK,
checkResponse: func(t *testing.T, resp PlaygroundResponse) {
if len(resp.ParsedRules) != 1 {
t.Errorf("expected 1 parsed rule, got %d", len(resp.ParsedRules))
}
if resp.ParsedRules[0].ValidationError != nil {
t.Errorf("expected rule to be valid, got error: %v", resp.ParsedRules[0].ValidationError)
}
if !resp.UpstreamCalled {
t.Error("expected upstream to be called")
}
},
},
{
name: "block syntax conditional rule",
request: PlaygroundRequest{
Rules: `header User-Agent Chrome {
error 403 Forbidden
}`,
MockRequest: MockRequest{
Method: "GET",
Path: "/",
Headers: map[string][]string{
"User-Agent": {"Chrome"},
},
},
},
wantStatusCode: http.StatusOK,
checkResponse: func(t *testing.T, resp PlaygroundResponse) {
if len(resp.ParsedRules) != 1 {
t.Errorf("expected 1 parsed rule, got %d", len(resp.ParsedRules))
}
if resp.ParsedRules[0].ValidationError != nil {
t.Errorf("expected rule to be valid, got error: %v", resp.ParsedRules[0].ValidationError)
}
if len(resp.MatchedRules) != 1 {
t.Errorf("expected 1 matched rule, got %d", len(resp.MatchedRules))
}
if resp.FinalResponse.StatusCode != http.StatusForbidden {
t.Errorf("expected status 403, got %d", resp.FinalResponse.StatusCode)
}
if resp.UpstreamCalled {
t.Error("expected upstream not to be called")
}
},
},
}
for _, tt := range tests {

View File

@@ -98,8 +98,8 @@ require (
golang.org/x/sys v0.41.0 // indirect
golang.org/x/text v0.34.0 // indirect
golang.org/x/tools v0.42.0 // indirect
google.golang.org/api v0.267.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260217215200-42d3e9bedb6d // indirect
google.golang.org/api v0.268.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260223185530-2f722ef697dc // indirect
google.golang.org/grpc v1.79.1 // indirect
google.golang.org/protobuf v1.36.11 // indirect
gopkg.in/ini.v1 v1.67.1 // indirect

View File

@@ -249,14 +249,14 @@ golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
google.golang.org/api v0.267.0 h1:w+vfWPMPYeRs8qH1aYYsFX68jMls5acWl/jocfLomwE=
google.golang.org/api v0.267.0/go.mod h1:Jzc0+ZfLnyvXma3UtaTl023TdhZu6OMBP9tJ+0EmFD0=
google.golang.org/api v0.268.0 h1:hgA3aS4lt9rpF5RCCkX0Q2l7DvHgvlb53y4T4u6iKkA=
google.golang.org/api v0.268.0/go.mod h1:HXMyMH496wz+dAJwD/GkAPLd3ZL33Kh0zEG32eNvy9w=
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 h1:VQZ/yAbAtjkHgH80teYd2em3xtIkkHd7ZhqfH2N9CsM=
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409/go.mod h1:rxKD3IEILWEu3P44seeNOAwZN4SaoKaQ/2eTg4mM6EM=
google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 h1:merA0rdPeUV3YIIfHHcH4qBkiQAc1nfCKSI7lB4cV2M=
google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409/go.mod h1:fl8J1IvUjCilwZzQowmw2b7HQB2eAuYBabMXzWurF+I=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260217215200-42d3e9bedb6d h1:t/LOSXPJ9R0B6fnZNyALBRfZBH0Uy0gT+uR+SJ6syqQ=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260217215200-42d3e9bedb6d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260223185530-2f722ef697dc h1:51Wupg8spF+5FC6D+iMKbOddFjMckETnNnEiZ+HX37s=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260223185530-2f722ef697dc/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
google.golang.org/grpc v1.79.1 h1:zGhSi45ODB9/p3VAawt9a+O/MULLl9dpizzNNpq7flY=
google.golang.org/grpc v1.79.1/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=

View File

@@ -13,6 +13,8 @@ This package implements a flexible HTTP middleware system for GoDoxy. Middleware
- **Bypass Rules**: Skip middleware based on request properties
- **Dynamic Loading**: Load middleware definitions from files at runtime
Response body rewriting is only applied to unencoded, text-like content types (for example `text/*`, JSON, YAML, XML). Response status and headers can always be modified.
## Architecture
```mermaid

View File

@@ -6,6 +6,7 @@ import (
"net/http"
"reflect"
"sort"
"strings"
"github.com/bytedance/sonic"
"github.com/rs/zerolog"
@@ -195,21 +196,15 @@ func (m *Middleware) ServeHTTP(next http.HandlerFunc, w http.ResponseWriter, r *
}
if exec, ok := m.impl.(ResponseModifier); ok {
lrm := httputils.NewLazyResponseModifier(w, needsBuffering)
rm := httputils.NewResponseModifier(w)
defer func() {
_, err := lrm.FlushRelease()
_, err := rm.FlushRelease()
if err != nil {
m.LogError(r).Err(err).Msg("failed to flush response")
}
}()
next(lrm, r)
next(rm, r)
// Skip modification if response wasn't buffered (non-HTML content)
if !lrm.IsBuffered() {
return
}
rm := lrm.ResponseModifier()
currentBody := rm.BodyReader()
currentResp := &http.Response{
StatusCode: rm.StatusCode(),
@@ -218,20 +213,31 @@ func (m *Middleware) ServeHTTP(next http.HandlerFunc, w http.ResponseWriter, r *
Body: currentBody,
Request: r,
}
if err := exec.modifyResponse(currentResp); err != nil {
allowBodyModification := canModifyResponseBody(currentResp)
respToModify := currentResp
if !allowBodyModification {
shadow := *currentResp
shadow.Body = eofReader{}
respToModify = &shadow
}
if err := exec.modifyResponse(respToModify); err != nil {
log.Err(err).Str("middleware", m.Name()).Str("url", fullURL(r)).Msg("failed to modify response")
}
// override the response status code
rm.WriteHeader(currentResp.StatusCode)
rm.WriteHeader(respToModify.StatusCode)
// overriding the response header
maps.Copy(rm.Header(), currentResp.Header)
maps.Copy(rm.Header(), respToModify.Header)
// override the content length and body if changed
if currentResp.Body != currentBody {
if err := rm.SetBody(currentResp.Body); err != nil {
m.LogError(r).Err(err).Msg("failed to set response body")
if respToModify.Body != currentBody {
if allowBodyModification {
if err := rm.SetBody(respToModify.Body); err != nil {
m.LogError(r).Err(err).Msg("failed to set response body")
}
} else {
respToModify.Body.Close()
}
}
} else {
@@ -239,10 +245,55 @@ func (m *Middleware) ServeHTTP(next http.HandlerFunc, w http.ResponseWriter, r *
}
}
// needsBuffering determines if a response should be buffered for modification.
// Only HTML responses need buffering; streaming content (video, audio, etc.) should pass through.
func needsBuffering(header http.Header) bool {
return httputils.GetContentType(header).IsHTML()
func canModifyResponseBody(resp *http.Response) bool {
if hasNonIdentityEncoding(resp.TransferEncoding) {
return false
}
if hasNonIdentityEncoding(resp.Header.Values("Transfer-Encoding")) {
return false
}
if hasNonIdentityEncoding(resp.Header.Values("Content-Encoding")) {
return false
}
return isTextLikeMediaType(string(httputils.GetContentType(resp.Header)))
}
func hasNonIdentityEncoding(values []string) bool {
for _, value := range values {
for _, token := range strings.Split(value, ",") {
if strings.TrimSpace(token) == "" || strings.EqualFold(strings.TrimSpace(token), "identity") {
continue
}
return true
}
}
return false
}
func isTextLikeMediaType(contentType string) bool {
if contentType == "" {
return false
}
contentType = strings.ToLower(contentType)
if strings.HasPrefix(contentType, "text/") {
return true
}
if contentType == "application/json" || strings.HasSuffix(contentType, "+json") {
return true
}
if contentType == "application/xml" || strings.HasSuffix(contentType, "+xml") {
return true
}
if strings.Contains(contentType, "yaml") || strings.Contains(contentType, "toml") {
return true
}
if strings.Contains(contentType, "javascript") || strings.Contains(contentType, "ecmascript") {
return true
}
if strings.Contains(contentType, "csv") {
return true
}
return contentType == "application/x-www-form-urlencoded"
}
func (m *Middleware) LogWarn(req *http.Request) *zerolog.Event {

View File

@@ -1,6 +1,7 @@
package middleware
import (
"maps"
"net/http"
"strconv"
@@ -46,10 +47,23 @@ func (m *middlewareChain) modifyResponse(resp *http.Response) error {
if len(m.modResps) == 0 {
return nil
}
allowBodyModification := canModifyResponseBody(resp)
for i, mr := range m.modResps {
if err := mr.modifyResponse(resp); err != nil {
respToModify := resp
if !allowBodyModification {
shadow := *resp
shadow.Body = eofReader{}
respToModify = &shadow
}
if err := mr.modifyResponse(respToModify); err != nil {
return gperr.PrependSubject(err, strconv.Itoa(i))
}
if !allowBodyModification {
resp.StatusCode = respToModify.StatusCode
if respToModify.Header != nil {
maps.Copy(resp.Header, respToModify.Header)
}
}
}
return nil
}

View File

@@ -1,6 +1,7 @@
package middleware
import (
"io"
"net/http"
"strconv"
"strings"
@@ -14,12 +15,27 @@ type testPriority struct {
}
var test = NewMiddleware[testPriority]()
var responseRewrite = NewMiddleware[testResponseRewrite]()
func (t testPriority) before(w http.ResponseWriter, r *http.Request) bool {
w.Header().Add("Test-Value", strconv.Itoa(t.Value))
return true
}
type testResponseRewrite struct {
StatusCode int `json:"status_code"`
HeaderKey string `json:"header_key"`
HeaderVal string `json:"header_val"`
Body string `json:"body"`
}
func (t testResponseRewrite) modifyResponse(resp *http.Response) error {
resp.StatusCode = t.StatusCode
resp.Header.Set(t.HeaderKey, t.HeaderVal)
resp.Body = io.NopCloser(strings.NewReader(t.Body))
return nil
}
func TestMiddlewarePriority(t *testing.T) {
priorities := []int{4, 7, 9, 0}
chain := make([]*Middleware, len(priorities))
@@ -35,3 +51,85 @@ func TestMiddlewarePriority(t *testing.T) {
expect.NoError(t, err)
expect.Equal(t, strings.Join(res.ResponseHeaders["Test-Value"], ","), "3,0,1,2")
}
func TestMiddlewareResponseRewriteGate(t *testing.T) {
opts := OptionsRaw{
"status_code": 418,
"header_key": "X-Rewrite",
"header_val": "1",
"body": "rewritten-body",
}
tests := []struct {
name string
respHeaders http.Header
respBody []byte
expectBody string
}{
{
name: "allow_body_rewrite_for_html",
respHeaders: http.Header{
"Content-Type": []string{"text/html; charset=utf-8"},
},
respBody: []byte("<html><body>original</body></html>"),
expectBody: "rewritten-body",
},
{
name: "allow_body_rewrite_for_json",
respHeaders: http.Header{
"Content-Type": []string{"application/json"},
},
respBody: []byte(`{"message":"original"}`),
expectBody: "rewritten-body",
},
{
name: "allow_body_rewrite_for_yaml",
respHeaders: http.Header{
"Content-Type": []string{"application/yaml"},
},
respBody: []byte("k: v"),
expectBody: "rewritten-body",
},
{
name: "block_body_rewrite_for_binary_content",
respHeaders: http.Header{
"Content-Type": []string{"application/octet-stream"},
},
respBody: []byte("binary"),
expectBody: "binary",
},
{
name: "block_body_rewrite_for_transfer_encoded_html",
respHeaders: http.Header{
"Content-Type": []string{"text/html"},
"Transfer-Encoding": []string{"chunked"},
},
respBody: []byte("<html><body>original</body></html>"),
expectBody: "<html><body>original</body></html>",
},
{
name: "block_body_rewrite_for_content_encoded_html",
respHeaders: http.Header{
"Content-Type": []string{"text/html"},
"Content-Encoding": []string{"gzip"},
},
respBody: []byte("<html><body>original</body></html>"),
expectBody: "<html><body>original</body></html>",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
result, err := newMiddlewareTest(responseRewrite, &testArgs{
middlewareOpt: opts,
respHeaders: tc.respHeaders,
respBody: tc.respBody,
respStatus: http.StatusOK,
})
expect.NoError(t, err)
expect.Equal(t, result.ResponseStatus, 418)
expect.Equal(t, result.ResponseHeaders.Get("X-Rewrite"), "1")
expect.Equal(t, string(result.Data), tc.expectBody)
})
}
}

View File

@@ -7,6 +7,7 @@ import (
"maps"
"net/http"
"net/http/httptest"
"strings"
"github.com/bytedance/sonic"
"github.com/yusing/godoxy/internal/common"
@@ -54,7 +55,7 @@ func (rt *requestRecorder) RoundTrip(req *http.Request) (resp *http.Response, er
resp = &http.Response{
Status: http.StatusText(rt.args.respStatus),
StatusCode: rt.args.respStatus,
Header: testHeaders,
Header: maps.Clone(testHeaders),
Body: io.NopCloser(bytes.NewReader(rt.args.respBody)),
ContentLength: int64(len(rt.args.respBody)),
Request: req,
@@ -65,9 +66,27 @@ func (rt *requestRecorder) RoundTrip(req *http.Request) (resp *http.Response, er
return nil, err
}
maps.Copy(resp.Header, rt.args.respHeaders)
if transferEncoding := resp.Header.Values("Transfer-Encoding"); len(transferEncoding) > 0 {
resp.TransferEncoding = parseHeaderTokens(transferEncoding)
resp.ContentLength = -1
}
return resp, nil
}
func parseHeaderTokens(values []string) []string {
var tokens []string
for _, value := range values {
for token := range strings.SplitSeq(value, ",") {
token = strings.TrimSpace(token)
if token == "" {
continue
}
tokens = append(tokens, token)
}
}
return tokens
}
type TestResult struct {
RequestHeaders http.Header
ResponseHeaders http.Header

View File

@@ -439,8 +439,19 @@ $remote_host # Client IP
# Dynamic variables
$header(Name) # Request header
$header(Name, index) # Header at index
$resp_header(Name) # Response header
$arg(Name) # Query argument
$form(Name) # Form field
$postform(Name) # POST form field
$cookie(Name) # Cookie value
# Function composition: pass result of one function to another
$redacted($header(Authorization)) # Redact the Authorization header value
$redacted($arg(token)) # Redact a query parameter value
$redacted($cookie(session)) # Redact a cookie value
# $redacted: masks a value, showing only first 2 and last 2 characters
$redacted(value) # Redact a plain string
# Environment variables
${ENV_VAR}

View File

@@ -152,6 +152,12 @@ func ExpandVars(w *httputils.ResponseModifier, req *http.Request, src string, ds
return phase, err
}
i = nextIdx
// Expand any nested $func(...) expressions in args
args, argPhase, err := expandArgs(args, w, req)
if err != nil {
return phase, err
}
phase |= argPhase
actual, err = getter.get(args, w, req)
if err != nil {
return phase, err
@@ -221,6 +227,18 @@ func extractArgs(src string, i int, funcName string) (args []string, nextIdx int
continue
}
// Nested function call: $func(...) as an argument
if ch == '$' && arg.Len() == 0 {
// Capture the entire $func(...) expression as a raw argument token
nestedEnd, nestedErr := extractNestedFuncExpr(src, nextIdx)
if nestedErr != nil {
return nil, 0, nestedErr
}
args = append(args, src[nextIdx:nestedEnd+1])
nextIdx = nestedEnd + 1
continue
}
if ch == ')' {
// End of arguments
if arg.Len() > 0 {
@@ -256,3 +274,70 @@ func extractArgs(src string, i int, funcName string) (args []string, nextIdx int
}
return nil, 0, ErrUnterminatedParenthesis.Withf("func %q", funcName)
}
// extractNestedFuncExpr finds the end index (inclusive) of a $func(...) expression
// starting at position start in src. It handles nested parentheses.
func extractNestedFuncExpr(src string, start int) (endIdx int, err error) {
// src[start] must be '$'
i := start + 1
// skip the function name (valid var name chars)
for i < len(src) && validVarNameCharset[src[i]] {
i++
}
if i >= len(src) || src[i] != '(' {
return 0, ErrUnterminatedParenthesis.Withf("nested func at position %d", start)
}
// Now find the matching closing parenthesis, respecting quotes and nesting
depth := 0
var quote byte
for i < len(src) {
ch := src[i]
if quote != 0 {
if ch == quote {
quote = 0
}
i++
continue
}
if quoteChars[ch] {
quote = ch
i++
continue
}
switch ch {
case '(':
depth++
case ')':
depth--
if depth == 0 {
return i, nil
}
}
i++
}
if quote != 0 {
return 0, ErrUnterminatedQuotes.Withf("nested func at position %d", start)
}
return 0, ErrUnterminatedParenthesis.Withf("nested func at position %d", start)
}
// expandArgs expands any args that are nested dynamic var expressions (starting with '$').
// It returns the expanded args and the combined phase flags.
func expandArgs(args []string, w *httputils.ResponseModifier, req *http.Request) (expanded []string, phase PhaseFlag, err error) {
expanded = make([]string, len(args))
for i, arg := range args {
if len(arg) > 0 && arg[0] == '$' {
var buf strings.Builder
var argPhase PhaseFlag
argPhase, err = ExpandVars(w, req, arg, &buf)
if err != nil {
return nil, phase, err
}
phase |= argPhase
expanded[i] = buf.String()
} else {
expanded[i] = arg
}
}
return expanded, phase, nil
}

View File

@@ -6,6 +6,7 @@ import (
"strconv"
httputils "github.com/yusing/goutils/http"
strutils "github.com/yusing/goutils/strings"
)
var (
@@ -15,6 +16,7 @@ var (
VarQuery = "arg"
VarForm = "form"
VarPostForm = "postform"
VarRedacted = "redacted"
)
type dynamicVarGetter struct {
@@ -94,6 +96,17 @@ var dynamicVarSubsMap = map[string]dynamicVarGetter{
return getValueByKeyAtIndex(req.PostForm, key, index)
},
},
// VarRedacted wraps the result of its single argument (which may be another dynamic var
// expression, already expanded by expandArgs) with strutils.Redact.
VarRedacted: {
phase: PhaseNone,
get: func(args []string, w *httputils.ResponseModifier, req *http.Request) (string, error) {
if len(args) != 1 {
return "", ErrExpectOneArg
}
return strutils.Redact(args[0]), nil
},
},
}
func getValueByKeyAtIndex[Values http.Header | url.Values](values Values, key string, index int) (string, error) {

View File

@@ -189,6 +189,64 @@ func TestExtractArgs(t *testing.T) {
}
}
func TestExtractArgs_NestedFunc(t *testing.T) {
tests := []struct {
name string
src string
startPos int
funcName string
wantArgs []string
wantNextIdx int
wantErr bool
}{
{
name: "nested func as single arg",
src: "redacted($header(Authorization))",
startPos: 0,
funcName: "redacted",
wantArgs: []string{"$header(Authorization)"},
wantNextIdx: 31,
},
{
name: "nested func with quoted arg inside",
src: `redacted($header("X-Secret"))`,
startPos: 0,
funcName: "redacted",
wantArgs: []string{`$header("X-Secret")`},
wantNextIdx: 28,
},
{
name: "nested func with two args inside",
src: "redacted($header(X-Multi, 1))",
startPos: 0,
funcName: "redacted",
wantArgs: []string{"$header(X-Multi, 1)"},
wantNextIdx: 28,
},
{
name: "nested func missing closing paren",
src: "redacted($header(Authorization)",
startPos: 0,
funcName: "redacted",
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
args, nextIdx, err := extractArgs(tt.src, tt.startPos, tt.funcName)
if tt.wantErr {
require.Error(t, err)
} else {
require.NoError(t, err)
require.Equal(t, tt.wantArgs, args)
require.Equal(t, tt.wantNextIdx, nextIdx)
}
})
}
}
func TestExpandVars(t *testing.T) {
// Create a comprehensive test request with form data
formData := url.Values{}
@@ -446,6 +504,27 @@ func TestExpandVars(t *testing.T) {
input: "Header: $header(User-Agent), Status: $status_code",
want: "Header: test-agent/1.0, Status: 200",
},
// $redacted function
{
name: "redacted with plain string arg",
input: "$redacted(secret)",
want: "se**et",
},
{
name: "redacted wrapping header",
input: "$redacted($header(User-Agent))",
want: "te**********.0",
},
{
name: "redacted wrapping arg",
input: "$redacted($arg(param1))",
want: "va**e1",
},
{
name: "redacted with no args",
input: "$redacted()",
wantErr: true,
},
// Escaped dollar signs
{
name: "escaped dollar",

View File

@@ -4,335 +4,336 @@ import { Glob } from "bun";
import { md2mdx } from "./api-md2mdx";
type ImplDoc = {
/** Directory path relative to this repo, e.g. "internal/health/check" */
pkgPath: string;
/** File name in wiki `src/impl/`, e.g. "internal-health-check.md" */
docFileName: string;
/** VitePress route path (extensionless), e.g. "/impl/internal-health-check" */
docRoute: string;
/** Absolute source README path */
srcPathAbs: string;
/** Absolute destination doc path */
dstPathAbs: string;
/** Directory path relative to this repo, e.g. "internal/health/check" */
pkgPath: string;
/** File name in wiki `src/impl/`, e.g. "internal-health-check.md" */
docFileName: string;
/** VitePress route path (extensionless), e.g. "/impl/internal-health-check" */
docRoute: string;
/** Absolute source README path */
srcPathAbs: string;
/** Absolute destination doc path */
dstPathAbs: string;
};
const skipSubmodules = [
"internal/go-oidc/",
"internal/gopsutil/",
"internal/go-proxmox/",
"internal/go-oidc/",
"internal/gopsutil/",
"internal/go-proxmox/",
];
function normalizeRepoUrl(raw: string) {
let url = (raw ?? "").trim();
if (!url) return "";
// Common typo: "https://https://github.com/..."
url = url.replace(/^https?:\/\/https?:\/\//i, "https://");
if (!/^https?:\/\//i.test(url)) url = `https://${url}`;
url = url.replace(/\/+$/, "");
return url;
let url = (raw ?? "").trim();
if (!url) return "";
// Common typo: "https://https://github.com/..."
url = url.replace(/^https?:\/\/https?:\/\//i, "https://");
if (!/^https?:\/\//i.test(url)) url = `https://${url}`;
url = url.replace(/\/+$/, "");
return url;
}
function sanitizeFileStemFromPkgPath(pkgPath: string) {
// Convert a package path into a stable filename.
// Example: "internal/go-oidc/example" -> "internal-go-oidc-example"
// Keep it readable and unique (uses full path).
const parts = pkgPath
.split("/")
.filter(Boolean)
.map((p) => p.replace(/[^A-Za-z0-9._-]+/g, "-"));
const joined = parts.join("-");
return joined.replace(/-+/g, "-").replace(/^-|-$/g, "");
// Convert a package path into a stable filename.
// Example: "internal/go-oidc/example" -> "internal-go-oidc-example"
// Keep it readable and unique (uses full path).
const parts = pkgPath
.split("/")
.filter(Boolean)
.map((p) => p.replace(/[^A-Za-z0-9._-]+/g, "-"));
const joined = parts.join("-");
return joined.replace(/-+/g, "-").replace(/^-|-$/g, "");
}
function splitUrlAndFragment(url: string): {
urlNoFragment: string;
fragment: string;
urlNoFragment: string;
fragment: string;
} {
const i = url.indexOf("#");
if (i === -1) return { urlNoFragment: url, fragment: "" };
return { urlNoFragment: url.slice(0, i), fragment: url.slice(i) };
const i = url.indexOf("#");
if (i === -1) return { urlNoFragment: url, fragment: "" };
return { urlNoFragment: url.slice(0, i), fragment: url.slice(i) };
}
function isExternalOrAbsoluteUrl(url: string) {
// - absolute site links: "/foo"
// - pure fragments: "#bar"
// - external schemes: "https:", "mailto:", "vscode:", etc.
// IMPORTANT: don't treat "config.go:29" as a scheme.
if (url.startsWith("/") || url.startsWith("#")) return true;
if (url.includes("://")) return true;
return /^(https?|mailto|tel|vscode|file|data|ssh|git):/i.test(url);
// - absolute site links: "/foo"
// - pure fragments: "#bar"
// - external schemes: "https:", "mailto:", "vscode:", etc.
// IMPORTANT: don't treat "config.go:29" as a scheme.
if (url.startsWith("/") || url.startsWith("#")) return true;
if (url.includes("://")) return true;
return /^(https?|mailto|tel|vscode|file|data|ssh|git):/i.test(url);
}
function isRepoSourceFilePath(filePath: string) {
// Conservative allow-list: avoid rewriting .md (non-README) which may be VitePress docs.
return /\.(go|ts|tsx|js|jsx|py|sh|yml|yaml|json|toml|env|css|html|txt)$/i.test(
filePath,
);
// Conservative allow-list: avoid rewriting .md (non-README) which may be VitePress docs.
return /\.(go|ts|tsx|js|jsx|py|sh|yml|yaml|json|toml|env|css|html|txt)$/i.test(
filePath,
);
}
function parseFileLineSuffix(urlNoFragment: string): {
filePath: string;
line?: string;
filePath: string;
line?: string;
} {
// Match "file.ext:123" (line suffix), while leaving "file.ext" untouched.
const m = urlNoFragment.match(/^(.*?):(\d+)$/);
if (!m) return { filePath: urlNoFragment };
return { filePath: m[1] ?? urlNoFragment, line: m[2] };
// Match "file.ext:123" (line suffix), while leaving "file.ext" untouched.
const m = urlNoFragment.match(/^(.*?):(\d+)$/);
if (!m) return { filePath: urlNoFragment };
return { filePath: m[1] ?? urlNoFragment, line: m[2] };
}
function rewriteMarkdownLinksOutsideFences(
md: string,
rewriteInline: (url: string) => string,
md: string,
rewriteInline: (url: string) => string,
) {
const lines = md.split("\n");
let inFence = false;
const lines = md.split("\n");
let inFence = false;
for (let i = 0; i < lines.length; i++) {
const line = lines[i] ?? "";
const trimmed = line.trimStart();
if (trimmed.startsWith("```")) {
inFence = !inFence;
continue;
}
if (inFence) continue;
for (let i = 0; i < lines.length; i++) {
const line = lines[i] ?? "";
const trimmed = line.trimStart();
if (trimmed.startsWith("```")) {
inFence = !inFence;
continue;
}
if (inFence) continue;
// Inline markdown links/images: [text](url "title") / ![alt](url)
lines[i] = line.replace(
/\]\(([^)\s]+)(\s+"[^"]*")?\)/g,
(_full, urlRaw: string, maybeTitle: string | undefined) => {
const rewritten = rewriteInline(urlRaw);
return `](${rewritten}${maybeTitle ?? ""})`;
},
);
}
// Inline markdown links/images: [text](url "title") / ![alt](url)
lines[i] = line.replace(
/\]\(([^)\s]+)(\s+"[^"]*")?\)/g,
(_full, urlRaw: string, maybeTitle: string | undefined) => {
const rewritten = rewriteInline(urlRaw);
return `](${rewritten}${maybeTitle ?? ""})`;
},
);
}
return lines.join("\n");
return lines.join("\n");
}
function rewriteImplMarkdown(params: {
md: string;
pkgPath: string;
readmeRelToDocRoute: Map<string, string>;
dirPathToDocRoute: Map<string, string>;
repoUrl: string;
md: string;
pkgPath: string;
readmeRelToDocRoute: Map<string, string>;
dirPathToDocRoute: Map<string, string>;
repoUrl: string;
}) {
const { md, pkgPath, readmeRelToDocRoute, dirPathToDocRoute, repoUrl } =
params;
const { md, pkgPath, readmeRelToDocRoute, dirPathToDocRoute, repoUrl } =
params;
return rewriteMarkdownLinksOutsideFences(md, (urlRaw) => {
// Handle angle-bracketed destinations: (<./foo/README.md>)
const angleWrapped =
urlRaw.startsWith("<") && urlRaw.endsWith(">")
? urlRaw.slice(1, -1)
: urlRaw;
return rewriteMarkdownLinksOutsideFences(md, (urlRaw) => {
// Handle angle-bracketed destinations: (<./foo/README.md>)
const angleWrapped =
urlRaw.startsWith("<") && urlRaw.endsWith(">")
? urlRaw.slice(1, -1)
: urlRaw;
const { urlNoFragment, fragment } = splitUrlAndFragment(angleWrapped);
if (!urlNoFragment) return urlRaw;
if (isExternalOrAbsoluteUrl(urlNoFragment)) return urlRaw;
const { urlNoFragment, fragment } = splitUrlAndFragment(angleWrapped);
if (!urlNoFragment) return urlRaw;
if (isExternalOrAbsoluteUrl(urlNoFragment)) return urlRaw;
// 1) Directory links like "common" or "common/" that have a README
const dirPathNormalized = urlNoFragment.replace(/\/+$/, "");
let rewritten: string | undefined;
// First try exact match
if (dirPathToDocRoute.has(dirPathNormalized)) {
rewritten = `${dirPathToDocRoute.get(dirPathNormalized)}${fragment}`;
} else {
// Fallback: check parent directories for a README
// This handles paths like "internal/watcher/events" where only the parent has a README
let parentPath = dirPathNormalized;
while (parentPath.includes("/")) {
parentPath = parentPath.slice(0, parentPath.lastIndexOf("/"));
if (dirPathToDocRoute.has(parentPath)) {
rewritten = `${dirPathToDocRoute.get(parentPath)}${fragment}`;
break;
}
}
}
if (rewritten) {
return angleWrapped === urlRaw ? rewritten : `<${rewritten}>`;
}
// 1) Directory links like "common" or "common/" that have a README
const dirPathNormalized = urlNoFragment.replace(/\/+$/, "");
let rewritten: string | undefined;
// First try exact match
if (dirPathToDocRoute.has(dirPathNormalized)) {
rewritten = `${dirPathToDocRoute.get(dirPathNormalized)}${fragment}`;
} else {
// Fallback: check parent directories for a README
// This handles paths like "internal/watcher/events" where only the parent has a README
let parentPath = dirPathNormalized;
while (parentPath.includes("/")) {
parentPath = parentPath.slice(0, parentPath.lastIndexOf("/"));
if (dirPathToDocRoute.has(parentPath)) {
rewritten = `${dirPathToDocRoute.get(parentPath)}${fragment}`;
break;
}
}
}
if (rewritten) {
return angleWrapped === urlRaw ? rewritten : `<${rewritten}>`;
}
// 2) Intra-repo README links -> VitePress impl routes
if (/(^|\/)README\.md$/.test(urlNoFragment)) {
const targetReadmeRel = path.posix.normalize(
path.posix.join(pkgPath, urlNoFragment),
);
const route = readmeRelToDocRoute.get(targetReadmeRel);
if (route) {
const rewritten = `${route}${fragment}`;
return angleWrapped === urlRaw ? rewritten : `<${rewritten}>`;
}
return urlRaw;
}
// 2) Intra-repo README links -> VitePress impl routes
if (/(^|\/)README\.md$/.test(urlNoFragment)) {
const targetReadmeRel = path.posix.normalize(
path.posix.join(pkgPath, urlNoFragment),
);
const route = readmeRelToDocRoute.get(targetReadmeRel);
if (route) {
const rewritten = `${route}${fragment}`;
return angleWrapped === urlRaw ? rewritten : `<${rewritten}>`;
}
return urlRaw;
}
// 3) Local source-file references like "config.go:29" -> GitHub blob link
if (repoUrl) {
const { filePath, line } = parseFileLineSuffix(urlNoFragment);
if (isRepoSourceFilePath(filePath)) {
const repoRel = path.posix.normalize(
path.posix.join(pkgPath, filePath),
);
const githubUrl = `${repoUrl}/blob/main/${repoRel}${
line ? `#L${line}` : ""
}`;
const rewritten = `${githubUrl}${fragment}`;
return angleWrapped === urlRaw ? rewritten : `<${rewritten}>`;
}
}
// 3) Local source-file references like "config.go:29" -> GitHub blob link
if (repoUrl) {
const { filePath, line } = parseFileLineSuffix(urlNoFragment);
if (isRepoSourceFilePath(filePath)) {
const repoRel = path.posix.normalize(
path.posix.join(pkgPath, filePath),
);
const githubUrl = `${repoUrl}/blob/main/${repoRel}${
line ? `#L${line}` : ""
}`;
const rewritten = `${githubUrl}${fragment}`;
return angleWrapped === urlRaw ? rewritten : `<${rewritten}>`;
}
}
return urlRaw;
});
return urlRaw;
});
}
async function listRepoReadmes(repoRootAbs: string): Promise<string[]> {
const glob = new Glob("**/README.md");
const readmes: string[] = [];
const glob = new Glob("**/README.md");
const readmes: string[] = [];
for await (const rel of glob.scan({
cwd: repoRootAbs,
onlyFiles: true,
dot: false,
})) {
// Bun returns POSIX-style rel paths.
if (rel === "README.md") continue; // exclude root README
if (rel.startsWith(".git/") || rel.includes("/.git/")) continue;
if (rel.startsWith("node_modules/") || rel.includes("/node_modules/"))
continue;
let skip = false;
for (const submodule of skipSubmodules) {
if (rel.startsWith(submodule)) {
skip = true;
break;
}
}
if (skip) continue;
readmes.push(rel);
}
for await (const rel of glob.scan({
cwd: repoRootAbs,
onlyFiles: true,
dot: false,
})) {
// Bun returns POSIX-style rel paths.
if (rel === "README.md") continue; // exclude root README
if (rel.startsWith(".git/") || rel.includes("/.git/")) continue;
if (rel.startsWith("node_modules/") || rel.includes("/node_modules/"))
continue;
let skip = false;
for (const submodule of skipSubmodules) {
if (rel.startsWith(submodule)) {
skip = true;
break;
}
}
if (skip) continue;
readmes.push(rel);
}
// Deterministic order.
readmes.sort((a, b) => a.localeCompare(b));
return readmes;
// Deterministic order.
readmes.sort((a, b) => a.localeCompare(b));
return readmes;
}
async function writeImplDocCopy(params: {
srcAbs: string;
dstAbs: string;
pkgPath: string;
readmeRelToDocRoute: Map<string, string>;
dirPathToDocRoute: Map<string, string>;
repoUrl: string;
async function writeImplDocToMdx(params: {
srcAbs: string;
dstAbs: string;
pkgPath: string;
readmeRelToDocRoute: Map<string, string>;
dirPathToDocRoute: Map<string, string>;
repoUrl: string;
}) {
const {
srcAbs,
dstAbs,
pkgPath,
readmeRelToDocRoute,
dirPathToDocRoute,
repoUrl,
} = params;
await mkdir(path.dirname(dstAbs), { recursive: true });
await rm(dstAbs, { force: true });
const {
srcAbs,
dstAbs,
pkgPath,
readmeRelToDocRoute,
dirPathToDocRoute,
repoUrl,
} = params;
await mkdir(path.dirname(dstAbs), { recursive: true });
const original = await readFile(srcAbs, "utf8");
const rewritten = rewriteImplMarkdown({
md: original,
pkgPath,
readmeRelToDocRoute,
dirPathToDocRoute,
repoUrl,
});
await writeFile(dstAbs, md2mdx(rewritten));
const original = await readFile(srcAbs, "utf8");
const current = await readFile(dstAbs, "utf-8");
const rewritten = md2mdx(
rewriteImplMarkdown({
md: original,
pkgPath,
readmeRelToDocRoute,
dirPathToDocRoute,
repoUrl,
}),
);
if (current === rewritten) {
return;
}
await writeFile(dstAbs, rewritten, "utf-8");
console.log(`[W] ${srcAbs} -> ${dstAbs}`);
}
async function syncImplDocs(
repoRootAbs: string,
wikiRootAbs: string,
): Promise<ImplDoc[]> {
const implDirAbs = path.join(wikiRootAbs, "content", "docs", "impl");
await mkdir(implDirAbs, { recursive: true });
repoRootAbs: string,
wikiRootAbs: string,
): Promise<void> {
const implDirAbs = path.join(wikiRootAbs, "content", "docs", "impl");
await mkdir(implDirAbs, { recursive: true });
const readmes = await listRepoReadmes(repoRootAbs);
const docs: ImplDoc[] = [];
const expectedFileNames = new Set<string>();
expectedFileNames.add("index.mdx");
expectedFileNames.add("meta.json");
const readmes = await listRepoReadmes(repoRootAbs);
const expectedFileNames = new Set<string>();
expectedFileNames.add("index.mdx");
expectedFileNames.add("meta.json");
const repoUrl = normalizeRepoUrl(
Bun.env.REPO_URL ?? "https://github.com/yusing/godoxy",
);
const repoUrl = normalizeRepoUrl(
Bun.env.REPO_URL ?? "https://github.com/yusing/godoxy",
);
// Precompute mapping from repo-relative README path -> VitePress route.
// This lets us rewrite intra-repo README links when copying content.
const readmeRelToDocRoute = new Map<string, string>();
// Precompute mapping from repo-relative README path -> VitePress route.
// This lets us rewrite intra-repo README links when copying content.
const readmeRelToDocRoute = new Map<string, string>();
// Also precompute mapping from directory path -> VitePress route.
// This handles links like "[`common/`](common)" that point to directories with READMEs.
const dirPathToDocRoute = new Map<string, string>();
// Also precompute mapping from directory path -> VitePress route.
// This handles links like "[`common/`](common)" that point to directories with READMEs.
const dirPathToDocRoute = new Map<string, string>();
for (const readmeRel of readmes) {
const pkgPath = path.posix.dirname(readmeRel);
if (!pkgPath || pkgPath === ".") continue;
for (const readmeRel of readmes) {
const pkgPath = path.posix.dirname(readmeRel);
if (!pkgPath || pkgPath === ".") continue;
const docStem = sanitizeFileStemFromPkgPath(pkgPath);
if (!docStem) continue;
const route = `/impl/${docStem}`;
readmeRelToDocRoute.set(readmeRel, route);
dirPathToDocRoute.set(pkgPath, route);
}
const docStem = sanitizeFileStemFromPkgPath(pkgPath);
if (!docStem) continue;
const route = `/impl/${docStem}`;
readmeRelToDocRoute.set(readmeRel, route);
dirPathToDocRoute.set(pkgPath, route);
}
for (const readmeRel of readmes) {
const pkgPath = path.posix.dirname(readmeRel);
if (!pkgPath || pkgPath === ".") continue;
for (const readmeRel of readmes) {
const pkgPath = path.posix.dirname(readmeRel);
if (!pkgPath || pkgPath === ".") continue;
const docStem = sanitizeFileStemFromPkgPath(pkgPath);
if (!docStem) continue;
const docFileName = `${docStem}.mdx`;
const docRoute = `/impl/${docStem}`;
const docStem = sanitizeFileStemFromPkgPath(pkgPath);
if (!docStem) continue;
const docFileName = `${docStem}.mdx`;
const srcPathAbs = path.join(repoRootAbs, readmeRel);
const dstPathAbs = path.join(implDirAbs, docFileName);
const srcPathAbs = path.join(repoRootAbs, readmeRel);
const dstPathAbs = path.join(implDirAbs, docFileName);
await writeImplDocCopy({
srcAbs: srcPathAbs,
dstAbs: dstPathAbs,
pkgPath,
readmeRelToDocRoute,
dirPathToDocRoute,
repoUrl,
});
await writeImplDocToMdx({
srcAbs: srcPathAbs,
dstAbs: dstPathAbs,
pkgPath,
readmeRelToDocRoute,
dirPathToDocRoute,
repoUrl,
});
docs.push({ pkgPath, docFileName, docRoute, srcPathAbs, dstPathAbs });
expectedFileNames.add(docFileName);
}
expectedFileNames.add(docFileName);
}
// Clean orphaned impl docs.
const existing = await readdir(implDirAbs, { withFileTypes: true });
for (const ent of existing) {
if (!ent.isFile()) continue;
if (!ent.name.endsWith(".md")) continue;
if (expectedFileNames.has(ent.name)) continue;
await rm(path.join(implDirAbs, ent.name), { force: true });
}
// Deterministic for sidebar.
docs.sort((a, b) => a.pkgPath.localeCompare(b.pkgPath));
return docs;
// Clean orphaned impl docs.
const existing = await readdir(implDirAbs, { withFileTypes: true });
for (const ent of existing) {
if (!ent.isFile()) continue;
if (!ent.name.endsWith(".md")) continue;
if (expectedFileNames.has(ent.name)) continue;
await rm(path.join(implDirAbs, ent.name), { force: true });
}
}
async function main() {
// This script lives in `scripts/update-wiki/`, so repo root is two levels up.
const repoRootAbs = path.resolve(import.meta.dir);
// This script lives in `scripts/update-wiki/`, so repo root is two levels up.
const repoRootAbs = path.resolve(import.meta.dir, "../..");
// Required by task, but allow overriding via env for convenience.
const wikiRootAbs = Bun.env.DOCS_DIR
? path.resolve(repoRootAbs, Bun.env.DOCS_DIR)
: undefined;
// Required by task, but allow overriding via env for convenience.
const wikiRootAbs = Bun.env.DOCS_DIR
? path.resolve(repoRootAbs, Bun.env.DOCS_DIR)
: undefined;
if (!wikiRootAbs) {
throw new Error("DOCS_DIR is not set");
}
if (!wikiRootAbs) {
throw new Error("DOCS_DIR is not set");
}
await syncImplDocs(repoRootAbs, wikiRootAbs);
await syncImplDocs(repoRootAbs, wikiRootAbs);
}
await main();