Compare commits

..

15 Commits

Author SHA1 Message Date
yusing
95ffd35585 fix(rules): remove empty segments from splitPipe output
Refactored splitPipe function to use forEachPipePart helper, which filters
out empty segments instead of including them in the result. Updated test
expectation to match the new behavior where empty parts between pipes
are no longer included.
2026-02-24 02:52:19 +08:00
yusing
7b0d846576 fix(rules): prevent appending empty command parts in forEachPipePart and remove redundant calculation in parseDoWithBlocks function 2026-02-24 02:05:18 +08:00
yusing
458c7779d3 fix(tests/rules): correct semantic 2026-02-24 02:01:29 +08:00
yusing
dc6c649f2c fix(tests/rules): update HTTP flow YAML test for correct indentation and syntax
Fixes previous commit:  2a51c2ef52
2026-02-24 01:46:26 +08:00
yusing
3c5c3ecac2 fix(rules): handle empty matcher as unconditional rule
The matcherSignature function now treats empty strings as unconditional rules
that match any request,
returning "(any)" as the signature instead of
rejecting them.

This enables proper dead code detection when an unconditional
terminating rule shadows later rules.

Adds test coverage for detecting dead rules caused by unconditional
terminating rules.
2026-02-24 01:42:40 +08:00
yusing
a94442b001 fix(rules): prevent appending empty parts in splitPipe function 2026-02-24 01:36:54 +08:00
yusing
2a51c2ef52 fix(tests/rules): correct HTTP flow YAML test to use new yaml syntax
This is a test in yaml_test which meant to be testing old YAML syntax instead of new DSL
2026-02-24 01:36:35 +08:00
yusing
6477c35b15 docs(rules): update examples to use block syntax 2026-02-24 01:30:50 +08:00
yusing
5b20bbeb6f refactor(rules): simplify nested block detection by removing @ prefix requirement
Changes the nested block syntax detection from requiring `@`
as the first non-space character on a line to using a line-ending brace heuristic.

The parser now recognizes nested blocks when a line ends with an unquoted `{`,
simplifying the syntax and removing the mandatory `@` prefix while maintaining the same functionality.
2026-02-24 01:30:32 +08:00
yusing
5ba475c489 refactor(api/rules): remove IsResponseRule field from ParsedRule and related logic 2026-02-24 01:07:35 +08:00
yusing
54be056530 refactor(rules): improve termination detection and block parsing logic
Refactors the termination detection in the rules DSL to properly handle if-block and if-else-block commands.

Adds new functions `commandsTerminateInPre`, `commandTerminatesInPre`, and `ifElseBlockTerminatesInPre`
to recursively check if command sequences terminate in the pre-phase.

Also improves the Parse function to try block syntax first with proper error handling and fallback to YAML.

Includes test cases for dead code detection with terminating handlers in conditional blocks.
2026-02-24 01:05:54 +08:00
yusing
08de9086c3 fix(rules): buffer log output before writing to stdout/stderr 2026-02-24 00:12:29 +08:00
yusing
1a17f3943a refactor(rules): change default rule from baseline to fallback behavior
The default rule should runs only when no non-default pre rule matches, instead of running first as a baseline.
This follows the old behavior as before the pr is established.:

- Default rules act as fallback handlers that execute only when no matching non-default rule exists in the pre phase
- IfElseBlockCommand now returns early when a condition matches with a nil Do block, instead of falling through to else blocks
- Add nil check for auth handler to allow requests when no auth is configured
- Fix unterminated environment variable parsing to preserve input

Updates tests to verify the new fallback behavior where special rules suppress default rule execution.
2026-02-24 00:11:03 +08:00
yusing
9bb5c54e7c refactor(rules): defer error logging until after FlushRelease
Split error handling into isUnexpectedError predicate and logFlushError
function. Use rm.AppendError() to collect unexpected errors during rule
execution, then log after FlushRelease completes rather than immediately.
Also updates goutils dependency for AppendError method availability.
2026-02-23 23:09:24 +08:00
yusing
faecbab2cb refactor(rules): introduce block DSL, phase-based execution, and flow validation
- add block syntax parser/scanner with nested @blocks and elif/else support
- restructure rule execution into explicit pre/post phases with phase flags
- classify commands by phase and termination behavior
- enforce flow semantics (default rule handling, dead-rule detection)
- expand HTTP flow coverage with block + YAML parity tests and benches
- refresh rules README/spec and update playground/docs integration
2026-02-23 22:24:15 +08:00
24 changed files with 383 additions and 841 deletions

View File

@@ -6,8 +6,8 @@ export GOOS = linux
REPO_URL ?= https://github.com/yusing/godoxy
WEBUI_DIR ?= $(shell pwd)/../godoxy-webui
DOCS_DIR ?= ${WEBUI_DIR}/wiki
WEBUI_DIR ?= ../godoxy-webui
DOCS_DIR ?= wiki
ifneq ($(BRANCH), compat)
GO_TAGS = sonic

View File

@@ -46,7 +46,7 @@ require (
github.com/docker/cli v29.2.1+incompatible // indirect
github.com/docker/go-connections v0.6.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/ebitengine/purego v0.10.0 // indirect
github.com/ebitengine/purego v0.9.1 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/gabriel-vasile/mimetype v1.4.13 // indirect
github.com/gin-contrib/sse v1.1.0 // indirect
@@ -91,8 +91,8 @@ require (
github.com/valyala/fasthttp v1.69.0 // indirect
github.com/yusing/ds v0.4.1 // indirect
github.com/yusing/gointernals v0.2.0 // indirect
github.com/yusing/goutils/http/reverseproxy v0.0.0-20260223150038-3be815cb6e3b // indirect
github.com/yusing/goutils/http/websocket v0.0.0-20260223150038-3be815cb6e3b // indirect
github.com/yusing/goutils/http/reverseproxy v0.0.0-20260218062549-0b0fa3a059ec // indirect
github.com/yusing/goutils/http/websocket v0.0.0-20260218062549-0b0fa3a059ec // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 // indirect

View File

@@ -43,8 +43,8 @@ github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pM
github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU=
github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
github.com/ebitengine/purego v0.9.1 h1:a/k2f2HQU3Pi399RPW1MOaZyhKJL9w/xFpKAg4q1s0A=
github.com/ebitengine/purego v0.9.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=

View File

@@ -1,46 +0,0 @@
services:
netbird-dashboard:
image: netbirdio/dashboard:latest
container_name: netbird-dashboard
restart: unless-stopped
networks: [netbird-net]
env_file: dashboard.env
labels:
proxy.aliases: netbird
proxy.#1.port: 80
proxy.#1.scheme: http
proxy.#1.homepage.name: NetBird
proxy.#1.homepage.icon: "@selfhst/netbird.svg"
proxy.#1.homepage.category: networking
proxy.#1.rules: | # https://docs.netbird.io/selfhosted/configuration-files
path glob(/signalexchange.SignalExchange/**) | path glob(/management.ManagementService/**) | path glob(/management.ProxyService/**) {
route netbird-grpc
}
path glob(/relay*) | path glob(/ws-proxy/**) | path glob(/api*) | path glob(/oauth2*) {
route netbird-api
}
default {
pass
}
netbird-server:
image: netbirdio/netbird-server:latest
container_name: netbird-server
restart: unless-stopped
networks:
- netbird-net
volumes:
- netbird_data:/var/lib/netbird
- ./config.yaml:/etc/netbird/config.yaml
command: ["--config", "/etc/netbird/config.yaml"]
labels:
proxy.aliases: netbird-api, netbird-grpc
proxy.*.port: 80
proxy.*.homepage.show: false
proxy.#1.scheme: http
proxy.#2.scheme: h2c
networks:
netbird-net:
volumes:
netbird_data:

16
go.mod
View File

@@ -58,13 +58,13 @@ require (
github.com/stretchr/testify v1.11.1 // testing framework
github.com/valyala/fasthttp v1.69.0 // fast http for health check
github.com/yusing/ds v0.4.1 // data structures and algorithms
github.com/yusing/godoxy/agent v0.0.0-20260224071728-0eba04510480
github.com/yusing/godoxy/internal/dnsproviders v0.0.0-20260224071728-0eba04510480
github.com/yusing/godoxy/agent v0.0.0-20260218101334-add7884a365e
github.com/yusing/godoxy/internal/dnsproviders v0.0.0-20260218101334-add7884a365e
github.com/yusing/gointernals v0.2.0
github.com/yusing/goutils v0.7.0
github.com/yusing/goutils/http/reverseproxy v0.0.0-20260223150038-3be815cb6e3b
github.com/yusing/goutils/http/websocket v0.0.0-20260223150038-3be815cb6e3b
github.com/yusing/goutils/server v0.0.0-20260223150038-3be815cb6e3b
github.com/yusing/goutils/http/reverseproxy v0.0.0-20260218062549-0b0fa3a059ec
github.com/yusing/goutils/http/websocket v0.0.0-20260218062549-0b0fa3a059ec
github.com/yusing/goutils/server v0.0.0-20260218062549-0b0fa3a059ec
)
require (
@@ -88,7 +88,7 @@ require (
github.com/djherbis/times v1.6.0 // indirect
github.com/docker/go-connections v0.6.0
github.com/docker/go-units v0.5.0 // indirect
github.com/ebitengine/purego v0.10.0 // indirect
github.com/ebitengine/purego v0.9.1 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/gabriel-vasile/mimetype v1.4.13 // indirect
github.com/go-jose/go-jose/v4 v4.1.3 // indirect
@@ -142,8 +142,8 @@ require (
golang.org/x/sys v0.41.0 // indirect
golang.org/x/text v0.34.0 // indirect
golang.org/x/tools v0.42.0 // indirect
google.golang.org/api v0.268.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260223185530-2f722ef697dc // indirect
google.golang.org/api v0.267.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260217215200-42d3e9bedb6d // indirect
google.golang.org/grpc v1.79.1 // indirect
google.golang.org/protobuf v1.36.11 // indirect
gopkg.in/ini.v1 v1.67.1 // indirect

12
go.sum
View File

@@ -82,8 +82,8 @@ github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pM
github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU=
github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
github.com/ebitengine/purego v0.9.1 h1:a/k2f2HQU3Pi399RPW1MOaZyhKJL9w/xFpKAg4q1s0A=
github.com/ebitengine/purego v0.9.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
github.com/elliotwutingfeng/asciiset v0.0.0-20230602022725-51bbb787efab h1:h1UgjJdAAhj+uPL68n7XASS6bU+07ZX1WJvVS2eyoeY=
github.com/elliotwutingfeng/asciiset v0.0.0-20230602022725-51bbb787efab/go.mod h1:GLo/8fDswSAniFG+BFIaiSPcK610jyzgEhWYPQwuQdw=
github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
@@ -447,14 +447,14 @@ golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
google.golang.org/api v0.268.0 h1:hgA3aS4lt9rpF5RCCkX0Q2l7DvHgvlb53y4T4u6iKkA=
google.golang.org/api v0.268.0/go.mod h1:HXMyMH496wz+dAJwD/GkAPLd3ZL33Kh0zEG32eNvy9w=
google.golang.org/api v0.267.0 h1:w+vfWPMPYeRs8qH1aYYsFX68jMls5acWl/jocfLomwE=
google.golang.org/api v0.267.0/go.mod h1:Jzc0+ZfLnyvXma3UtaTl023TdhZu6OMBP9tJ+0EmFD0=
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 h1:VQZ/yAbAtjkHgH80teYd2em3xtIkkHd7ZhqfH2N9CsM=
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409/go.mod h1:rxKD3IEILWEu3P44seeNOAwZN4SaoKaQ/2eTg4mM6EM=
google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 h1:merA0rdPeUV3YIIfHHcH4qBkiQAc1nfCKSI7lB4cV2M=
google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409/go.mod h1:fl8J1IvUjCilwZzQowmw2b7HQB2eAuYBabMXzWurF+I=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260223185530-2f722ef697dc h1:51Wupg8spF+5FC6D+iMKbOddFjMckETnNnEiZ+HX37s=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260223185530-2f722ef697dc/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260217215200-42d3e9bedb6d h1:t/LOSXPJ9R0B6fnZNyALBRfZBH0Uy0gT+uR+SJ6syqQ=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260217215200-42d3e9bedb6d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
google.golang.org/grpc v1.79.1 h1:zGhSi45ODB9/p3VAawt9a+O/MULLl9dpizzNNpq7flY=
google.golang.org/grpc v1.79.1/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=

Submodule goutils updated: 246c8e9ba1...3be815cb6e

View File

@@ -5125,7 +5125,10 @@
"$ref": "#/definitions/MockResponse"
},
"rules": {
"type": "string",
"type": "array",
"items": {
"$ref": "#/definitions/routeApi.RawRule"
},
"x-nullable": false,
"x-omitempty": false
}
@@ -6923,6 +6926,28 @@
"x-nullable": false,
"x-omitempty": false
},
"routeApi.RawRule": {
"type": "object",
"properties": {
"do": {
"type": "string",
"x-nullable": false,
"x-omitempty": false
},
"name": {
"type": "string",
"x-nullable": false,
"x-omitempty": false
},
"on": {
"type": "string",
"x-nullable": false,
"x-omitempty": false
}
},
"x-nullable": false,
"x-omitempty": false
},
"routeApi.RoutesByProvider": {
"type": "object",
"additionalProperties": {

View File

@@ -905,7 +905,9 @@ definitions:
mockResponse:
$ref: '#/definitions/MockResponse'
rules:
type: string
items:
$ref: '#/definitions/routeApi.RawRule'
type: array
required:
- rules
type: object
@@ -1856,6 +1858,15 @@ definitions:
uptime:
type: string
type: object
routeApi.RawRule:
properties:
do:
type: string
name:
type: string
"on":
type: string
type: object
routeApi.RoutesByProvider:
additionalProperties:
items:

View File

@@ -9,7 +9,6 @@ import (
"strings"
"github.com/gin-gonic/gin"
"github.com/goccy/go-yaml"
"github.com/yusing/godoxy/internal/common"
"github.com/yusing/godoxy/internal/route/rules"
apitypes "github.com/yusing/goutils/apitypes"
@@ -24,7 +23,7 @@ type RawRule struct {
}
type PlaygroundRequest struct {
Rules string `json:"rules" binding:"required"`
Rules []RawRule `json:"rules" binding:"required"`
MockRequest MockRequest `json:"mockRequest"`
MockResponse MockResponse `json:"mockResponse"`
} // @name PlaygroundRequest
@@ -256,35 +255,7 @@ func handlerWithRecover(w http.ResponseWriter, r *http.Request, h http.HandlerFu
h(w, r)
}
func parseRules(config string) ([]ParsedRule, rules.Rules, error) {
config = strings.TrimSpace(config)
if config == "" {
return []ParsedRule{}, nil, nil
}
var rawRules []RawRule
if err := yaml.Unmarshal([]byte(config), &rawRules); err == nil && len(rawRules) > 0 {
return parseRawRules(rawRules)
}
var rulesList rules.Rules
if err := rulesList.Parse(config); err != nil {
return nil, nil, err
}
parsedRules := make([]ParsedRule, 0, len(rulesList))
for _, rule := range rulesList {
parsedRules = append(parsedRules, ParsedRule{
Name: rule.Name,
On: rule.On.String(),
Do: rule.Do.String(),
})
}
return parsedRules, rulesList, nil
}
func parseRawRules(rawRules []RawRule) ([]ParsedRule, rules.Rules, error) {
func parseRules(rawRules []RawRule) ([]ParsedRule, rules.Rules, error) {
parsedRules := make([]ParsedRule, 0, len(rawRules))
rulesList := make(rules.Rules, 0, len(rawRules))

View File

@@ -22,10 +22,13 @@ func TestPlayground(t *testing.T) {
{
name: "simple path matching rule",
request: PlaygroundRequest{
Rules: `- name: test rule
on: path /api
do: pass
`,
Rules: []RawRule{
{
Name: "test rule",
On: "path /api",
Do: "pass",
},
},
MockRequest: MockRequest{
Method: "GET",
Path: "/api",
@@ -50,10 +53,13 @@ func TestPlayground(t *testing.T) {
{
name: "header matching rule",
request: PlaygroundRequest{
Rules: `- name: check user agent
on: header User-Agent Chrome
do: error 403 Forbidden
`,
Rules: []RawRule{
{
Name: "check user agent",
On: "header User-Agent Chrome",
Do: "error 403 Forbidden",
},
},
MockRequest: MockRequest{
Method: "GET",
Path: "/",
@@ -84,10 +90,13 @@ func TestPlayground(t *testing.T) {
{
name: "invalid rule syntax",
request: PlaygroundRequest{
Rules: `- name: bad rule
on: invalid_checker something
do: pass
`,
Rules: []RawRule{
{
Name: "bad rule",
On: "invalid_checker something",
Do: "pass",
},
},
MockRequest: MockRequest{
Method: "GET",
Path: "/",
@@ -106,10 +115,13 @@ func TestPlayground(t *testing.T) {
{
name: "rewrite path rule",
request: PlaygroundRequest{
Rules: `- name: rewrite rule
on: path glob(/api/*)
do: rewrite /api/ /v1/
`,
Rules: []RawRule{
{
Name: "rewrite rule",
On: "path glob(/api/*)",
Do: "rewrite /api/ /v1/",
},
},
MockRequest: MockRequest{
Method: "GET",
Path: "/api/users",
@@ -136,10 +148,13 @@ func TestPlayground(t *testing.T) {
{
name: "method matching rule",
request: PlaygroundRequest{
Rules: `- name: block POST
on: method POST
do: error "405" "Method Not Allowed"
`,
Rules: []RawRule{
{
Name: "block POST",
On: "method POST",
Do: `error "405" "Method Not Allowed"`,
},
},
MockRequest: MockRequest{
Method: "POST",
Path: "/api",
@@ -158,63 +173,6 @@ func TestPlayground(t *testing.T) {
}
},
},
{
name: "block syntax default rule",
request: PlaygroundRequest{
Rules: `default {
pass
}`,
MockRequest: MockRequest{
Method: "GET",
Path: "/",
},
},
wantStatusCode: http.StatusOK,
checkResponse: func(t *testing.T, resp PlaygroundResponse) {
if len(resp.ParsedRules) != 1 {
t.Errorf("expected 1 parsed rule, got %d", len(resp.ParsedRules))
}
if resp.ParsedRules[0].ValidationError != nil {
t.Errorf("expected rule to be valid, got error: %v", resp.ParsedRules[0].ValidationError)
}
if !resp.UpstreamCalled {
t.Error("expected upstream to be called")
}
},
},
{
name: "block syntax conditional rule",
request: PlaygroundRequest{
Rules: `header User-Agent Chrome {
error 403 Forbidden
}`,
MockRequest: MockRequest{
Method: "GET",
Path: "/",
Headers: map[string][]string{
"User-Agent": {"Chrome"},
},
},
},
wantStatusCode: http.StatusOK,
checkResponse: func(t *testing.T, resp PlaygroundResponse) {
if len(resp.ParsedRules) != 1 {
t.Errorf("expected 1 parsed rule, got %d", len(resp.ParsedRules))
}
if resp.ParsedRules[0].ValidationError != nil {
t.Errorf("expected rule to be valid, got error: %v", resp.ParsedRules[0].ValidationError)
}
if len(resp.MatchedRules) != 1 {
t.Errorf("expected 1 matched rule, got %d", len(resp.MatchedRules))
}
if resp.FinalResponse.StatusCode != http.StatusForbidden {
t.Errorf("expected status 403, got %d", resp.FinalResponse.StatusCode)
}
if resp.UpstreamCalled {
t.Error("expected upstream not to be called")
}
},
},
}
for _, tt := range tests {

View File

@@ -98,8 +98,8 @@ require (
golang.org/x/sys v0.41.0 // indirect
golang.org/x/text v0.34.0 // indirect
golang.org/x/tools v0.42.0 // indirect
google.golang.org/api v0.268.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260223185530-2f722ef697dc // indirect
google.golang.org/api v0.267.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260217215200-42d3e9bedb6d // indirect
google.golang.org/grpc v1.79.1 // indirect
google.golang.org/protobuf v1.36.11 // indirect
gopkg.in/ini.v1 v1.67.1 // indirect

View File

@@ -249,14 +249,14 @@ golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
google.golang.org/api v0.268.0 h1:hgA3aS4lt9rpF5RCCkX0Q2l7DvHgvlb53y4T4u6iKkA=
google.golang.org/api v0.268.0/go.mod h1:HXMyMH496wz+dAJwD/GkAPLd3ZL33Kh0zEG32eNvy9w=
google.golang.org/api v0.267.0 h1:w+vfWPMPYeRs8qH1aYYsFX68jMls5acWl/jocfLomwE=
google.golang.org/api v0.267.0/go.mod h1:Jzc0+ZfLnyvXma3UtaTl023TdhZu6OMBP9tJ+0EmFD0=
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 h1:VQZ/yAbAtjkHgH80teYd2em3xtIkkHd7ZhqfH2N9CsM=
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409/go.mod h1:rxKD3IEILWEu3P44seeNOAwZN4SaoKaQ/2eTg4mM6EM=
google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 h1:merA0rdPeUV3YIIfHHcH4qBkiQAc1nfCKSI7lB4cV2M=
google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409/go.mod h1:fl8J1IvUjCilwZzQowmw2b7HQB2eAuYBabMXzWurF+I=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260223185530-2f722ef697dc h1:51Wupg8spF+5FC6D+iMKbOddFjMckETnNnEiZ+HX37s=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260223185530-2f722ef697dc/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260217215200-42d3e9bedb6d h1:t/LOSXPJ9R0B6fnZNyALBRfZBH0Uy0gT+uR+SJ6syqQ=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260217215200-42d3e9bedb6d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
google.golang.org/grpc v1.79.1 h1:zGhSi45ODB9/p3VAawt9a+O/MULLl9dpizzNNpq7flY=
google.golang.org/grpc v1.79.1/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=

View File

@@ -13,8 +13,6 @@ This package implements a flexible HTTP middleware system for GoDoxy. Middleware
- **Bypass Rules**: Skip middleware based on request properties
- **Dynamic Loading**: Load middleware definitions from files at runtime
Response body rewriting is only applied to unencoded, text-like content types (for example `text/*`, JSON, YAML, XML). Response status and headers can always be modified.
## Architecture
```mermaid

View File

@@ -6,7 +6,6 @@ import (
"net/http"
"reflect"
"sort"
"strings"
"github.com/bytedance/sonic"
"github.com/rs/zerolog"
@@ -196,15 +195,21 @@ func (m *Middleware) ServeHTTP(next http.HandlerFunc, w http.ResponseWriter, r *
}
if exec, ok := m.impl.(ResponseModifier); ok {
rm := httputils.NewResponseModifier(w)
lrm := httputils.NewLazyResponseModifier(w, needsBuffering)
defer func() {
_, err := rm.FlushRelease()
_, err := lrm.FlushRelease()
if err != nil {
m.LogError(r).Err(err).Msg("failed to flush response")
}
}()
next(rm, r)
next(lrm, r)
// Skip modification if response wasn't buffered (non-HTML content)
if !lrm.IsBuffered() {
return
}
rm := lrm.ResponseModifier()
currentBody := rm.BodyReader()
currentResp := &http.Response{
StatusCode: rm.StatusCode(),
@@ -213,31 +218,20 @@ func (m *Middleware) ServeHTTP(next http.HandlerFunc, w http.ResponseWriter, r *
Body: currentBody,
Request: r,
}
allowBodyModification := canModifyResponseBody(currentResp)
respToModify := currentResp
if !allowBodyModification {
shadow := *currentResp
shadow.Body = eofReader{}
respToModify = &shadow
}
if err := exec.modifyResponse(respToModify); err != nil {
if err := exec.modifyResponse(currentResp); err != nil {
log.Err(err).Str("middleware", m.Name()).Str("url", fullURL(r)).Msg("failed to modify response")
}
// override the response status code
rm.WriteHeader(respToModify.StatusCode)
rm.WriteHeader(currentResp.StatusCode)
// overriding the response header
maps.Copy(rm.Header(), respToModify.Header)
maps.Copy(rm.Header(), currentResp.Header)
// override the content length and body if changed
if respToModify.Body != currentBody {
if allowBodyModification {
if err := rm.SetBody(respToModify.Body); err != nil {
m.LogError(r).Err(err).Msg("failed to set response body")
}
} else {
respToModify.Body.Close()
if currentResp.Body != currentBody {
if err := rm.SetBody(currentResp.Body); err != nil {
m.LogError(r).Err(err).Msg("failed to set response body")
}
}
} else {
@@ -245,55 +239,10 @@ func (m *Middleware) ServeHTTP(next http.HandlerFunc, w http.ResponseWriter, r *
}
}
func canModifyResponseBody(resp *http.Response) bool {
if hasNonIdentityEncoding(resp.TransferEncoding) {
return false
}
if hasNonIdentityEncoding(resp.Header.Values("Transfer-Encoding")) {
return false
}
if hasNonIdentityEncoding(resp.Header.Values("Content-Encoding")) {
return false
}
return isTextLikeMediaType(string(httputils.GetContentType(resp.Header)))
}
func hasNonIdentityEncoding(values []string) bool {
for _, value := range values {
for _, token := range strings.Split(value, ",") {
if strings.TrimSpace(token) == "" || strings.EqualFold(strings.TrimSpace(token), "identity") {
continue
}
return true
}
}
return false
}
func isTextLikeMediaType(contentType string) bool {
if contentType == "" {
return false
}
contentType = strings.ToLower(contentType)
if strings.HasPrefix(contentType, "text/") {
return true
}
if contentType == "application/json" || strings.HasSuffix(contentType, "+json") {
return true
}
if contentType == "application/xml" || strings.HasSuffix(contentType, "+xml") {
return true
}
if strings.Contains(contentType, "yaml") || strings.Contains(contentType, "toml") {
return true
}
if strings.Contains(contentType, "javascript") || strings.Contains(contentType, "ecmascript") {
return true
}
if strings.Contains(contentType, "csv") {
return true
}
return contentType == "application/x-www-form-urlencoded"
// needsBuffering determines if a response should be buffered for modification.
// Only HTML responses need buffering; streaming content (video, audio, etc.) should pass through.
func needsBuffering(header http.Header) bool {
return httputils.GetContentType(header).IsHTML()
}
func (m *Middleware) LogWarn(req *http.Request) *zerolog.Event {

View File

@@ -1,7 +1,6 @@
package middleware
import (
"maps"
"net/http"
"strconv"
@@ -47,23 +46,10 @@ func (m *middlewareChain) modifyResponse(resp *http.Response) error {
if len(m.modResps) == 0 {
return nil
}
allowBodyModification := canModifyResponseBody(resp)
for i, mr := range m.modResps {
respToModify := resp
if !allowBodyModification {
shadow := *resp
shadow.Body = eofReader{}
respToModify = &shadow
}
if err := mr.modifyResponse(respToModify); err != nil {
if err := mr.modifyResponse(resp); err != nil {
return gperr.PrependSubject(err, strconv.Itoa(i))
}
if !allowBodyModification {
resp.StatusCode = respToModify.StatusCode
if respToModify.Header != nil {
maps.Copy(resp.Header, respToModify.Header)
}
}
}
return nil
}

View File

@@ -1,7 +1,6 @@
package middleware
import (
"io"
"net/http"
"strconv"
"strings"
@@ -15,27 +14,12 @@ type testPriority struct {
}
var test = NewMiddleware[testPriority]()
var responseRewrite = NewMiddleware[testResponseRewrite]()
func (t testPriority) before(w http.ResponseWriter, r *http.Request) bool {
w.Header().Add("Test-Value", strconv.Itoa(t.Value))
return true
}
type testResponseRewrite struct {
StatusCode int `json:"status_code"`
HeaderKey string `json:"header_key"`
HeaderVal string `json:"header_val"`
Body string `json:"body"`
}
func (t testResponseRewrite) modifyResponse(resp *http.Response) error {
resp.StatusCode = t.StatusCode
resp.Header.Set(t.HeaderKey, t.HeaderVal)
resp.Body = io.NopCloser(strings.NewReader(t.Body))
return nil
}
func TestMiddlewarePriority(t *testing.T) {
priorities := []int{4, 7, 9, 0}
chain := make([]*Middleware, len(priorities))
@@ -51,85 +35,3 @@ func TestMiddlewarePriority(t *testing.T) {
expect.NoError(t, err)
expect.Equal(t, strings.Join(res.ResponseHeaders["Test-Value"], ","), "3,0,1,2")
}
func TestMiddlewareResponseRewriteGate(t *testing.T) {
opts := OptionsRaw{
"status_code": 418,
"header_key": "X-Rewrite",
"header_val": "1",
"body": "rewritten-body",
}
tests := []struct {
name string
respHeaders http.Header
respBody []byte
expectBody string
}{
{
name: "allow_body_rewrite_for_html",
respHeaders: http.Header{
"Content-Type": []string{"text/html; charset=utf-8"},
},
respBody: []byte("<html><body>original</body></html>"),
expectBody: "rewritten-body",
},
{
name: "allow_body_rewrite_for_json",
respHeaders: http.Header{
"Content-Type": []string{"application/json"},
},
respBody: []byte(`{"message":"original"}`),
expectBody: "rewritten-body",
},
{
name: "allow_body_rewrite_for_yaml",
respHeaders: http.Header{
"Content-Type": []string{"application/yaml"},
},
respBody: []byte("k: v"),
expectBody: "rewritten-body",
},
{
name: "block_body_rewrite_for_binary_content",
respHeaders: http.Header{
"Content-Type": []string{"application/octet-stream"},
},
respBody: []byte("binary"),
expectBody: "binary",
},
{
name: "block_body_rewrite_for_transfer_encoded_html",
respHeaders: http.Header{
"Content-Type": []string{"text/html"},
"Transfer-Encoding": []string{"chunked"},
},
respBody: []byte("<html><body>original</body></html>"),
expectBody: "<html><body>original</body></html>",
},
{
name: "block_body_rewrite_for_content_encoded_html",
respHeaders: http.Header{
"Content-Type": []string{"text/html"},
"Content-Encoding": []string{"gzip"},
},
respBody: []byte("<html><body>original</body></html>"),
expectBody: "<html><body>original</body></html>",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
result, err := newMiddlewareTest(responseRewrite, &testArgs{
middlewareOpt: opts,
respHeaders: tc.respHeaders,
respBody: tc.respBody,
respStatus: http.StatusOK,
})
expect.NoError(t, err)
expect.Equal(t, result.ResponseStatus, 418)
expect.Equal(t, result.ResponseHeaders.Get("X-Rewrite"), "1")
expect.Equal(t, string(result.Data), tc.expectBody)
})
}
}

View File

@@ -7,7 +7,6 @@ import (
"maps"
"net/http"
"net/http/httptest"
"strings"
"github.com/bytedance/sonic"
"github.com/yusing/godoxy/internal/common"
@@ -55,7 +54,7 @@ func (rt *requestRecorder) RoundTrip(req *http.Request) (resp *http.Response, er
resp = &http.Response{
Status: http.StatusText(rt.args.respStatus),
StatusCode: rt.args.respStatus,
Header: maps.Clone(testHeaders),
Header: testHeaders,
Body: io.NopCloser(bytes.NewReader(rt.args.respBody)),
ContentLength: int64(len(rt.args.respBody)),
Request: req,
@@ -66,27 +65,9 @@ func (rt *requestRecorder) RoundTrip(req *http.Request) (resp *http.Response, er
return nil, err
}
maps.Copy(resp.Header, rt.args.respHeaders)
if transferEncoding := resp.Header.Values("Transfer-Encoding"); len(transferEncoding) > 0 {
resp.TransferEncoding = parseHeaderTokens(transferEncoding)
resp.ContentLength = -1
}
return resp, nil
}
func parseHeaderTokens(values []string) []string {
var tokens []string
for _, value := range values {
for token := range strings.SplitSeq(value, ",") {
token = strings.TrimSpace(token)
if token == "" {
continue
}
tokens = append(tokens, token)
}
}
return tokens
}
type TestResult struct {
RequestHeaders http.Header
ResponseHeaders http.Header

View File

@@ -439,19 +439,8 @@ $remote_host # Client IP
# Dynamic variables
$header(Name) # Request header
$header(Name, index) # Header at index
$resp_header(Name) # Response header
$arg(Name) # Query argument
$form(Name) # Form field
$postform(Name) # POST form field
$cookie(Name) # Cookie value
# Function composition: pass result of one function to another
$redacted($header(Authorization)) # Redact the Authorization header value
$redacted($arg(token)) # Redact a query parameter value
$redacted($cookie(session)) # Redact a cookie value
# $redacted: masks a value, showing only first 2 and last 2 characters
$redacted(value) # Redact a plain string
# Environment variables
${ENV_VAR}

View File

@@ -1,7 +1,6 @@
package rules
import (
"errors"
"fmt"
"io"
"net/http"
@@ -234,9 +233,6 @@ var commands = map[string]struct {
route := args.(string)
return func(w *httputils.ResponseModifier, req *http.Request, upstream http.HandlerFunc) error {
ep := entrypoint.FromCtx(req.Context())
if ep == nil {
return errors.New("entrypoint not found")
}
r, ok := ep.HTTPRoutes().Get(route)
if !ok {
excluded, has := ep.ExcludedRoutes().Get(route)

View File

@@ -152,12 +152,6 @@ func ExpandVars(w *httputils.ResponseModifier, req *http.Request, src string, ds
return phase, err
}
i = nextIdx
// Expand any nested $func(...) expressions in args
args, argPhase, err := expandArgs(args, w, req)
if err != nil {
return phase, err
}
phase |= argPhase
actual, err = getter.get(args, w, req)
if err != nil {
return phase, err
@@ -227,18 +221,6 @@ func extractArgs(src string, i int, funcName string) (args []string, nextIdx int
continue
}
// Nested function call: $func(...) as an argument
if ch == '$' && arg.Len() == 0 {
// Capture the entire $func(...) expression as a raw argument token
nestedEnd, nestedErr := extractNestedFuncExpr(src, nextIdx)
if nestedErr != nil {
return nil, 0, nestedErr
}
args = append(args, src[nextIdx:nestedEnd+1])
nextIdx = nestedEnd + 1
continue
}
if ch == ')' {
// End of arguments
if arg.Len() > 0 {
@@ -274,70 +256,3 @@ func extractArgs(src string, i int, funcName string) (args []string, nextIdx int
}
return nil, 0, ErrUnterminatedParenthesis.Withf("func %q", funcName)
}
// extractNestedFuncExpr finds the end index (inclusive) of a $func(...) expression
// starting at position start in src. It handles nested parentheses.
func extractNestedFuncExpr(src string, start int) (endIdx int, err error) {
// src[start] must be '$'
i := start + 1
// skip the function name (valid var name chars)
for i < len(src) && validVarNameCharset[src[i]] {
i++
}
if i >= len(src) || src[i] != '(' {
return 0, ErrUnterminatedParenthesis.Withf("nested func at position %d", start)
}
// Now find the matching closing parenthesis, respecting quotes and nesting
depth := 0
var quote byte
for i < len(src) {
ch := src[i]
if quote != 0 {
if ch == quote {
quote = 0
}
i++
continue
}
if quoteChars[ch] {
quote = ch
i++
continue
}
switch ch {
case '(':
depth++
case ')':
depth--
if depth == 0 {
return i, nil
}
}
i++
}
if quote != 0 {
return 0, ErrUnterminatedQuotes.Withf("nested func at position %d", start)
}
return 0, ErrUnterminatedParenthesis.Withf("nested func at position %d", start)
}
// expandArgs expands any args that are nested dynamic var expressions (starting with '$').
// It returns the expanded args and the combined phase flags.
func expandArgs(args []string, w *httputils.ResponseModifier, req *http.Request) (expanded []string, phase PhaseFlag, err error) {
expanded = make([]string, len(args))
for i, arg := range args {
if len(arg) > 0 && arg[0] == '$' {
var buf strings.Builder
var argPhase PhaseFlag
argPhase, err = ExpandVars(w, req, arg, &buf)
if err != nil {
return nil, phase, err
}
phase |= argPhase
expanded[i] = buf.String()
} else {
expanded[i] = arg
}
}
return expanded, phase, nil
}

View File

@@ -6,7 +6,6 @@ import (
"strconv"
httputils "github.com/yusing/goutils/http"
strutils "github.com/yusing/goutils/strings"
)
var (
@@ -16,7 +15,6 @@ var (
VarQuery = "arg"
VarForm = "form"
VarPostForm = "postform"
VarRedacted = "redacted"
)
type dynamicVarGetter struct {
@@ -96,17 +94,6 @@ var dynamicVarSubsMap = map[string]dynamicVarGetter{
return getValueByKeyAtIndex(req.PostForm, key, index)
},
},
// VarRedacted wraps the result of its single argument (which may be another dynamic var
// expression, already expanded by expandArgs) with strutils.Redact.
VarRedacted: {
phase: PhaseNone,
get: func(args []string, w *httputils.ResponseModifier, req *http.Request) (string, error) {
if len(args) != 1 {
return "", ErrExpectOneArg
}
return strutils.Redact(args[0]), nil
},
},
}
func getValueByKeyAtIndex[Values http.Header | url.Values](values Values, key string, index int) (string, error) {

View File

@@ -189,64 +189,6 @@ func TestExtractArgs(t *testing.T) {
}
}
func TestExtractArgs_NestedFunc(t *testing.T) {
tests := []struct {
name string
src string
startPos int
funcName string
wantArgs []string
wantNextIdx int
wantErr bool
}{
{
name: "nested func as single arg",
src: "redacted($header(Authorization))",
startPos: 0,
funcName: "redacted",
wantArgs: []string{"$header(Authorization)"},
wantNextIdx: 31,
},
{
name: "nested func with quoted arg inside",
src: `redacted($header("X-Secret"))`,
startPos: 0,
funcName: "redacted",
wantArgs: []string{`$header("X-Secret")`},
wantNextIdx: 28,
},
{
name: "nested func with two args inside",
src: "redacted($header(X-Multi, 1))",
startPos: 0,
funcName: "redacted",
wantArgs: []string{"$header(X-Multi, 1)"},
wantNextIdx: 28,
},
{
name: "nested func missing closing paren",
src: "redacted($header(Authorization)",
startPos: 0,
funcName: "redacted",
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
args, nextIdx, err := extractArgs(tt.src, tt.startPos, tt.funcName)
if tt.wantErr {
require.Error(t, err)
} else {
require.NoError(t, err)
require.Equal(t, tt.wantArgs, args)
require.Equal(t, tt.wantNextIdx, nextIdx)
}
})
}
}
func TestExpandVars(t *testing.T) {
// Create a comprehensive test request with form data
formData := url.Values{}
@@ -504,27 +446,6 @@ func TestExpandVars(t *testing.T) {
input: "Header: $header(User-Agent), Status: $status_code",
want: "Header: test-agent/1.0, Status: 200",
},
// $redacted function
{
name: "redacted with plain string arg",
input: "$redacted(secret)",
want: "se**et",
},
{
name: "redacted wrapping header",
input: "$redacted($header(User-Agent))",
want: "te**********.0",
},
{
name: "redacted wrapping arg",
input: "$redacted($arg(param1))",
want: "va**e1",
},
{
name: "redacted with no args",
input: "$redacted()",
wantErr: true,
},
// Escaped dollar signs
{
name: "escaped dollar",

View File

@@ -4,336 +4,335 @@ import { Glob } from "bun";
import { md2mdx } from "./api-md2mdx";
type ImplDoc = {
/** Directory path relative to this repo, e.g. "internal/health/check" */
pkgPath: string;
/** File name in wiki `src/impl/`, e.g. "internal-health-check.md" */
docFileName: string;
/** VitePress route path (extensionless), e.g. "/impl/internal-health-check" */
docRoute: string;
/** Absolute source README path */
srcPathAbs: string;
/** Absolute destination doc path */
dstPathAbs: string;
/** Directory path relative to this repo, e.g. "internal/health/check" */
pkgPath: string;
/** File name in wiki `src/impl/`, e.g. "internal-health-check.md" */
docFileName: string;
/** VitePress route path (extensionless), e.g. "/impl/internal-health-check" */
docRoute: string;
/** Absolute source README path */
srcPathAbs: string;
/** Absolute destination doc path */
dstPathAbs: string;
};
const skipSubmodules = [
"internal/go-oidc/",
"internal/gopsutil/",
"internal/go-proxmox/",
"internal/go-oidc/",
"internal/gopsutil/",
"internal/go-proxmox/",
];
function normalizeRepoUrl(raw: string) {
let url = (raw ?? "").trim();
if (!url) return "";
// Common typo: "https://https://github.com/..."
url = url.replace(/^https?:\/\/https?:\/\//i, "https://");
if (!/^https?:\/\//i.test(url)) url = `https://${url}`;
url = url.replace(/\/+$/, "");
return url;
let url = (raw ?? "").trim();
if (!url) return "";
// Common typo: "https://https://github.com/..."
url = url.replace(/^https?:\/\/https?:\/\//i, "https://");
if (!/^https?:\/\//i.test(url)) url = `https://${url}`;
url = url.replace(/\/+$/, "");
return url;
}
function sanitizeFileStemFromPkgPath(pkgPath: string) {
// Convert a package path into a stable filename.
// Example: "internal/go-oidc/example" -> "internal-go-oidc-example"
// Keep it readable and unique (uses full path).
const parts = pkgPath
.split("/")
.filter(Boolean)
.map((p) => p.replace(/[^A-Za-z0-9._-]+/g, "-"));
const joined = parts.join("-");
return joined.replace(/-+/g, "-").replace(/^-|-$/g, "");
// Convert a package path into a stable filename.
// Example: "internal/go-oidc/example" -> "internal-go-oidc-example"
// Keep it readable and unique (uses full path).
const parts = pkgPath
.split("/")
.filter(Boolean)
.map((p) => p.replace(/[^A-Za-z0-9._-]+/g, "-"));
const joined = parts.join("-");
return joined.replace(/-+/g, "-").replace(/^-|-$/g, "");
}
function splitUrlAndFragment(url: string): {
urlNoFragment: string;
fragment: string;
urlNoFragment: string;
fragment: string;
} {
const i = url.indexOf("#");
if (i === -1) return { urlNoFragment: url, fragment: "" };
return { urlNoFragment: url.slice(0, i), fragment: url.slice(i) };
const i = url.indexOf("#");
if (i === -1) return { urlNoFragment: url, fragment: "" };
return { urlNoFragment: url.slice(0, i), fragment: url.slice(i) };
}
function isExternalOrAbsoluteUrl(url: string) {
// - absolute site links: "/foo"
// - pure fragments: "#bar"
// - external schemes: "https:", "mailto:", "vscode:", etc.
// IMPORTANT: don't treat "config.go:29" as a scheme.
if (url.startsWith("/") || url.startsWith("#")) return true;
if (url.includes("://")) return true;
return /^(https?|mailto|tel|vscode|file|data|ssh|git):/i.test(url);
// - absolute site links: "/foo"
// - pure fragments: "#bar"
// - external schemes: "https:", "mailto:", "vscode:", etc.
// IMPORTANT: don't treat "config.go:29" as a scheme.
if (url.startsWith("/") || url.startsWith("#")) return true;
if (url.includes("://")) return true;
return /^(https?|mailto|tel|vscode|file|data|ssh|git):/i.test(url);
}
function isRepoSourceFilePath(filePath: string) {
// Conservative allow-list: avoid rewriting .md (non-README) which may be VitePress docs.
return /\.(go|ts|tsx|js|jsx|py|sh|yml|yaml|json|toml|env|css|html|txt)$/i.test(
filePath,
);
// Conservative allow-list: avoid rewriting .md (non-README) which may be VitePress docs.
return /\.(go|ts|tsx|js|jsx|py|sh|yml|yaml|json|toml|env|css|html|txt)$/i.test(
filePath,
);
}
function parseFileLineSuffix(urlNoFragment: string): {
filePath: string;
line?: string;
filePath: string;
line?: string;
} {
// Match "file.ext:123" (line suffix), while leaving "file.ext" untouched.
const m = urlNoFragment.match(/^(.*?):(\d+)$/);
if (!m) return { filePath: urlNoFragment };
return { filePath: m[1] ?? urlNoFragment, line: m[2] };
// Match "file.ext:123" (line suffix), while leaving "file.ext" untouched.
const m = urlNoFragment.match(/^(.*?):(\d+)$/);
if (!m) return { filePath: urlNoFragment };
return { filePath: m[1] ?? urlNoFragment, line: m[2] };
}
function rewriteMarkdownLinksOutsideFences(
md: string,
rewriteInline: (url: string) => string,
md: string,
rewriteInline: (url: string) => string,
) {
const lines = md.split("\n");
let inFence = false;
const lines = md.split("\n");
let inFence = false;
for (let i = 0; i < lines.length; i++) {
const line = lines[i] ?? "";
const trimmed = line.trimStart();
if (trimmed.startsWith("```")) {
inFence = !inFence;
continue;
}
if (inFence) continue;
for (let i = 0; i < lines.length; i++) {
const line = lines[i] ?? "";
const trimmed = line.trimStart();
if (trimmed.startsWith("```")) {
inFence = !inFence;
continue;
}
if (inFence) continue;
// Inline markdown links/images: [text](url "title") / ![alt](url)
lines[i] = line.replace(
/\]\(([^)\s]+)(\s+"[^"]*")?\)/g,
(_full, urlRaw: string, maybeTitle: string | undefined) => {
const rewritten = rewriteInline(urlRaw);
return `](${rewritten}${maybeTitle ?? ""})`;
},
);
}
// Inline markdown links/images: [text](url "title") / ![alt](url)
lines[i] = line.replace(
/\]\(([^)\s]+)(\s+"[^"]*")?\)/g,
(_full, urlRaw: string, maybeTitle: string | undefined) => {
const rewritten = rewriteInline(urlRaw);
return `](${rewritten}${maybeTitle ?? ""})`;
},
);
}
return lines.join("\n");
return lines.join("\n");
}
function rewriteImplMarkdown(params: {
md: string;
pkgPath: string;
readmeRelToDocRoute: Map<string, string>;
dirPathToDocRoute: Map<string, string>;
repoUrl: string;
md: string;
pkgPath: string;
readmeRelToDocRoute: Map<string, string>;
dirPathToDocRoute: Map<string, string>;
repoUrl: string;
}) {
const { md, pkgPath, readmeRelToDocRoute, dirPathToDocRoute, repoUrl } =
params;
const { md, pkgPath, readmeRelToDocRoute, dirPathToDocRoute, repoUrl } =
params;
return rewriteMarkdownLinksOutsideFences(md, (urlRaw) => {
// Handle angle-bracketed destinations: (<./foo/README.md>)
const angleWrapped =
urlRaw.startsWith("<") && urlRaw.endsWith(">")
? urlRaw.slice(1, -1)
: urlRaw;
return rewriteMarkdownLinksOutsideFences(md, (urlRaw) => {
// Handle angle-bracketed destinations: (<./foo/README.md>)
const angleWrapped =
urlRaw.startsWith("<") && urlRaw.endsWith(">")
? urlRaw.slice(1, -1)
: urlRaw;
const { urlNoFragment, fragment } = splitUrlAndFragment(angleWrapped);
if (!urlNoFragment) return urlRaw;
if (isExternalOrAbsoluteUrl(urlNoFragment)) return urlRaw;
const { urlNoFragment, fragment } = splitUrlAndFragment(angleWrapped);
if (!urlNoFragment) return urlRaw;
if (isExternalOrAbsoluteUrl(urlNoFragment)) return urlRaw;
// 1) Directory links like "common" or "common/" that have a README
const dirPathNormalized = urlNoFragment.replace(/\/+$/, "");
let rewritten: string | undefined;
// First try exact match
if (dirPathToDocRoute.has(dirPathNormalized)) {
rewritten = `${dirPathToDocRoute.get(dirPathNormalized)}${fragment}`;
} else {
// Fallback: check parent directories for a README
// This handles paths like "internal/watcher/events" where only the parent has a README
let parentPath = dirPathNormalized;
while (parentPath.includes("/")) {
parentPath = parentPath.slice(0, parentPath.lastIndexOf("/"));
if (dirPathToDocRoute.has(parentPath)) {
rewritten = `${dirPathToDocRoute.get(parentPath)}${fragment}`;
break;
}
}
}
if (rewritten) {
return angleWrapped === urlRaw ? rewritten : `<${rewritten}>`;
}
// 1) Directory links like "common" or "common/" that have a README
const dirPathNormalized = urlNoFragment.replace(/\/+$/, "");
let rewritten: string | undefined;
// First try exact match
if (dirPathToDocRoute.has(dirPathNormalized)) {
rewritten = `${dirPathToDocRoute.get(dirPathNormalized)}${fragment}`;
} else {
// Fallback: check parent directories for a README
// This handles paths like "internal/watcher/events" where only the parent has a README
let parentPath = dirPathNormalized;
while (parentPath.includes("/")) {
parentPath = parentPath.slice(0, parentPath.lastIndexOf("/"));
if (dirPathToDocRoute.has(parentPath)) {
rewritten = `${dirPathToDocRoute.get(parentPath)}${fragment}`;
break;
}
}
}
if (rewritten) {
return angleWrapped === urlRaw ? rewritten : `<${rewritten}>`;
}
// 2) Intra-repo README links -> VitePress impl routes
if (/(^|\/)README\.md$/.test(urlNoFragment)) {
const targetReadmeRel = path.posix.normalize(
path.posix.join(pkgPath, urlNoFragment),
);
const route = readmeRelToDocRoute.get(targetReadmeRel);
if (route) {
const rewritten = `${route}${fragment}`;
return angleWrapped === urlRaw ? rewritten : `<${rewritten}>`;
}
return urlRaw;
}
// 2) Intra-repo README links -> VitePress impl routes
if (/(^|\/)README\.md$/.test(urlNoFragment)) {
const targetReadmeRel = path.posix.normalize(
path.posix.join(pkgPath, urlNoFragment),
);
const route = readmeRelToDocRoute.get(targetReadmeRel);
if (route) {
const rewritten = `${route}${fragment}`;
return angleWrapped === urlRaw ? rewritten : `<${rewritten}>`;
}
return urlRaw;
}
// 3) Local source-file references like "config.go:29" -> GitHub blob link
if (repoUrl) {
const { filePath, line } = parseFileLineSuffix(urlNoFragment);
if (isRepoSourceFilePath(filePath)) {
const repoRel = path.posix.normalize(
path.posix.join(pkgPath, filePath),
);
const githubUrl = `${repoUrl}/blob/main/${repoRel}${
line ? `#L${line}` : ""
}`;
const rewritten = `${githubUrl}${fragment}`;
return angleWrapped === urlRaw ? rewritten : `<${rewritten}>`;
}
}
// 3) Local source-file references like "config.go:29" -> GitHub blob link
if (repoUrl) {
const { filePath, line } = parseFileLineSuffix(urlNoFragment);
if (isRepoSourceFilePath(filePath)) {
const repoRel = path.posix.normalize(
path.posix.join(pkgPath, filePath),
);
const githubUrl = `${repoUrl}/blob/main/${repoRel}${
line ? `#L${line}` : ""
}`;
const rewritten = `${githubUrl}${fragment}`;
return angleWrapped === urlRaw ? rewritten : `<${rewritten}>`;
}
}
return urlRaw;
});
return urlRaw;
});
}
async function listRepoReadmes(repoRootAbs: string): Promise<string[]> {
const glob = new Glob("**/README.md");
const readmes: string[] = [];
const glob = new Glob("**/README.md");
const readmes: string[] = [];
for await (const rel of glob.scan({
cwd: repoRootAbs,
onlyFiles: true,
dot: false,
})) {
// Bun returns POSIX-style rel paths.
if (rel === "README.md") continue; // exclude root README
if (rel.startsWith(".git/") || rel.includes("/.git/")) continue;
if (rel.startsWith("node_modules/") || rel.includes("/node_modules/"))
continue;
let skip = false;
for (const submodule of skipSubmodules) {
if (rel.startsWith(submodule)) {
skip = true;
break;
}
}
if (skip) continue;
readmes.push(rel);
}
for await (const rel of glob.scan({
cwd: repoRootAbs,
onlyFiles: true,
dot: false,
})) {
// Bun returns POSIX-style rel paths.
if (rel === "README.md") continue; // exclude root README
if (rel.startsWith(".git/") || rel.includes("/.git/")) continue;
if (rel.startsWith("node_modules/") || rel.includes("/node_modules/"))
continue;
let skip = false;
for (const submodule of skipSubmodules) {
if (rel.startsWith(submodule)) {
skip = true;
break;
}
}
if (skip) continue;
readmes.push(rel);
}
// Deterministic order.
readmes.sort((a, b) => a.localeCompare(b));
return readmes;
// Deterministic order.
readmes.sort((a, b) => a.localeCompare(b));
return readmes;
}
async function writeImplDocToMdx(params: {
srcAbs: string;
dstAbs: string;
pkgPath: string;
readmeRelToDocRoute: Map<string, string>;
dirPathToDocRoute: Map<string, string>;
repoUrl: string;
async function writeImplDocCopy(params: {
srcAbs: string;
dstAbs: string;
pkgPath: string;
readmeRelToDocRoute: Map<string, string>;
dirPathToDocRoute: Map<string, string>;
repoUrl: string;
}) {
const {
srcAbs,
dstAbs,
pkgPath,
readmeRelToDocRoute,
dirPathToDocRoute,
repoUrl,
} = params;
await mkdir(path.dirname(dstAbs), { recursive: true });
const {
srcAbs,
dstAbs,
pkgPath,
readmeRelToDocRoute,
dirPathToDocRoute,
repoUrl,
} = params;
await mkdir(path.dirname(dstAbs), { recursive: true });
await rm(dstAbs, { force: true });
const original = await readFile(srcAbs, "utf8");
const current = await readFile(dstAbs, "utf-8");
const rewritten = md2mdx(
rewriteImplMarkdown({
md: original,
pkgPath,
readmeRelToDocRoute,
dirPathToDocRoute,
repoUrl,
}),
);
if (current === rewritten) {
return;
}
await writeFile(dstAbs, rewritten, "utf-8");
console.log(`[W] ${srcAbs} -> ${dstAbs}`);
const original = await readFile(srcAbs, "utf8");
const rewritten = rewriteImplMarkdown({
md: original,
pkgPath,
readmeRelToDocRoute,
dirPathToDocRoute,
repoUrl,
});
await writeFile(dstAbs, md2mdx(rewritten));
}
async function syncImplDocs(
repoRootAbs: string,
wikiRootAbs: string,
): Promise<void> {
const implDirAbs = path.join(wikiRootAbs, "content", "docs", "impl");
await mkdir(implDirAbs, { recursive: true });
repoRootAbs: string,
wikiRootAbs: string,
): Promise<ImplDoc[]> {
const implDirAbs = path.join(wikiRootAbs, "content", "docs", "impl");
await mkdir(implDirAbs, { recursive: true });
const readmes = await listRepoReadmes(repoRootAbs);
const expectedFileNames = new Set<string>();
expectedFileNames.add("index.mdx");
expectedFileNames.add("meta.json");
const readmes = await listRepoReadmes(repoRootAbs);
const docs: ImplDoc[] = [];
const expectedFileNames = new Set<string>();
expectedFileNames.add("index.mdx");
expectedFileNames.add("meta.json");
const repoUrl = normalizeRepoUrl(
Bun.env.REPO_URL ?? "https://github.com/yusing/godoxy",
);
const repoUrl = normalizeRepoUrl(
Bun.env.REPO_URL ?? "https://github.com/yusing/godoxy",
);
// Precompute mapping from repo-relative README path -> VitePress route.
// This lets us rewrite intra-repo README links when copying content.
const readmeRelToDocRoute = new Map<string, string>();
// Precompute mapping from repo-relative README path -> VitePress route.
// This lets us rewrite intra-repo README links when copying content.
const readmeRelToDocRoute = new Map<string, string>();
// Also precompute mapping from directory path -> VitePress route.
// This handles links like "[`common/`](common)" that point to directories with READMEs.
const dirPathToDocRoute = new Map<string, string>();
// Also precompute mapping from directory path -> VitePress route.
// This handles links like "[`common/`](common)" that point to directories with READMEs.
const dirPathToDocRoute = new Map<string, string>();
for (const readmeRel of readmes) {
const pkgPath = path.posix.dirname(readmeRel);
if (!pkgPath || pkgPath === ".") continue;
for (const readmeRel of readmes) {
const pkgPath = path.posix.dirname(readmeRel);
if (!pkgPath || pkgPath === ".") continue;
const docStem = sanitizeFileStemFromPkgPath(pkgPath);
if (!docStem) continue;
const route = `/impl/${docStem}`;
readmeRelToDocRoute.set(readmeRel, route);
dirPathToDocRoute.set(pkgPath, route);
}
const docStem = sanitizeFileStemFromPkgPath(pkgPath);
if (!docStem) continue;
const route = `/impl/${docStem}`;
readmeRelToDocRoute.set(readmeRel, route);
dirPathToDocRoute.set(pkgPath, route);
}
for (const readmeRel of readmes) {
const pkgPath = path.posix.dirname(readmeRel);
if (!pkgPath || pkgPath === ".") continue;
for (const readmeRel of readmes) {
const pkgPath = path.posix.dirname(readmeRel);
if (!pkgPath || pkgPath === ".") continue;
const docStem = sanitizeFileStemFromPkgPath(pkgPath);
if (!docStem) continue;
const docFileName = `${docStem}.mdx`;
const docStem = sanitizeFileStemFromPkgPath(pkgPath);
if (!docStem) continue;
const docFileName = `${docStem}.mdx`;
const docRoute = `/impl/${docStem}`;
const srcPathAbs = path.join(repoRootAbs, readmeRel);
const dstPathAbs = path.join(implDirAbs, docFileName);
const srcPathAbs = path.join(repoRootAbs, readmeRel);
const dstPathAbs = path.join(implDirAbs, docFileName);
await writeImplDocToMdx({
srcAbs: srcPathAbs,
dstAbs: dstPathAbs,
pkgPath,
readmeRelToDocRoute,
dirPathToDocRoute,
repoUrl,
});
await writeImplDocCopy({
srcAbs: srcPathAbs,
dstAbs: dstPathAbs,
pkgPath,
readmeRelToDocRoute,
dirPathToDocRoute,
repoUrl,
});
expectedFileNames.add(docFileName);
}
docs.push({ pkgPath, docFileName, docRoute, srcPathAbs, dstPathAbs });
expectedFileNames.add(docFileName);
}
// Clean orphaned impl docs.
const existing = await readdir(implDirAbs, { withFileTypes: true });
for (const ent of existing) {
if (!ent.isFile()) continue;
if (!ent.name.endsWith(".md")) continue;
if (expectedFileNames.has(ent.name)) continue;
await rm(path.join(implDirAbs, ent.name), { force: true });
}
// Clean orphaned impl docs.
const existing = await readdir(implDirAbs, { withFileTypes: true });
for (const ent of existing) {
if (!ent.isFile()) continue;
if (!ent.name.endsWith(".md")) continue;
if (expectedFileNames.has(ent.name)) continue;
await rm(path.join(implDirAbs, ent.name), { force: true });
}
// Deterministic for sidebar.
docs.sort((a, b) => a.pkgPath.localeCompare(b.pkgPath));
return docs;
}
async function main() {
// This script lives in `scripts/update-wiki/`, so repo root is two levels up.
const repoRootAbs = path.resolve(import.meta.dir, "../..");
// This script lives in `scripts/update-wiki/`, so repo root is two levels up.
const repoRootAbs = path.resolve(import.meta.dir);
// Required by task, but allow overriding via env for convenience.
const wikiRootAbs = Bun.env.DOCS_DIR
? path.resolve(repoRootAbs, Bun.env.DOCS_DIR)
: undefined;
// Required by task, but allow overriding via env for convenience.
const wikiRootAbs = Bun.env.DOCS_DIR
? path.resolve(repoRootAbs, Bun.env.DOCS_DIR)
: undefined;
if (!wikiRootAbs) {
throw new Error("DOCS_DIR is not set");
}
if (!wikiRootAbs) {
throw new Error("DOCS_DIR is not set");
}
await syncImplDocs(repoRootAbs, wikiRootAbs);
await syncImplDocs(repoRootAbs, wikiRootAbs);
}
await main();