Compare commits

..

15 Commits

Author SHA1 Message Date
yusing
95ffd35585 fix(rules): remove empty segments from splitPipe output
Refactored splitPipe function to use forEachPipePart helper, which filters
out empty segments instead of including them in the result. Updated test
expectation to match the new behavior where empty parts between pipes
are no longer included.
2026-02-24 02:52:19 +08:00
yusing
7b0d846576 fix(rules): prevent appending empty command parts in forEachPipePart and remove redundant calculation in parseDoWithBlocks function 2026-02-24 02:05:18 +08:00
yusing
458c7779d3 fix(tests/rules): correct semantic 2026-02-24 02:01:29 +08:00
yusing
dc6c649f2c fix(tests/rules): update HTTP flow YAML test for correct indentation and syntax
Fixes previous commit:  2a51c2ef52
2026-02-24 01:46:26 +08:00
yusing
3c5c3ecac2 fix(rules): handle empty matcher as unconditional rule
The matcherSignature function now treats empty strings as unconditional rules
that match any request,
returning "(any)" as the signature instead of
rejecting them.

This enables proper dead code detection when an unconditional
terminating rule shadows later rules.

Adds test coverage for detecting dead rules caused by unconditional
terminating rules.
2026-02-24 01:42:40 +08:00
yusing
a94442b001 fix(rules): prevent appending empty parts in splitPipe function 2026-02-24 01:36:54 +08:00
yusing
2a51c2ef52 fix(tests/rules): correct HTTP flow YAML test to use new yaml syntax
This is a test in yaml_test which meant to be testing old YAML syntax instead of new DSL
2026-02-24 01:36:35 +08:00
yusing
6477c35b15 docs(rules): update examples to use block syntax 2026-02-24 01:30:50 +08:00
yusing
5b20bbeb6f refactor(rules): simplify nested block detection by removing @ prefix requirement
Changes the nested block syntax detection from requiring `@`
as the first non-space character on a line to using a line-ending brace heuristic.

The parser now recognizes nested blocks when a line ends with an unquoted `{`,
simplifying the syntax and removing the mandatory `@` prefix while maintaining the same functionality.
2026-02-24 01:30:32 +08:00
yusing
5ba475c489 refactor(api/rules): remove IsResponseRule field from ParsedRule and related logic 2026-02-24 01:07:35 +08:00
yusing
54be056530 refactor(rules): improve termination detection and block parsing logic
Refactors the termination detection in the rules DSL to properly handle if-block and if-else-block commands.

Adds new functions `commandsTerminateInPre`, `commandTerminatesInPre`, and `ifElseBlockTerminatesInPre`
to recursively check if command sequences terminate in the pre-phase.

Also improves the Parse function to try block syntax first with proper error handling and fallback to YAML.

Includes test cases for dead code detection with terminating handlers in conditional blocks.
2026-02-24 01:05:54 +08:00
yusing
08de9086c3 fix(rules): buffer log output before writing to stdout/stderr 2026-02-24 00:12:29 +08:00
yusing
1a17f3943a refactor(rules): change default rule from baseline to fallback behavior
The default rule should runs only when no non-default pre rule matches, instead of running first as a baseline.
This follows the old behavior as before the pr is established.:

- Default rules act as fallback handlers that execute only when no matching non-default rule exists in the pre phase
- IfElseBlockCommand now returns early when a condition matches with a nil Do block, instead of falling through to else blocks
- Add nil check for auth handler to allow requests when no auth is configured
- Fix unterminated environment variable parsing to preserve input

Updates tests to verify the new fallback behavior where special rules suppress default rule execution.
2026-02-24 00:11:03 +08:00
yusing
9bb5c54e7c refactor(rules): defer error logging until after FlushRelease
Split error handling into isUnexpectedError predicate and logFlushError
function. Use rm.AppendError() to collect unexpected errors during rule
execution, then log after FlushRelease completes rather than immediately.
Also updates goutils dependency for AppendError method availability.
2026-02-23 23:09:24 +08:00
yusing
faecbab2cb refactor(rules): introduce block DSL, phase-based execution, and flow validation
- add block syntax parser/scanner with nested @blocks and elif/else support
- restructure rule execution into explicit pre/post phases with phase flags
- classify commands by phase and termination behavior
- enforce flow semantics (default rule handling, dead-rule detection)
- expand HTTP flow coverage with block + YAML parity tests and benches
- refresh rules README/spec and update playground/docs integration
2026-02-23 22:24:15 +08:00
9 changed files with 280 additions and 642 deletions

View File

@@ -13,8 +13,6 @@ This package implements a flexible HTTP middleware system for GoDoxy. Middleware
- **Bypass Rules**: Skip middleware based on request properties
- **Dynamic Loading**: Load middleware definitions from files at runtime
Response body rewriting is only applied to unencoded, text-like content types (for example `text/*`, JSON, YAML, XML). Response status and headers can always be modified.
## Architecture
```mermaid

View File

@@ -6,7 +6,6 @@ import (
"net/http"
"reflect"
"sort"
"strings"
"github.com/bytedance/sonic"
"github.com/rs/zerolog"
@@ -196,15 +195,21 @@ func (m *Middleware) ServeHTTP(next http.HandlerFunc, w http.ResponseWriter, r *
}
if exec, ok := m.impl.(ResponseModifier); ok {
rm := httputils.NewResponseModifier(w)
lrm := httputils.NewLazyResponseModifier(w, needsBuffering)
defer func() {
_, err := rm.FlushRelease()
_, err := lrm.FlushRelease()
if err != nil {
m.LogError(r).Err(err).Msg("failed to flush response")
}
}()
next(rm, r)
next(lrm, r)
// Skip modification if response wasn't buffered (non-HTML content)
if !lrm.IsBuffered() {
return
}
rm := lrm.ResponseModifier()
currentBody := rm.BodyReader()
currentResp := &http.Response{
StatusCode: rm.StatusCode(),
@@ -213,31 +218,20 @@ func (m *Middleware) ServeHTTP(next http.HandlerFunc, w http.ResponseWriter, r *
Body: currentBody,
Request: r,
}
allowBodyModification := canModifyResponseBody(currentResp)
respToModify := currentResp
if !allowBodyModification {
shadow := *currentResp
shadow.Body = eofReader{}
respToModify = &shadow
}
if err := exec.modifyResponse(respToModify); err != nil {
if err := exec.modifyResponse(currentResp); err != nil {
log.Err(err).Str("middleware", m.Name()).Str("url", fullURL(r)).Msg("failed to modify response")
}
// override the response status code
rm.WriteHeader(respToModify.StatusCode)
rm.WriteHeader(currentResp.StatusCode)
// overriding the response header
maps.Copy(rm.Header(), respToModify.Header)
maps.Copy(rm.Header(), currentResp.Header)
// override the content length and body if changed
if respToModify.Body != currentBody {
if allowBodyModification {
if err := rm.SetBody(respToModify.Body); err != nil {
m.LogError(r).Err(err).Msg("failed to set response body")
}
} else {
respToModify.Body.Close()
if currentResp.Body != currentBody {
if err := rm.SetBody(currentResp.Body); err != nil {
m.LogError(r).Err(err).Msg("failed to set response body")
}
}
} else {
@@ -245,55 +239,10 @@ func (m *Middleware) ServeHTTP(next http.HandlerFunc, w http.ResponseWriter, r *
}
}
func canModifyResponseBody(resp *http.Response) bool {
if hasNonIdentityEncoding(resp.TransferEncoding) {
return false
}
if hasNonIdentityEncoding(resp.Header.Values("Transfer-Encoding")) {
return false
}
if hasNonIdentityEncoding(resp.Header.Values("Content-Encoding")) {
return false
}
return isTextLikeMediaType(string(httputils.GetContentType(resp.Header)))
}
func hasNonIdentityEncoding(values []string) bool {
for _, value := range values {
for _, token := range strings.Split(value, ",") {
if strings.TrimSpace(token) == "" || strings.EqualFold(strings.TrimSpace(token), "identity") {
continue
}
return true
}
}
return false
}
func isTextLikeMediaType(contentType string) bool {
if contentType == "" {
return false
}
contentType = strings.ToLower(contentType)
if strings.HasPrefix(contentType, "text/") {
return true
}
if contentType == "application/json" || strings.HasSuffix(contentType, "+json") {
return true
}
if contentType == "application/xml" || strings.HasSuffix(contentType, "+xml") {
return true
}
if strings.Contains(contentType, "yaml") || strings.Contains(contentType, "toml") {
return true
}
if strings.Contains(contentType, "javascript") || strings.Contains(contentType, "ecmascript") {
return true
}
if strings.Contains(contentType, "csv") {
return true
}
return contentType == "application/x-www-form-urlencoded"
// needsBuffering determines if a response should be buffered for modification.
// Only HTML responses need buffering; streaming content (video, audio, etc.) should pass through.
func needsBuffering(header http.Header) bool {
return httputils.GetContentType(header).IsHTML()
}
func (m *Middleware) LogWarn(req *http.Request) *zerolog.Event {

View File

@@ -1,7 +1,6 @@
package middleware
import (
"maps"
"net/http"
"strconv"
@@ -47,23 +46,10 @@ func (m *middlewareChain) modifyResponse(resp *http.Response) error {
if len(m.modResps) == 0 {
return nil
}
allowBodyModification := canModifyResponseBody(resp)
for i, mr := range m.modResps {
respToModify := resp
if !allowBodyModification {
shadow := *resp
shadow.Body = eofReader{}
respToModify = &shadow
}
if err := mr.modifyResponse(respToModify); err != nil {
if err := mr.modifyResponse(resp); err != nil {
return gperr.PrependSubject(err, strconv.Itoa(i))
}
if !allowBodyModification {
resp.StatusCode = respToModify.StatusCode
if respToModify.Header != nil {
maps.Copy(resp.Header, respToModify.Header)
}
}
}
return nil
}

View File

@@ -1,7 +1,6 @@
package middleware
import (
"io"
"net/http"
"strconv"
"strings"
@@ -15,27 +14,12 @@ type testPriority struct {
}
var test = NewMiddleware[testPriority]()
var responseRewrite = NewMiddleware[testResponseRewrite]()
func (t testPriority) before(w http.ResponseWriter, r *http.Request) bool {
w.Header().Add("Test-Value", strconv.Itoa(t.Value))
return true
}
type testResponseRewrite struct {
StatusCode int `json:"status_code"`
HeaderKey string `json:"header_key"`
HeaderVal string `json:"header_val"`
Body string `json:"body"`
}
func (t testResponseRewrite) modifyResponse(resp *http.Response) error {
resp.StatusCode = t.StatusCode
resp.Header.Set(t.HeaderKey, t.HeaderVal)
resp.Body = io.NopCloser(strings.NewReader(t.Body))
return nil
}
func TestMiddlewarePriority(t *testing.T) {
priorities := []int{4, 7, 9, 0}
chain := make([]*Middleware, len(priorities))
@@ -51,85 +35,3 @@ func TestMiddlewarePriority(t *testing.T) {
expect.NoError(t, err)
expect.Equal(t, strings.Join(res.ResponseHeaders["Test-Value"], ","), "3,0,1,2")
}
func TestMiddlewareResponseRewriteGate(t *testing.T) {
opts := OptionsRaw{
"status_code": 418,
"header_key": "X-Rewrite",
"header_val": "1",
"body": "rewritten-body",
}
tests := []struct {
name string
respHeaders http.Header
respBody []byte
expectBody string
}{
{
name: "allow_body_rewrite_for_html",
respHeaders: http.Header{
"Content-Type": []string{"text/html; charset=utf-8"},
},
respBody: []byte("<html><body>original</body></html>"),
expectBody: "rewritten-body",
},
{
name: "allow_body_rewrite_for_json",
respHeaders: http.Header{
"Content-Type": []string{"application/json"},
},
respBody: []byte(`{"message":"original"}`),
expectBody: "rewritten-body",
},
{
name: "allow_body_rewrite_for_yaml",
respHeaders: http.Header{
"Content-Type": []string{"application/yaml"},
},
respBody: []byte("k: v"),
expectBody: "rewritten-body",
},
{
name: "block_body_rewrite_for_binary_content",
respHeaders: http.Header{
"Content-Type": []string{"application/octet-stream"},
},
respBody: []byte("binary"),
expectBody: "binary",
},
{
name: "block_body_rewrite_for_transfer_encoded_html",
respHeaders: http.Header{
"Content-Type": []string{"text/html"},
"Transfer-Encoding": []string{"chunked"},
},
respBody: []byte("<html><body>original</body></html>"),
expectBody: "<html><body>original</body></html>",
},
{
name: "block_body_rewrite_for_content_encoded_html",
respHeaders: http.Header{
"Content-Type": []string{"text/html"},
"Content-Encoding": []string{"gzip"},
},
respBody: []byte("<html><body>original</body></html>"),
expectBody: "<html><body>original</body></html>",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
result, err := newMiddlewareTest(responseRewrite, &testArgs{
middlewareOpt: opts,
respHeaders: tc.respHeaders,
respBody: tc.respBody,
respStatus: http.StatusOK,
})
expect.NoError(t, err)
expect.Equal(t, result.ResponseStatus, 418)
expect.Equal(t, result.ResponseHeaders.Get("X-Rewrite"), "1")
expect.Equal(t, string(result.Data), tc.expectBody)
})
}
}

View File

@@ -7,7 +7,6 @@ import (
"maps"
"net/http"
"net/http/httptest"
"strings"
"github.com/bytedance/sonic"
"github.com/yusing/godoxy/internal/common"
@@ -55,7 +54,7 @@ func (rt *requestRecorder) RoundTrip(req *http.Request) (resp *http.Response, er
resp = &http.Response{
Status: http.StatusText(rt.args.respStatus),
StatusCode: rt.args.respStatus,
Header: maps.Clone(testHeaders),
Header: testHeaders,
Body: io.NopCloser(bytes.NewReader(rt.args.respBody)),
ContentLength: int64(len(rt.args.respBody)),
Request: req,
@@ -66,27 +65,9 @@ func (rt *requestRecorder) RoundTrip(req *http.Request) (resp *http.Response, er
return nil, err
}
maps.Copy(resp.Header, rt.args.respHeaders)
if transferEncoding := resp.Header.Values("Transfer-Encoding"); len(transferEncoding) > 0 {
resp.TransferEncoding = parseHeaderTokens(transferEncoding)
resp.ContentLength = -1
}
return resp, nil
}
func parseHeaderTokens(values []string) []string {
var tokens []string
for _, value := range values {
for token := range strings.SplitSeq(value, ",") {
token = strings.TrimSpace(token)
if token == "" {
continue
}
tokens = append(tokens, token)
}
}
return tokens
}
type TestResult struct {
RequestHeaders http.Header
ResponseHeaders http.Header

View File

@@ -152,12 +152,6 @@ func ExpandVars(w *httputils.ResponseModifier, req *http.Request, src string, ds
return phase, err
}
i = nextIdx
// Expand any nested $func(...) expressions in args
args, argPhase, err := expandArgs(args, w, req)
if err != nil {
return phase, err
}
phase |= argPhase
actual, err = getter.get(args, w, req)
if err != nil {
return phase, err
@@ -227,18 +221,6 @@ func extractArgs(src string, i int, funcName string) (args []string, nextIdx int
continue
}
// Nested function call: $func(...) as an argument
if ch == '$' && arg.Len() == 0 {
// Capture the entire $func(...) expression as a raw argument token
nestedEnd, nestedErr := extractNestedFuncExpr(src, nextIdx)
if nestedErr != nil {
return nil, 0, nestedErr
}
args = append(args, src[nextIdx:nestedEnd+1])
nextIdx = nestedEnd + 1
continue
}
if ch == ')' {
// End of arguments
if arg.Len() > 0 {
@@ -274,70 +256,3 @@ func extractArgs(src string, i int, funcName string) (args []string, nextIdx int
}
return nil, 0, ErrUnterminatedParenthesis.Withf("func %q", funcName)
}
// extractNestedFuncExpr finds the end index (inclusive) of a $func(...) expression
// starting at position start in src. It handles nested parentheses.
func extractNestedFuncExpr(src string, start int) (endIdx int, err error) {
// src[start] must be '$'
i := start + 1
// skip the function name (valid var name chars)
for i < len(src) && validVarNameCharset[src[i]] {
i++
}
if i >= len(src) || src[i] != '(' {
return 0, ErrUnterminatedParenthesis.Withf("nested func at position %d", start)
}
// Now find the matching closing parenthesis, respecting quotes and nesting
depth := 0
var quote byte
for i < len(src) {
ch := src[i]
if quote != 0 {
if ch == quote {
quote = 0
}
i++
continue
}
if quoteChars[ch] {
quote = ch
i++
continue
}
switch ch {
case '(':
depth++
case ')':
depth--
if depth == 0 {
return i, nil
}
}
i++
}
if quote != 0 {
return 0, ErrUnterminatedQuotes.Withf("nested func at position %d", start)
}
return 0, ErrUnterminatedParenthesis.Withf("nested func at position %d", start)
}
// expandArgs expands any args that are nested dynamic var expressions (starting with '$').
// It returns the expanded args and the combined phase flags.
func expandArgs(args []string, w *httputils.ResponseModifier, req *http.Request) (expanded []string, phase PhaseFlag, err error) {
expanded = make([]string, len(args))
for i, arg := range args {
if len(arg) > 0 && arg[0] == '$' {
var buf strings.Builder
var argPhase PhaseFlag
argPhase, err = ExpandVars(w, req, arg, &buf)
if err != nil {
return nil, phase, err
}
phase |= argPhase
expanded[i] = buf.String()
} else {
expanded[i] = arg
}
}
return expanded, phase, nil
}

View File

@@ -6,7 +6,6 @@ import (
"strconv"
httputils "github.com/yusing/goutils/http"
strutils "github.com/yusing/goutils/strings"
)
var (
@@ -16,7 +15,6 @@ var (
VarQuery = "arg"
VarForm = "form"
VarPostForm = "postform"
VarRedacted = "redacted"
)
type dynamicVarGetter struct {
@@ -96,17 +94,6 @@ var dynamicVarSubsMap = map[string]dynamicVarGetter{
return getValueByKeyAtIndex(req.PostForm, key, index)
},
},
// VarRedacted wraps the result of its single argument (which may be another dynamic var
// expression, already expanded by expandArgs) with strutils.Redact.
VarRedacted: {
phase: PhaseNone,
get: func(args []string, w *httputils.ResponseModifier, req *http.Request) (string, error) {
if len(args) != 1 {
return "", ErrExpectOneArg
}
return strutils.Redact(args[0]), nil
},
},
}
func getValueByKeyAtIndex[Values http.Header | url.Values](values Values, key string, index int) (string, error) {

View File

@@ -189,64 +189,6 @@ func TestExtractArgs(t *testing.T) {
}
}
func TestExtractArgs_NestedFunc(t *testing.T) {
tests := []struct {
name string
src string
startPos int
funcName string
wantArgs []string
wantNextIdx int
wantErr bool
}{
{
name: "nested func as single arg",
src: "redacted($header(Authorization))",
startPos: 0,
funcName: "redacted",
wantArgs: []string{"$header(Authorization)"},
wantNextIdx: 31,
},
{
name: "nested func with quoted arg inside",
src: `redacted($header("X-Secret"))`,
startPos: 0,
funcName: "redacted",
wantArgs: []string{`$header("X-Secret")`},
wantNextIdx: 28,
},
{
name: "nested func with two args inside",
src: "redacted($header(X-Multi, 1))",
startPos: 0,
funcName: "redacted",
wantArgs: []string{"$header(X-Multi, 1)"},
wantNextIdx: 28,
},
{
name: "nested func missing closing paren",
src: "redacted($header(Authorization)",
startPos: 0,
funcName: "redacted",
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
args, nextIdx, err := extractArgs(tt.src, tt.startPos, tt.funcName)
if tt.wantErr {
require.Error(t, err)
} else {
require.NoError(t, err)
require.Equal(t, tt.wantArgs, args)
require.Equal(t, tt.wantNextIdx, nextIdx)
}
})
}
}
func TestExpandVars(t *testing.T) {
// Create a comprehensive test request with form data
formData := url.Values{}
@@ -504,27 +446,6 @@ func TestExpandVars(t *testing.T) {
input: "Header: $header(User-Agent), Status: $status_code",
want: "Header: test-agent/1.0, Status: 200",
},
// $redacted function
{
name: "redacted with plain string arg",
input: "$redacted(secret)",
want: "se**et",
},
{
name: "redacted wrapping header",
input: "$redacted($header(User-Agent))",
want: "te**********.0",
},
{
name: "redacted wrapping arg",
input: "$redacted($arg(param1))",
want: "va**e1",
},
{
name: "redacted with no args",
input: "$redacted()",
wantErr: true,
},
// Escaped dollar signs
{
name: "escaped dollar",

View File

@@ -4,336 +4,335 @@ import { Glob } from "bun";
import { md2mdx } from "./api-md2mdx";
type ImplDoc = {
/** Directory path relative to this repo, e.g. "internal/health/check" */
pkgPath: string;
/** File name in wiki `src/impl/`, e.g. "internal-health-check.md" */
docFileName: string;
/** VitePress route path (extensionless), e.g. "/impl/internal-health-check" */
docRoute: string;
/** Absolute source README path */
srcPathAbs: string;
/** Absolute destination doc path */
dstPathAbs: string;
/** Directory path relative to this repo, e.g. "internal/health/check" */
pkgPath: string;
/** File name in wiki `src/impl/`, e.g. "internal-health-check.md" */
docFileName: string;
/** VitePress route path (extensionless), e.g. "/impl/internal-health-check" */
docRoute: string;
/** Absolute source README path */
srcPathAbs: string;
/** Absolute destination doc path */
dstPathAbs: string;
};
const skipSubmodules = [
"internal/go-oidc/",
"internal/gopsutil/",
"internal/go-proxmox/",
"internal/go-oidc/",
"internal/gopsutil/",
"internal/go-proxmox/",
];
function normalizeRepoUrl(raw: string) {
let url = (raw ?? "").trim();
if (!url) return "";
// Common typo: "https://https://github.com/..."
url = url.replace(/^https?:\/\/https?:\/\//i, "https://");
if (!/^https?:\/\//i.test(url)) url = `https://${url}`;
url = url.replace(/\/+$/, "");
return url;
let url = (raw ?? "").trim();
if (!url) return "";
// Common typo: "https://https://github.com/..."
url = url.replace(/^https?:\/\/https?:\/\//i, "https://");
if (!/^https?:\/\//i.test(url)) url = `https://${url}`;
url = url.replace(/\/+$/, "");
return url;
}
function sanitizeFileStemFromPkgPath(pkgPath: string) {
// Convert a package path into a stable filename.
// Example: "internal/go-oidc/example" -> "internal-go-oidc-example"
// Keep it readable and unique (uses full path).
const parts = pkgPath
.split("/")
.filter(Boolean)
.map((p) => p.replace(/[^A-Za-z0-9._-]+/g, "-"));
const joined = parts.join("-");
return joined.replace(/-+/g, "-").replace(/^-|-$/g, "");
// Convert a package path into a stable filename.
// Example: "internal/go-oidc/example" -> "internal-go-oidc-example"
// Keep it readable and unique (uses full path).
const parts = pkgPath
.split("/")
.filter(Boolean)
.map((p) => p.replace(/[^A-Za-z0-9._-]+/g, "-"));
const joined = parts.join("-");
return joined.replace(/-+/g, "-").replace(/^-|-$/g, "");
}
function splitUrlAndFragment(url: string): {
urlNoFragment: string;
fragment: string;
urlNoFragment: string;
fragment: string;
} {
const i = url.indexOf("#");
if (i === -1) return { urlNoFragment: url, fragment: "" };
return { urlNoFragment: url.slice(0, i), fragment: url.slice(i) };
const i = url.indexOf("#");
if (i === -1) return { urlNoFragment: url, fragment: "" };
return { urlNoFragment: url.slice(0, i), fragment: url.slice(i) };
}
function isExternalOrAbsoluteUrl(url: string) {
// - absolute site links: "/foo"
// - pure fragments: "#bar"
// - external schemes: "https:", "mailto:", "vscode:", etc.
// IMPORTANT: don't treat "config.go:29" as a scheme.
if (url.startsWith("/") || url.startsWith("#")) return true;
if (url.includes("://")) return true;
return /^(https?|mailto|tel|vscode|file|data|ssh|git):/i.test(url);
// - absolute site links: "/foo"
// - pure fragments: "#bar"
// - external schemes: "https:", "mailto:", "vscode:", etc.
// IMPORTANT: don't treat "config.go:29" as a scheme.
if (url.startsWith("/") || url.startsWith("#")) return true;
if (url.includes("://")) return true;
return /^(https?|mailto|tel|vscode|file|data|ssh|git):/i.test(url);
}
function isRepoSourceFilePath(filePath: string) {
// Conservative allow-list: avoid rewriting .md (non-README) which may be VitePress docs.
return /\.(go|ts|tsx|js|jsx|py|sh|yml|yaml|json|toml|env|css|html|txt)$/i.test(
filePath,
);
// Conservative allow-list: avoid rewriting .md (non-README) which may be VitePress docs.
return /\.(go|ts|tsx|js|jsx|py|sh|yml|yaml|json|toml|env|css|html|txt)$/i.test(
filePath,
);
}
function parseFileLineSuffix(urlNoFragment: string): {
filePath: string;
line?: string;
filePath: string;
line?: string;
} {
// Match "file.ext:123" (line suffix), while leaving "file.ext" untouched.
const m = urlNoFragment.match(/^(.*?):(\d+)$/);
if (!m) return { filePath: urlNoFragment };
return { filePath: m[1] ?? urlNoFragment, line: m[2] };
// Match "file.ext:123" (line suffix), while leaving "file.ext" untouched.
const m = urlNoFragment.match(/^(.*?):(\d+)$/);
if (!m) return { filePath: urlNoFragment };
return { filePath: m[1] ?? urlNoFragment, line: m[2] };
}
function rewriteMarkdownLinksOutsideFences(
md: string,
rewriteInline: (url: string) => string,
md: string,
rewriteInline: (url: string) => string,
) {
const lines = md.split("\n");
let inFence = false;
const lines = md.split("\n");
let inFence = false;
for (let i = 0; i < lines.length; i++) {
const line = lines[i] ?? "";
const trimmed = line.trimStart();
if (trimmed.startsWith("```")) {
inFence = !inFence;
continue;
}
if (inFence) continue;
for (let i = 0; i < lines.length; i++) {
const line = lines[i] ?? "";
const trimmed = line.trimStart();
if (trimmed.startsWith("```")) {
inFence = !inFence;
continue;
}
if (inFence) continue;
// Inline markdown links/images: [text](url "title") / ![alt](url)
lines[i] = line.replace(
/\]\(([^)\s]+)(\s+"[^"]*")?\)/g,
(_full, urlRaw: string, maybeTitle: string | undefined) => {
const rewritten = rewriteInline(urlRaw);
return `](${rewritten}${maybeTitle ?? ""})`;
},
);
}
// Inline markdown links/images: [text](url "title") / ![alt](url)
lines[i] = line.replace(
/\]\(([^)\s]+)(\s+"[^"]*")?\)/g,
(_full, urlRaw: string, maybeTitle: string | undefined) => {
const rewritten = rewriteInline(urlRaw);
return `](${rewritten}${maybeTitle ?? ""})`;
},
);
}
return lines.join("\n");
return lines.join("\n");
}
function rewriteImplMarkdown(params: {
md: string;
pkgPath: string;
readmeRelToDocRoute: Map<string, string>;
dirPathToDocRoute: Map<string, string>;
repoUrl: string;
md: string;
pkgPath: string;
readmeRelToDocRoute: Map<string, string>;
dirPathToDocRoute: Map<string, string>;
repoUrl: string;
}) {
const { md, pkgPath, readmeRelToDocRoute, dirPathToDocRoute, repoUrl } =
params;
const { md, pkgPath, readmeRelToDocRoute, dirPathToDocRoute, repoUrl } =
params;
return rewriteMarkdownLinksOutsideFences(md, (urlRaw) => {
// Handle angle-bracketed destinations: (<./foo/README.md>)
const angleWrapped =
urlRaw.startsWith("<") && urlRaw.endsWith(">")
? urlRaw.slice(1, -1)
: urlRaw;
return rewriteMarkdownLinksOutsideFences(md, (urlRaw) => {
// Handle angle-bracketed destinations: (<./foo/README.md>)
const angleWrapped =
urlRaw.startsWith("<") && urlRaw.endsWith(">")
? urlRaw.slice(1, -1)
: urlRaw;
const { urlNoFragment, fragment } = splitUrlAndFragment(angleWrapped);
if (!urlNoFragment) return urlRaw;
if (isExternalOrAbsoluteUrl(urlNoFragment)) return urlRaw;
const { urlNoFragment, fragment } = splitUrlAndFragment(angleWrapped);
if (!urlNoFragment) return urlRaw;
if (isExternalOrAbsoluteUrl(urlNoFragment)) return urlRaw;
// 1) Directory links like "common" or "common/" that have a README
const dirPathNormalized = urlNoFragment.replace(/\/+$/, "");
let rewritten: string | undefined;
// First try exact match
if (dirPathToDocRoute.has(dirPathNormalized)) {
rewritten = `${dirPathToDocRoute.get(dirPathNormalized)}${fragment}`;
} else {
// Fallback: check parent directories for a README
// This handles paths like "internal/watcher/events" where only the parent has a README
let parentPath = dirPathNormalized;
while (parentPath.includes("/")) {
parentPath = parentPath.slice(0, parentPath.lastIndexOf("/"));
if (dirPathToDocRoute.has(parentPath)) {
rewritten = `${dirPathToDocRoute.get(parentPath)}${fragment}`;
break;
}
}
}
if (rewritten) {
return angleWrapped === urlRaw ? rewritten : `<${rewritten}>`;
}
// 1) Directory links like "common" or "common/" that have a README
const dirPathNormalized = urlNoFragment.replace(/\/+$/, "");
let rewritten: string | undefined;
// First try exact match
if (dirPathToDocRoute.has(dirPathNormalized)) {
rewritten = `${dirPathToDocRoute.get(dirPathNormalized)}${fragment}`;
} else {
// Fallback: check parent directories for a README
// This handles paths like "internal/watcher/events" where only the parent has a README
let parentPath = dirPathNormalized;
while (parentPath.includes("/")) {
parentPath = parentPath.slice(0, parentPath.lastIndexOf("/"));
if (dirPathToDocRoute.has(parentPath)) {
rewritten = `${dirPathToDocRoute.get(parentPath)}${fragment}`;
break;
}
}
}
if (rewritten) {
return angleWrapped === urlRaw ? rewritten : `<${rewritten}>`;
}
// 2) Intra-repo README links -> VitePress impl routes
if (/(^|\/)README\.md$/.test(urlNoFragment)) {
const targetReadmeRel = path.posix.normalize(
path.posix.join(pkgPath, urlNoFragment),
);
const route = readmeRelToDocRoute.get(targetReadmeRel);
if (route) {
const rewritten = `${route}${fragment}`;
return angleWrapped === urlRaw ? rewritten : `<${rewritten}>`;
}
return urlRaw;
}
// 2) Intra-repo README links -> VitePress impl routes
if (/(^|\/)README\.md$/.test(urlNoFragment)) {
const targetReadmeRel = path.posix.normalize(
path.posix.join(pkgPath, urlNoFragment),
);
const route = readmeRelToDocRoute.get(targetReadmeRel);
if (route) {
const rewritten = `${route}${fragment}`;
return angleWrapped === urlRaw ? rewritten : `<${rewritten}>`;
}
return urlRaw;
}
// 3) Local source-file references like "config.go:29" -> GitHub blob link
if (repoUrl) {
const { filePath, line } = parseFileLineSuffix(urlNoFragment);
if (isRepoSourceFilePath(filePath)) {
const repoRel = path.posix.normalize(
path.posix.join(pkgPath, filePath),
);
const githubUrl = `${repoUrl}/blob/main/${repoRel}${
line ? `#L${line}` : ""
}`;
const rewritten = `${githubUrl}${fragment}`;
return angleWrapped === urlRaw ? rewritten : `<${rewritten}>`;
}
}
// 3) Local source-file references like "config.go:29" -> GitHub blob link
if (repoUrl) {
const { filePath, line } = parseFileLineSuffix(urlNoFragment);
if (isRepoSourceFilePath(filePath)) {
const repoRel = path.posix.normalize(
path.posix.join(pkgPath, filePath),
);
const githubUrl = `${repoUrl}/blob/main/${repoRel}${
line ? `#L${line}` : ""
}`;
const rewritten = `${githubUrl}${fragment}`;
return angleWrapped === urlRaw ? rewritten : `<${rewritten}>`;
}
}
return urlRaw;
});
return urlRaw;
});
}
async function listRepoReadmes(repoRootAbs: string): Promise<string[]> {
const glob = new Glob("**/README.md");
const readmes: string[] = [];
const glob = new Glob("**/README.md");
const readmes: string[] = [];
for await (const rel of glob.scan({
cwd: repoRootAbs,
onlyFiles: true,
dot: false,
})) {
// Bun returns POSIX-style rel paths.
if (rel === "README.md") continue; // exclude root README
if (rel.startsWith(".git/") || rel.includes("/.git/")) continue;
if (rel.startsWith("node_modules/") || rel.includes("/node_modules/"))
continue;
let skip = false;
for (const submodule of skipSubmodules) {
if (rel.startsWith(submodule)) {
skip = true;
break;
}
}
if (skip) continue;
readmes.push(rel);
}
for await (const rel of glob.scan({
cwd: repoRootAbs,
onlyFiles: true,
dot: false,
})) {
// Bun returns POSIX-style rel paths.
if (rel === "README.md") continue; // exclude root README
if (rel.startsWith(".git/") || rel.includes("/.git/")) continue;
if (rel.startsWith("node_modules/") || rel.includes("/node_modules/"))
continue;
let skip = false;
for (const submodule of skipSubmodules) {
if (rel.startsWith(submodule)) {
skip = true;
break;
}
}
if (skip) continue;
readmes.push(rel);
}
// Deterministic order.
readmes.sort((a, b) => a.localeCompare(b));
return readmes;
// Deterministic order.
readmes.sort((a, b) => a.localeCompare(b));
return readmes;
}
async function writeImplDocToMdx(params: {
srcAbs: string;
dstAbs: string;
pkgPath: string;
readmeRelToDocRoute: Map<string, string>;
dirPathToDocRoute: Map<string, string>;
repoUrl: string;
async function writeImplDocCopy(params: {
srcAbs: string;
dstAbs: string;
pkgPath: string;
readmeRelToDocRoute: Map<string, string>;
dirPathToDocRoute: Map<string, string>;
repoUrl: string;
}) {
const {
srcAbs,
dstAbs,
pkgPath,
readmeRelToDocRoute,
dirPathToDocRoute,
repoUrl,
} = params;
await mkdir(path.dirname(dstAbs), { recursive: true });
const {
srcAbs,
dstAbs,
pkgPath,
readmeRelToDocRoute,
dirPathToDocRoute,
repoUrl,
} = params;
await mkdir(path.dirname(dstAbs), { recursive: true });
await rm(dstAbs, { force: true });
const original = await readFile(srcAbs, "utf8");
const current = await readFile(dstAbs, "utf-8");
const rewritten = md2mdx(
rewriteImplMarkdown({
md: original,
pkgPath,
readmeRelToDocRoute,
dirPathToDocRoute,
repoUrl,
}),
);
if (current === rewritten) {
return;
}
await writeFile(dstAbs, rewritten, "utf-8");
console.log(`[W] ${srcAbs} -> ${dstAbs}`);
const original = await readFile(srcAbs, "utf8");
const rewritten = rewriteImplMarkdown({
md: original,
pkgPath,
readmeRelToDocRoute,
dirPathToDocRoute,
repoUrl,
});
await writeFile(dstAbs, md2mdx(rewritten));
}
async function syncImplDocs(
repoRootAbs: string,
wikiRootAbs: string,
): Promise<void> {
const implDirAbs = path.join(wikiRootAbs, "content", "docs", "impl");
await mkdir(implDirAbs, { recursive: true });
repoRootAbs: string,
wikiRootAbs: string,
): Promise<ImplDoc[]> {
const implDirAbs = path.join(wikiRootAbs, "content", "docs", "impl");
await mkdir(implDirAbs, { recursive: true });
const readmes = await listRepoReadmes(repoRootAbs);
const expectedFileNames = new Set<string>();
expectedFileNames.add("index.mdx");
expectedFileNames.add("meta.json");
const readmes = await listRepoReadmes(repoRootAbs);
const docs: ImplDoc[] = [];
const expectedFileNames = new Set<string>();
expectedFileNames.add("index.mdx");
expectedFileNames.add("meta.json");
const repoUrl = normalizeRepoUrl(
Bun.env.REPO_URL ?? "https://github.com/yusing/godoxy",
);
const repoUrl = normalizeRepoUrl(
Bun.env.REPO_URL ?? "https://github.com/yusing/godoxy",
);
// Precompute mapping from repo-relative README path -> VitePress route.
// This lets us rewrite intra-repo README links when copying content.
const readmeRelToDocRoute = new Map<string, string>();
// Precompute mapping from repo-relative README path -> VitePress route.
// This lets us rewrite intra-repo README links when copying content.
const readmeRelToDocRoute = new Map<string, string>();
// Also precompute mapping from directory path -> VitePress route.
// This handles links like "[`common/`](common)" that point to directories with READMEs.
const dirPathToDocRoute = new Map<string, string>();
// Also precompute mapping from directory path -> VitePress route.
// This handles links like "[`common/`](common)" that point to directories with READMEs.
const dirPathToDocRoute = new Map<string, string>();
for (const readmeRel of readmes) {
const pkgPath = path.posix.dirname(readmeRel);
if (!pkgPath || pkgPath === ".") continue;
for (const readmeRel of readmes) {
const pkgPath = path.posix.dirname(readmeRel);
if (!pkgPath || pkgPath === ".") continue;
const docStem = sanitizeFileStemFromPkgPath(pkgPath);
if (!docStem) continue;
const route = `/impl/${docStem}`;
readmeRelToDocRoute.set(readmeRel, route);
dirPathToDocRoute.set(pkgPath, route);
}
const docStem = sanitizeFileStemFromPkgPath(pkgPath);
if (!docStem) continue;
const route = `/impl/${docStem}`;
readmeRelToDocRoute.set(readmeRel, route);
dirPathToDocRoute.set(pkgPath, route);
}
for (const readmeRel of readmes) {
const pkgPath = path.posix.dirname(readmeRel);
if (!pkgPath || pkgPath === ".") continue;
for (const readmeRel of readmes) {
const pkgPath = path.posix.dirname(readmeRel);
if (!pkgPath || pkgPath === ".") continue;
const docStem = sanitizeFileStemFromPkgPath(pkgPath);
if (!docStem) continue;
const docFileName = `${docStem}.mdx`;
const docStem = sanitizeFileStemFromPkgPath(pkgPath);
if (!docStem) continue;
const docFileName = `${docStem}.mdx`;
const docRoute = `/impl/${docStem}`;
const srcPathAbs = path.join(repoRootAbs, readmeRel);
const dstPathAbs = path.join(implDirAbs, docFileName);
const srcPathAbs = path.join(repoRootAbs, readmeRel);
const dstPathAbs = path.join(implDirAbs, docFileName);
await writeImplDocToMdx({
srcAbs: srcPathAbs,
dstAbs: dstPathAbs,
pkgPath,
readmeRelToDocRoute,
dirPathToDocRoute,
repoUrl,
});
await writeImplDocCopy({
srcAbs: srcPathAbs,
dstAbs: dstPathAbs,
pkgPath,
readmeRelToDocRoute,
dirPathToDocRoute,
repoUrl,
});
expectedFileNames.add(docFileName);
}
docs.push({ pkgPath, docFileName, docRoute, srcPathAbs, dstPathAbs });
expectedFileNames.add(docFileName);
}
// Clean orphaned impl docs.
const existing = await readdir(implDirAbs, { withFileTypes: true });
for (const ent of existing) {
if (!ent.isFile()) continue;
if (!ent.name.endsWith(".md")) continue;
if (expectedFileNames.has(ent.name)) continue;
await rm(path.join(implDirAbs, ent.name), { force: true });
}
// Clean orphaned impl docs.
const existing = await readdir(implDirAbs, { withFileTypes: true });
for (const ent of existing) {
if (!ent.isFile()) continue;
if (!ent.name.endsWith(".md")) continue;
if (expectedFileNames.has(ent.name)) continue;
await rm(path.join(implDirAbs, ent.name), { force: true });
}
// Deterministic for sidebar.
docs.sort((a, b) => a.pkgPath.localeCompare(b.pkgPath));
return docs;
}
async function main() {
// This script lives in `scripts/update-wiki/`, so repo root is two levels up.
const repoRootAbs = path.resolve(import.meta.dir, "../..");
// This script lives in `scripts/update-wiki/`, so repo root is two levels up.
const repoRootAbs = path.resolve(import.meta.dir);
// Required by task, but allow overriding via env for convenience.
const wikiRootAbs = Bun.env.DOCS_DIR
? path.resolve(repoRootAbs, Bun.env.DOCS_DIR)
: undefined;
// Required by task, but allow overriding via env for convenience.
const wikiRootAbs = Bun.env.DOCS_DIR
? path.resolve(repoRootAbs, Bun.env.DOCS_DIR)
: undefined;
if (!wikiRootAbs) {
throw new Error("DOCS_DIR is not set");
}
if (!wikiRootAbs) {
throw new Error("DOCS_DIR is not set");
}
await syncImplDocs(repoRootAbs, wikiRootAbs);
await syncImplDocs(repoRootAbs, wikiRootAbs);
}
await main();