refactor(scripts/wiki): rewrite markdown links when syncing impl docs to wiki

- Convert intra-repo README links to VitePress routes for SPA navigation
- Rewrite source file references (e.g., config.go:29) to GitHub blob links
- Makefile now passes REPO_URL to update-wiki for link rewriting
- Correct agent README.md file links from full to relative paths
- skip introduction.md when syncing
This commit is contained in:
yusing
2026-01-10 13:54:22 +08:00
parent 4ec352f1f6
commit cc1fe30045
3 changed files with 232 additions and 29 deletions

View File

@@ -3,6 +3,8 @@ export VERSION ?= $(shell git describe --tags --abbrev=0)
export BUILD_DATE ?= $(shell date -u +'%Y%m%d-%H%M') export BUILD_DATE ?= $(shell date -u +'%Y%m%d-%H%M')
export GOOS = linux export GOOS = linux
REPO_URL ?= https://github.com/yusing/godoxy
WEBUI_DIR ?= ../godoxy-webui WEBUI_DIR ?= ../godoxy-webui
DOCS_DIR ?= ${WEBUI_DIR}/wiki DOCS_DIR ?= ${WEBUI_DIR}/wiki
@@ -175,4 +177,4 @@ gen-api-types: gen-swagger
.PHONY: update-wiki .PHONY: update-wiki
update-wiki: update-wiki:
DOCS_DIR=${DOCS_DIR} bun --bun scripts/update-wiki/main.ts DOCS_DIR=${DOCS_DIR} REPO_URL=${REPO_URL} bun --bun scripts/update-wiki/main.ts

View File

@@ -27,26 +27,26 @@ graph TD
## File Structure ## File Structure
| File | Purpose | | File | Purpose |
| -------------------------------------------------------- | --------------------------------------------------------- | | ---------------------------------------- | --------------------------------------------------------- |
| [`config.go`](agent/pkg/agent/config.go) | Core configuration, initialization, and API client logic. | | [`config.go`](config.go) | Core configuration, initialization, and API client logic. |
| [`new_agent.go`](agent/pkg/agent/new_agent.go) | Agent creation and certificate generation logic. | | [`new_agent.go`](new_agent.go) | Agent creation and certificate generation logic. |
| [`docker_compose.go`](agent/pkg/agent/docker_compose.go) | Generator for agent Docker Compose configurations. | | [`docker_compose.go`](docker_compose.go) | Generator for agent Docker Compose configurations. |
| [`bare_metal.go`](agent/pkg/agent/bare_metal.go) | Generator for bare metal installation scripts. | | [`bare_metal.go`](bare_metal.go) | Generator for bare metal installation scripts. |
| [`env.go`](agent/pkg/agent/env.go) | Environment configuration types and constants. | | [`env.go`](env.go) | Environment configuration types and constants. |
| [`common/`](agent/pkg/agent/common) | Shared constants and utilities for agents. | | `common/` | Shared constants and utilities for agents. |
## Core Types ## Core Types
### [`AgentConfig`](agent/pkg/agent/config.go:29) ### [`AgentConfig`](config.go:29)
The primary struct used by the GoDoxy server to manage a connection to an agent. It stores the agent's address, metadata, and TLS configuration. The primary struct used by the GoDoxy server to manage a connection to an agent. It stores the agent's address, metadata, and TLS configuration.
### [`AgentInfo`](agent/pkg/agent/config.go:45) ### [`AgentInfo`](config.go:45)
Contains basic metadata about the agent, including its version, name, and container runtime (Docker or Podman). Contains basic metadata about the agent, including its version, name, and container runtime (Docker or Podman).
### [`PEMPair`](agent/pkg/agent/new_agent.go:53) ### [`PEMPair`](new_agent.go:53)
A utility struct for handling PEM-encoded certificate and key pairs, supporting encryption, decryption, and conversion to `tls.Certificate`. A utility struct for handling PEM-encoded certificate and key pairs, supporting encryption, decryption, and conversion to `tls.Certificate`.
@@ -54,7 +54,7 @@ A utility struct for handling PEM-encoded certificate and key pairs, supporting
### Certificate Generation ### Certificate Generation
The [`NewAgent`](agent/pkg/agent/new_agent.go:147) function creates a complete certificate infrastructure for an agent: The [`NewAgent`](new_agent.go:147) function creates a complete certificate infrastructure for an agent:
- **CA Certificate**: Self-signed root certificate with 1000-year validity. - **CA Certificate**: Self-signed root certificate with 1000-year validity.
- **Server Certificate**: For the agent's HTTPS server, signed by the CA. - **Server Certificate**: For the agent's HTTPS server, signed by the CA.
@@ -65,18 +65,18 @@ All certificates use ECDSA with P-256 curve and SHA-256 signatures.
### Certificate Security ### Certificate Security
- Certificates are encrypted using AES-GCM with a provided encryption key. - Certificates are encrypted using AES-GCM with a provided encryption key.
- The [`PEMPair`](agent/pkg/agent/new_agent.go:53) struct provides methods for encryption, decryption, and conversion to `tls.Certificate`. - The [`PEMPair`](new_agent.go:53) struct provides methods for encryption, decryption, and conversion to `tls.Certificate`.
- Base64 encoding is used for certificate storage and transmission. - Base64 encoding is used for certificate storage and transmission.
## Key Features ## Key Features
### 1. Secure Communication ### 1. Secure Communication
All communication between the GoDoxy server and agents is secured using mutual TLS (mTLS). The [`AgentConfig`](agent/pkg/agent/config.go:29) handles the loading of CA and client certificates to establish secure connections. All communication between the GoDoxy server and agents is secured using mutual TLS (mTLS). The [`AgentConfig`](config.go:29) handles the loading of CA and client certificates to establish secure connections.
### 2. Agent Discovery and Initialization ### 2. Agent Discovery and Initialization
The [`Init`](agent/pkg/agent/config.go:231) and [`InitWithCerts`](agent/pkg/agent/config.go:110) methods allow the server to: The [`Init`](config.go:231) and [`InitWithCerts`](config.go:110) methods allow the server to:
- Fetch agent metadata (version, name, runtime). - Fetch agent metadata (version, name, runtime).
- Verify compatibility between server and agent versions. - Verify compatibility between server and agent versions.
@@ -86,12 +86,12 @@ The [`Init`](agent/pkg/agent/config.go:231) and [`InitWithCerts`](agent/pkg/agen
The package provides interfaces and implementations for generating deployment artifacts: The package provides interfaces and implementations for generating deployment artifacts:
- **Docker Compose**: Generates a `docker-compose.yml` for running the agent as a container via [`AgentComposeConfig.Generate()`](agent/pkg/agent/docker_compose.go:21). - **Docker Compose**: Generates a `docker-compose.yml` for running the agent as a container via [`AgentComposeConfig.Generate()`](docker_compose.go:21).
- **Bare Metal**: Generates a shell script to install and run the agent as a systemd service via [`AgentEnvConfig.Generate()`](agent/pkg/agent/bare_metal.go:27). - **Bare Metal**: Generates a shell script to install and run the agent as a systemd service via [`AgentEnvConfig.Generate()`](bare_metal.go:27).
### 4. Fake Docker Host ### 4. Fake Docker Host
The package supports a "fake" Docker host scheme (`agent://<addr>`) to identify containers managed by an agent, allowing the GoDoxy server to route requests appropriately. See [`IsDockerHostAgent`](agent/pkg/agent/config.go:90) and [`GetAgentAddrFromDockerHost`](agent/pkg/agent/config.go:94). The package supports a "fake" Docker host scheme (`agent://<addr>`) to identify containers managed by an agent, allowing the GoDoxy server to route requests appropriately. See [`IsDockerHostAgent`](config.go:90) and [`GetAgentAddrFromDockerHost`](config.go:94).
## Usage Example ## Usage Example

View File

@@ -1,5 +1,4 @@
import { Glob } from "bun"; import { Glob } from "bun";
import { linkSync } from "fs";
import { mkdir, readdir, readFile, rm, writeFile } from "fs/promises"; import { mkdir, readdir, readFile, rm, writeFile } from "fs/promises";
import path from "path"; import path from "path";
@@ -8,6 +7,8 @@ type ImplDoc = {
pkgPath: string; pkgPath: string;
/** File name in wiki `src/impl/`, e.g. "internal-health-check.md" */ /** File name in wiki `src/impl/`, e.g. "internal-health-check.md" */
docFileName: string; docFileName: string;
/** VitePress route path (extensionless), e.g. "/impl/internal-health-check" */
docRoute: string;
/** Absolute source README path */ /** Absolute source README path */
srcPathAbs: string; srcPathAbs: string;
/** Absolute destination doc path */ /** Absolute destination doc path */
@@ -17,6 +18,8 @@ type ImplDoc = {
const START_MARKER = "// GENERATED-IMPL-SIDEBAR-START"; const START_MARKER = "// GENERATED-IMPL-SIDEBAR-START";
const END_MARKER = "// GENERATED-IMPL-SIDEBAR-END"; const END_MARKER = "// GENERATED-IMPL-SIDEBAR-END";
const skipSubmodules = ["internal/go-oidc/", "internal/gopsutil/"];
function escapeRegex(s: string) { function escapeRegex(s: string) {
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
} }
@@ -25,6 +28,16 @@ function escapeSingleQuotedTs(s: string) {
return s.replace(/\\/g, "\\\\").replace(/'/g, "\\'"); return s.replace(/\\/g, "\\\\").replace(/'/g, "\\'");
} }
function normalizeRepoUrl(raw: string) {
let url = (raw ?? "").trim();
if (!url) return "";
// Common typo: "https://https://github.com/..."
url = url.replace(/^https?:\/\/https?:\/\//i, "https://");
if (!/^https?:\/\//i.test(url)) url = `https://${url}`;
url = url.replace(/\/+$/, "");
return url;
}
function sanitizeFileStemFromPkgPath(pkgPath: string) { function sanitizeFileStemFromPkgPath(pkgPath: string) {
// Convert a package path into a stable filename. // Convert a package path into a stable filename.
// Example: "internal/go-oidc/example" -> "internal-go-oidc-example" // Example: "internal/go-oidc/example" -> "internal-go-oidc-example"
@@ -37,6 +50,133 @@ function sanitizeFileStemFromPkgPath(pkgPath: string) {
return joined.replace(/-+/g, "-").replace(/^-|-$/g, ""); return joined.replace(/-+/g, "-").replace(/^-|-$/g, "");
} }
function splitUrlAndFragment(url: string): {
urlNoFragment: string;
fragment: string;
} {
const i = url.indexOf("#");
if (i === -1) return { urlNoFragment: url, fragment: "" };
return { urlNoFragment: url.slice(0, i), fragment: url.slice(i) };
}
function isExternalOrAbsoluteUrl(url: string) {
// - absolute site links: "/foo"
// - pure fragments: "#bar"
// - external schemes: "https:", "mailto:", "vscode:", etc.
// IMPORTANT: don't treat "config.go:29" as a scheme.
if (url.startsWith("/") || url.startsWith("#")) return true;
if (url.includes("://")) return true;
return /^(https?|mailto|tel|vscode|file|data|ssh|git):/i.test(url);
}
function isRepoSourceFilePath(filePath: string) {
// Conservative allow-list: avoid rewriting .md (non-README) which may be VitePress docs.
return /\.(go|ts|tsx|js|jsx|py|sh|yml|yaml|json|toml|env|css|html|txt)$/i.test(
filePath
);
}
function parseFileLineSuffix(urlNoFragment: string): {
filePath: string;
line?: string;
} {
// Match "file.ext:123" (line suffix), while leaving "file.ext" untouched.
const m = urlNoFragment.match(/^(.*?):(\d+)$/);
if (!m) return { filePath: urlNoFragment };
return { filePath: m[1] ?? urlNoFragment, line: m[2] };
}
function rewriteMarkdownLinksOutsideFences(
md: string,
rewriteInline: (url: string) => string
) {
const lines = md.split("\n");
let inFence = false;
for (let i = 0; i < lines.length; i++) {
const line = lines[i] ?? "";
const trimmed = line.trimStart();
if (trimmed.startsWith("```")) {
inFence = !inFence;
continue;
}
if (inFence) continue;
// Inline markdown links/images: [text](url "title") / ![alt](url)
lines[i] = line.replace(
/\]\(([^)\s]+)(\s+"[^"]*")?\)/g,
(_full, urlRaw: string, maybeTitle: string | undefined) => {
const rewritten = rewriteInline(urlRaw);
return `](${rewritten}${maybeTitle ?? ""})`;
}
);
}
return lines.join("\n");
}
function rewriteImplMarkdown(params: {
md: string;
pkgPath: string;
readmeRelToDocRoute: Map<string, string>;
dirPathToDocRoute: Map<string, string>;
repoUrl: string;
}) {
const { md, pkgPath, readmeRelToDocRoute, dirPathToDocRoute, repoUrl } =
params;
return rewriteMarkdownLinksOutsideFences(md, (urlRaw) => {
// Handle angle-bracketed destinations: (<./foo/README.md>)
const angleWrapped =
urlRaw.startsWith("<") && urlRaw.endsWith(">")
? urlRaw.slice(1, -1)
: urlRaw;
const { urlNoFragment, fragment } = splitUrlAndFragment(angleWrapped);
if (!urlNoFragment) return urlRaw;
if (isExternalOrAbsoluteUrl(urlNoFragment)) return urlRaw;
// 1) Directory links like "common" or "common/" that have a README
const dirPathNormalized = urlNoFragment.replace(/\/+$/, "");
if (dirPathToDocRoute.has(dirPathNormalized)) {
const rewritten = `${dirPathToDocRoute.get(
dirPathNormalized
)!}${fragment}`;
return angleWrapped === urlRaw ? rewritten : `<${rewritten}>`;
}
// 2) Intra-repo README links -> VitePress impl routes
if (/(^|\/)README\.md$/.test(urlNoFragment)) {
const targetReadmeRel = path.posix.normalize(
path.posix.join(pkgPath, urlNoFragment)
);
const route = readmeRelToDocRoute.get(targetReadmeRel);
if (route) {
const rewritten = `${route}${fragment}`;
return angleWrapped === urlRaw ? rewritten : `<${rewritten}>`;
}
return urlRaw;
}
// 3) Local source-file references like "config.go:29" -> GitHub blob link
if (repoUrl) {
const { filePath, line } = parseFileLineSuffix(urlNoFragment);
if (isRepoSourceFilePath(filePath)) {
const repoRel = path.posix.normalize(
path.posix.join(pkgPath, filePath)
);
const githubUrl = `${repoUrl}/blob/main/${repoRel}${
line ? `#L${line}` : ""
}`;
const rewritten = `${githubUrl}${fragment}`;
return angleWrapped === urlRaw ? rewritten : `<${rewritten}>`;
}
}
return urlRaw;
});
}
async function listRepoReadmes(repoRootAbs: string): Promise<string[]> { async function listRepoReadmes(repoRootAbs: string): Promise<string[]> {
const glob = new Glob("**/README.md"); const glob = new Glob("**/README.md");
const readmes: string[] = []; const readmes: string[] = [];
@@ -51,8 +191,14 @@ async function listRepoReadmes(repoRootAbs: string): Promise<string[]> {
if (rel.startsWith(".git/") || rel.includes("/.git/")) continue; if (rel.startsWith(".git/") || rel.includes("/.git/")) continue;
if (rel.startsWith("node_modules/") || rel.includes("/node_modules/")) if (rel.startsWith("node_modules/") || rel.includes("/node_modules/"))
continue; continue;
if (rel.startsWith("internal/go-oidc/")) continue; let skip = false;
if (rel.startsWith("internal/gopsutil/")) continue; for (const submodule of skipSubmodules) {
if (rel.startsWith(submodule)) {
skip = true;
break;
}
}
if (skip) continue;
readmes.push(rel); readmes.push(rel);
} }
@@ -61,11 +207,34 @@ async function listRepoReadmes(repoRootAbs: string): Promise<string[]> {
return readmes; return readmes;
} }
async function ensureHardLink(srcAbs: string, dstAbs: string) { async function writeImplDocCopy(params: {
srcAbs: string;
dstAbs: string;
pkgPath: string;
readmeRelToDocRoute: Map<string, string>;
dirPathToDocRoute: Map<string, string>;
repoUrl: string;
}) {
const {
srcAbs,
dstAbs,
pkgPath,
readmeRelToDocRoute,
dirPathToDocRoute,
repoUrl,
} = params;
await mkdir(path.dirname(dstAbs), { recursive: true }); await mkdir(path.dirname(dstAbs), { recursive: true });
await rm(dstAbs, { force: true }); await rm(dstAbs, { force: true });
// Prefer sync for better error surfaces in Bun on some platforms.
linkSync(srcAbs, dstAbs); const original = await readFile(srcAbs, "utf8");
const rewritten = rewriteImplMarkdown({
md: original,
pkgPath,
readmeRelToDocRoute,
dirPathToDocRoute,
repoUrl,
});
await writeFile(dstAbs, rewritten);
} }
async function syncImplDocs( async function syncImplDocs(
@@ -78,6 +247,30 @@ async function syncImplDocs(
const readmes = await listRepoReadmes(repoRootAbs); const readmes = await listRepoReadmes(repoRootAbs);
const docs: ImplDoc[] = []; const docs: ImplDoc[] = [];
const expectedFileNames = new Set<string>(); const expectedFileNames = new Set<string>();
expectedFileNames.add("introduction.md");
const repoUrl = normalizeRepoUrl(
Bun.env.REPO_URL ?? "https://github.com/yusing/godoxy"
);
// Precompute mapping from repo-relative README path -> VitePress route.
// This lets us rewrite intra-repo README links when copying content.
const readmeRelToDocRoute = new Map<string, string>();
// Also precompute mapping from directory path -> VitePress route.
// This handles links like "[`common/`](common)" that point to directories with READMEs.
const dirPathToDocRoute = new Map<string, string>();
for (const readmeRel of readmes) {
const pkgPath = path.posix.dirname(readmeRel);
if (!pkgPath || pkgPath === ".") continue;
const docStem = sanitizeFileStemFromPkgPath(pkgPath);
if (!docStem) continue;
const route = `/impl/${docStem}`;
readmeRelToDocRoute.set(readmeRel, route);
dirPathToDocRoute.set(pkgPath, route);
}
for (const readmeRel of readmes) { for (const readmeRel of readmes) {
const pkgPath = path.posix.dirname(readmeRel); const pkgPath = path.posix.dirname(readmeRel);
@@ -86,13 +279,21 @@ async function syncImplDocs(
const docStem = sanitizeFileStemFromPkgPath(pkgPath); const docStem = sanitizeFileStemFromPkgPath(pkgPath);
if (!docStem) continue; if (!docStem) continue;
const docFileName = `${docStem}.md`; const docFileName = `${docStem}.md`;
const docRoute = `/impl/${docStem}`;
const srcPathAbs = path.join(repoRootAbs, readmeRel); const srcPathAbs = path.join(repoRootAbs, readmeRel);
const dstPathAbs = path.join(implDirAbs, docFileName); const dstPathAbs = path.join(implDirAbs, docFileName);
await ensureHardLink(srcPathAbs, dstPathAbs); await writeImplDocCopy({
srcAbs: srcPathAbs,
dstAbs: dstPathAbs,
pkgPath,
readmeRelToDocRoute,
dirPathToDocRoute,
repoUrl,
});
docs.push({ pkgPath, docFileName, srcPathAbs, dstPathAbs }); docs.push({ pkgPath, docFileName, docRoute, srcPathAbs, dstPathAbs });
expectedFileNames.add(docFileName); expectedFileNames.add(docFileName);
} }
@@ -111,13 +312,13 @@ async function syncImplDocs(
} }
function renderSidebarItems(docs: ImplDoc[], indent: string) { function renderSidebarItems(docs: ImplDoc[], indent: string) {
// link: '/impl/<file>.md' because VitePress `srcDir = "src"`. // link: '/impl/<stem>' (extensionless) because VitePress `srcDir = "src"`.
if (docs.length === 0) return ""; if (docs.length === 0) return "";
return ( return (
docs docs
.map((d) => { .map((d) => {
const text = escapeSingleQuotedTs(d.pkgPath); const text = escapeSingleQuotedTs(d.pkgPath);
const link = escapeSingleQuotedTs(`/impl/${d.docFileName}`); const link = escapeSingleQuotedTs(d.docRoute);
return `${indent}{ text: '${text}', link: '${link}' },`; return `${indent}{ text: '${text}', link: '${link}' },`;
}) })
.join("\n") + "\n" .join("\n") + "\n"