diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..61264b94 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,128 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the + overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or + advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email + address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +yusing@6uo.me. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series +of actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within +the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.0, available at +https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct +enforcement ladder](https://github.com/mozilla/diversity). + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at +https://www.contributor-covenant.org/translations. diff --git a/agent/go.mod b/agent/go.mod index 42c4e70c..8f2a2a3e 100644 --- a/agent/go.mod +++ b/agent/go.mod @@ -2,6 +2,11 @@ module github.com/yusing/godoxy/agent go 1.25.6 +exclude ( + github.com/moby/moby/api v1.53.0 // allow older daemon versions + github.com/moby/moby/client v0.2.2 // allow older daemon versions +) + replace ( github.com/shirou/gopsutil/v4 => ../internal/gopsutil github.com/yusing/godoxy => ../ @@ -22,7 +27,7 @@ require ( github.com/pion/transport/v3 v3.1.1 github.com/rs/zerolog v1.34.0 github.com/stretchr/testify v1.11.1 - github.com/yusing/godoxy v0.25.0 + github.com/yusing/godoxy v0.25.2 github.com/yusing/godoxy/socketproxy v0.0.0-00010101000000-000000000000 github.com/yusing/goutils v0.7.0 ) @@ -38,7 +43,7 @@ require ( github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/distribution/reference v0.6.0 // indirect - github.com/docker/cli v29.1.5+incompatible // indirect + github.com/docker/cli v29.2.0+incompatible // indirect github.com/docker/go-connections v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/ebitengine/purego v0.9.1 // indirect @@ -86,8 +91,8 @@ require ( github.com/valyala/fasthttp v1.69.0 // indirect github.com/yusing/ds v0.4.1 // indirect github.com/yusing/gointernals v0.1.16 // indirect - github.com/yusing/goutils/http/reverseproxy v0.0.0-20260125040745-bcc4b498f878 // indirect - github.com/yusing/goutils/http/websocket v0.0.0-20260125040745-bcc4b498f878 // indirect + github.com/yusing/goutils/http/reverseproxy v0.0.0-20260129081554-24e52ede7468 // indirect + github.com/yusing/goutils/http/websocket v0.0.0-20260129081554-24e52ede7468 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0 // indirect diff --git a/agent/go.sum b/agent/go.sum index edbd0f29..af98293f 100644 --- a/agent/go.sum +++ b/agent/go.sum @@ -37,8 +37,8 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c= github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0= -github.com/docker/cli v29.1.5+incompatible h1:GckbANUt3j+lsnQ6eCcQd70mNSOismSHWt8vk2AX8ao= -github.com/docker/cli v29.1.5+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v29.2.0+incompatible h1:9oBd9+YM7rxjZLfyMGxjraKBKE4/nVyvVfN4qNl9XRM= +github.com/docker/cli v29.2.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -82,8 +82,8 @@ github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PU github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= -github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -153,8 +153,8 @@ github.com/pion/transport/v3 v3.1.1 h1:Tr684+fnnKlhPceU+ICdrw6KKkTms+5qHMgw6bIkY github.com/pion/transport/v3 v3.1.1/go.mod h1:+c2eewC5WJQHiAA46fkMMzoYZSuGzA/7E2FPrOYHctQ= github.com/pion/transport/v4 v4.0.1 h1:sdROELU6BZ63Ab7FrOLn13M6YdJLY20wldXW2Cu2k8o= github.com/pion/transport/v4 v4.0.1/go.mod h1:nEuEA4AD5lPdcIegQDpVLgNoDGreqM/YqmEx3ovP4jM= -github.com/pires/go-proxyproto v0.9.1 h1:wTPjpyk41pJm1Im9BqHtPLuhxfjxL+qNfSikx9ux0WY= -github.com/pires/go-proxyproto v0.9.1/go.mod h1:ZKAAyp3cgy5Y5Mo4n9AlScrkCZwUy0g3Jf+slqQVcuU= +github.com/pires/go-proxyproto v0.9.2 h1:H1UdHn695zUVVmB0lQ354lOWHOy6TZSpzBl3tgN0s1U= +github.com/pires/go-proxyproto v0.9.2/go.mod h1:ZKAAyp3cgy5Y5Mo4n9AlScrkCZwUy0g3Jf+slqQVcuU= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= diff --git a/agent/pkg/agent/config.go b/agent/pkg/agent/config.go index d7128f39..404554e4 100644 --- a/agent/pkg/agent/config.go +++ b/agent/pkg/agent/config.go @@ -4,7 +4,6 @@ import ( "context" "crypto/tls" "crypto/x509" - "encoding/json" "encoding/pem" "errors" "fmt" @@ -16,6 +15,7 @@ import ( "strings" "time" + "github.com/bytedance/sonic" "github.com/rs/zerolog" "github.com/rs/zerolog/log" "github.com/yusing/godoxy/agent/pkg/agent/common" @@ -150,7 +150,7 @@ func (cfg *AgentConfig) InitWithCerts(ctx context.Context, ca, crt, key []byte) // test stream server connection const fakeAddress = "localhost:8080" // it won't be used, just for testing // test TCP stream support - err := agentstream.TCPHealthCheck(cfg.Addr, cfg.caCert, cfg.clientCert) + err := agentstream.TCPHealthCheck(ctx, cfg.Addr, cfg.caCert, cfg.clientCert) if err != nil { streamUnsupportedErrs.Addf("failed to connect to stream server via TCP: %w", err) } else { @@ -158,7 +158,7 @@ func (cfg *AgentConfig) InitWithCerts(ctx context.Context, ca, crt, key []byte) } // test UDP stream support - err = agentstream.UDPHealthCheck(cfg.Addr, cfg.caCert, cfg.clientCert) + err = agentstream.UDPHealthCheck(ctx, cfg.Addr, cfg.caCert, cfg.clientCert) if err != nil { streamUnsupportedErrs.Addf("failed to connect to stream server via UDP: %w", err) } else { @@ -313,8 +313,18 @@ func (cfg *AgentConfig) do(ctx context.Context, method, endpoint string, body io if err != nil { return nil, err } + + timeout := 5 * time.Second + if deadline, ok := ctx.Deadline(); ok { + remaining := time.Until(deadline) + if remaining > 0 { + timeout = remaining + } + } + client := http.Client{ Transport: cfg.Transport(), + Timeout: timeout, } return client.Do(req) } @@ -356,7 +366,7 @@ func (cfg *AgentConfig) fetchJSON(ctx context.Context, endpoint string, out any) return resp.StatusCode, nil } - err = json.Unmarshal(data, out) + err = sonic.Unmarshal(data, out) if err != nil { return 0, err } diff --git a/agent/pkg/agent/stream/tcp_client.go b/agent/pkg/agent/stream/tcp_client.go index 3a9db398..75dad5c3 100644 --- a/agent/pkg/agent/stream/tcp_client.go +++ b/agent/pkg/agent/stream/tcp_client.go @@ -1,6 +1,7 @@ package stream import ( + "context" "crypto/tls" "crypto/x509" "net" @@ -34,13 +35,13 @@ func NewTCPClient(serverAddr, targetAddress string, caCert *x509.Certificate, cl return nil, err } - return newTCPClientWIthHeader(serverAddr, header, caCert, clientCert) + return newTCPClientWIthHeader(context.Background(), serverAddr, header, caCert, clientCert) } -func TCPHealthCheck(serverAddr string, caCert *x509.Certificate, clientCert *tls.Certificate) error { +func TCPHealthCheck(ctx context.Context, serverAddr string, caCert *x509.Certificate, clientCert *tls.Certificate) error { header := NewStreamHealthCheckHeader() - conn, err := newTCPClientWIthHeader(serverAddr, header, caCert, clientCert) + conn, err := newTCPClientWIthHeader(ctx, serverAddr, header, caCert, clientCert) if err != nil { return err } @@ -49,7 +50,7 @@ func TCPHealthCheck(serverAddr string, caCert *x509.Certificate, clientCert *tls return nil } -func newTCPClientWIthHeader(serverAddr string, header *StreamRequestHeader, caCert *x509.Certificate, clientCert *tls.Certificate) (net.Conn, error) { +func newTCPClientWIthHeader(ctx context.Context, serverAddr string, header *StreamRequestHeader, caCert *x509.Certificate, clientCert *tls.Certificate) (net.Conn, error) { // Setup TLS configuration caCertPool := x509.NewCertPool() caCertPool.AddCert(caCert) @@ -62,17 +63,43 @@ func newTCPClientWIthHeader(serverAddr string, header *StreamRequestHeader, caCe ServerName: common.CertsDNSName, } + dialer := &net.Dialer{ + Timeout: dialTimeout, + } + tlsDialer := &tls.Dialer{ + NetDialer: dialer, + Config: tlsConfig, + } + // Establish TLS connection - conn, err := tls.DialWithDialer(&net.Dialer{Timeout: dialTimeout}, "tcp", serverAddr, tlsConfig) + conn, err := tlsDialer.DialContext(ctx, "tcp", serverAddr) if err != nil { return nil, err } + + deadline, hasDeadline := ctx.Deadline() + if hasDeadline { + err := conn.SetWriteDeadline(deadline) + if err != nil { + _ = conn.Close() + return nil, err + } + } // Send the stream header once as a handshake. if _, err := conn.Write(header.Bytes()); err != nil { _ = conn.Close() return nil, err } + if hasDeadline { + // reset write deadline + err = conn.SetWriteDeadline(time.Time{}) + if err != nil { + _ = conn.Close() + return nil, err + } + } + return &TCPClient{ conn: conn, }, nil diff --git a/agent/pkg/agent/stream/tests/healthcheck_test.go b/agent/pkg/agent/stream/tests/healthcheck_test.go index 320e29a6..282d4caf 100644 --- a/agent/pkg/agent/stream/tests/healthcheck_test.go +++ b/agent/pkg/agent/stream/tests/healthcheck_test.go @@ -12,7 +12,7 @@ func TestTCPHealthCheck(t *testing.T) { srv := startTCPServer(t, certs) - err := stream.TCPHealthCheck(srv.Addr.String(), certs.CaCert, certs.ClientCert) + err := stream.TCPHealthCheck(t.Context(), srv.Addr.String(), certs.CaCert, certs.ClientCert) require.NoError(t, err, "health check") } @@ -21,6 +21,6 @@ func TestUDPHealthCheck(t *testing.T) { srv := startUDPServer(t, certs) - err := stream.UDPHealthCheck(srv.Addr.String(), certs.CaCert, certs.ClientCert) + err := stream.UDPHealthCheck(t.Context(), srv.Addr.String(), certs.CaCert, certs.ClientCert) require.NoError(t, err, "health check") } diff --git a/agent/pkg/agent/stream/udp_client.go b/agent/pkg/agent/stream/udp_client.go index 4d372be8..24941991 100644 --- a/agent/pkg/agent/stream/udp_client.go +++ b/agent/pkg/agent/stream/udp_client.go @@ -1,6 +1,7 @@ package stream import ( + "context" "crypto/tls" "crypto/x509" "net" @@ -35,10 +36,10 @@ func NewUDPClient(serverAddr, targetAddress string, caCert *x509.Certificate, cl return nil, err } - return newUDPClientWIthHeader(serverAddr, header, caCert, clientCert) + return newUDPClientWIthHeader(context.Background(), serverAddr, header, caCert, clientCert) } -func newUDPClientWIthHeader(serverAddr string, header *StreamRequestHeader, caCert *x509.Certificate, clientCert *tls.Certificate) (net.Conn, error) { +func newUDPClientWIthHeader(ctx context.Context, serverAddr string, header *StreamRequestHeader, caCert *x509.Certificate, clientCert *tls.Certificate) (net.Conn, error) { // Setup DTLS configuration caCertPool := x509.NewCertPool() caCertPool.AddCert(caCert) @@ -62,21 +63,40 @@ func newUDPClientWIthHeader(serverAddr string, header *StreamRequestHeader, caCe if err != nil { return nil, err } + + deadline, hasDeadline := ctx.Deadline() + if hasDeadline { + err := conn.SetWriteDeadline(deadline) + if err != nil { + _ = conn.Close() + return nil, err + } + } + // Send the stream header once as a handshake. if _, err := conn.Write(header.Bytes()); err != nil { _ = conn.Close() return nil, err } + if hasDeadline { + // reset write deadline + err = conn.SetWriteDeadline(time.Time{}) + if err != nil { + _ = conn.Close() + return nil, err + } + } + return &UDPClient{ conn: conn, }, nil } -func UDPHealthCheck(serverAddr string, caCert *x509.Certificate, clientCert *tls.Certificate) error { +func UDPHealthCheck(ctx context.Context, serverAddr string, caCert *x509.Certificate, clientCert *tls.Certificate) error { header := NewStreamHealthCheckHeader() - conn, err := newUDPClientWIthHeader(serverAddr, header, caCert, clientCert) + conn, err := newUDPClientWIthHeader(ctx, serverAddr, header, caCert, clientCert) if err != nil { return err } diff --git a/go.mod b/go.mod index 4bfc390c..333256c4 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,11 @@ module github.com/yusing/godoxy go 1.25.6 +exclude ( + github.com/moby/moby/api v1.53.0 // allow older daemon versions + github.com/moby/moby/client v0.2.2 // allow older daemon versions +) + replace ( github.com/coreos/go-oidc/v3 => ./internal/go-oidc github.com/luthermonson/go-proxmox => ./internal/go-proxmox @@ -25,7 +30,7 @@ require ( github.com/gorilla/websocket v1.5.3 // websocket for API and agent github.com/gotify/server/v2 v2.8.0 // reference the Message struct for json response github.com/lithammer/fuzzysearch v1.1.8 // fuzzy search for searching icons and filtering metrics - github.com/pires/go-proxyproto v0.9.1 // proxy protocol support + github.com/pires/go-proxyproto v0.9.2 // proxy protocol support github.com/puzpuzpuz/xsync/v4 v4.4.0 // lock free map for concurrent operations github.com/rs/zerolog v1.34.0 // logging github.com/vincent-petithory/dataurl v1.0.0 // data url for fav icon @@ -39,9 +44,9 @@ require ( require ( github.com/bytedance/gopkg v0.1.3 // xxhash64 for fast hash github.com/bytedance/sonic v1.15.0 // fast json parsing - github.com/docker/cli v29.1.5+incompatible // needs docker/cli/cli/connhelper connection helper for docker client + github.com/docker/cli v29.2.0+incompatible // needs docker/cli/cli/connhelper connection helper for docker client github.com/goccy/go-yaml v1.19.2 // yaml parsing for different config files - github.com/golang-jwt/jwt/v5 v5.3.0 // jwt authentication + github.com/golang-jwt/jwt/v5 v5.3.1 // jwt authentication github.com/luthermonson/go-proxmox v0.3.2 // proxmox API client github.com/moby/moby/api v1.52.0 // docker API github.com/moby/moby/client v0.2.1 // docker client @@ -52,13 +57,13 @@ require ( github.com/stretchr/testify v1.11.1 // testing framework github.com/valyala/fasthttp v1.69.0 // fast http for health check github.com/yusing/ds v0.4.1 // data structures and algorithms - github.com/yusing/godoxy/agent v0.0.0-20260125091326-9c2051840fd9 - github.com/yusing/godoxy/internal/dnsproviders v0.0.0-20260124133347-9a96f3cc539e + github.com/yusing/godoxy/agent v0.0.0-20260129101716-0f13004ad6ba + github.com/yusing/godoxy/internal/dnsproviders v0.0.0-20260129101716-0f13004ad6ba github.com/yusing/gointernals v0.1.16 github.com/yusing/goutils v0.7.0 - github.com/yusing/goutils/http/reverseproxy v0.0.0-20260125040745-bcc4b498f878 - github.com/yusing/goutils/http/websocket v0.0.0-20260125040745-bcc4b498f878 - github.com/yusing/goutils/server v0.0.0-20260125040745-bcc4b498f878 + github.com/yusing/goutils/http/reverseproxy v0.0.0-20260129081554-24e52ede7468 + github.com/yusing/goutils/http/websocket v0.0.0-20260129081554-24e52ede7468 + github.com/yusing/goutils/server v0.0.0-20260129081554-24e52ede7468 ) require ( @@ -136,8 +141,8 @@ require ( golang.org/x/sys v0.40.0 // indirect golang.org/x/text v0.33.0 // indirect golang.org/x/tools v0.41.0 // indirect - google.golang.org/api v0.262.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260122232226-8e98ce8d340d // indirect + google.golang.org/api v0.263.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 // indirect google.golang.org/grpc v1.78.0 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/ini.v1 v1.67.1 // indirect @@ -170,8 +175,8 @@ require ( github.com/linode/linodego v1.64.0 // indirect github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 // indirect github.com/nrdcg/goinwx v0.12.0 // indirect - github.com/nrdcg/oci-go-sdk/common/v1065 v1065.106.1 // indirect - github.com/nrdcg/oci-go-sdk/dns/v1065 v1065.106.1 // indirect + github.com/nrdcg/oci-go-sdk/common/v1065 v1065.107.0 // indirect + github.com/nrdcg/oci-go-sdk/dns/v1065 v1065.107.0 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pion/dtls/v3 v3.0.10 // indirect github.com/pion/logging v0.2.4 // indirect diff --git a/go.sum b/go.sum index 4a12a0d0..9e251e67 100644 --- a/go.sum +++ b/go.sum @@ -76,8 +76,8 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c= github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0= -github.com/docker/cli v29.1.5+incompatible h1:GckbANUt3j+lsnQ6eCcQd70mNSOismSHWt8vk2AX8ao= -github.com/docker/cli v29.1.5+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v29.2.0+incompatible h1:9oBd9+YM7rxjZLfyMGxjraKBKE4/nVyvVfN4qNl9XRM= +github.com/docker/cli v29.2.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -137,8 +137,8 @@ github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7Lk github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw= github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0= -github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= -github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -227,10 +227,10 @@ github.com/nrdcg/goacmedns v0.2.0 h1:ADMbThobzEMnr6kg2ohs4KGa3LFqmgiBA22/6jUWJR0 github.com/nrdcg/goacmedns v0.2.0/go.mod h1:T5o6+xvSLrQpugmwHvrSNkzWht0UGAwj2ACBMhh73Cg= github.com/nrdcg/goinwx v0.12.0 h1:ujdUqDBnaRSFwzVnImvPHYw3w3m9XgmGImNUw1GyMb4= github.com/nrdcg/goinwx v0.12.0/go.mod h1:IrVKd3ZDbFiMjdPgML4CSxZAY9wOoqLvH44zv3NodJ0= -github.com/nrdcg/oci-go-sdk/common/v1065 v1065.106.1 h1:+fx2mbWeR8XX/vidwpRMepJMtRIYQP44Iezm2oeObVM= -github.com/nrdcg/oci-go-sdk/common/v1065 v1065.106.1/go.mod h1:Gcs8GCaZXL3FdiDWgdnMxlOLEdRprJJnPYB22TX1jw8= -github.com/nrdcg/oci-go-sdk/dns/v1065 v1065.106.1 h1:GDhBiaIAm/QXLzHJ0ASDdY/6R/9w60+gk8lY5rgfxEQ= -github.com/nrdcg/oci-go-sdk/dns/v1065 v1065.106.1/go.mod h1:EHScJdbM0gg5Is7e3C0ceRYAFMMsfP4Vf8sBRoxoTgk= +github.com/nrdcg/oci-go-sdk/common/v1065 v1065.107.0 h1:eMzyN+jGJbxG4ut278uwIsUo9XacXc711lFjhKnaUso= +github.com/nrdcg/oci-go-sdk/common/v1065 v1065.107.0/go.mod h1:Gcs8GCaZXL3FdiDWgdnMxlOLEdRprJJnPYB22TX1jw8= +github.com/nrdcg/oci-go-sdk/dns/v1065 v1065.107.0 h1:t34IpOa+8NfmjkU8bdWtYrLrmr346/FGhu8FlpJDQok= +github.com/nrdcg/oci-go-sdk/dns/v1065 v1065.107.0/go.mod h1:p95/OxVsdx71I2Qrck1GtIS87sRxcTRKXzUi5nWm9NY= github.com/nrdcg/porkbun v0.4.0 h1:rWweKlwo1PToQ3H+tEO9gPRW0wzzgmI/Ob3n2Guticw= github.com/nrdcg/porkbun v0.4.0/go.mod h1:/QMskrHEIM0IhC/wY7iTCUgINsxdT2WcOphktJ9+Q54= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= @@ -251,8 +251,8 @@ github.com/pion/logging v0.2.4 h1:tTew+7cmQ+Mc1pTBLKH2puKsOvhm32dROumOZ655zB8= github.com/pion/logging v0.2.4/go.mod h1:DffhXTKYdNZU+KtJ5pyQDjvOAh/GsNSyv1lbkFbe3so= github.com/pion/transport/v4 v4.0.1 h1:sdROELU6BZ63Ab7FrOLn13M6YdJLY20wldXW2Cu2k8o= github.com/pion/transport/v4 v4.0.1/go.mod h1:nEuEA4AD5lPdcIegQDpVLgNoDGreqM/YqmEx3ovP4jM= -github.com/pires/go-proxyproto v0.9.1 h1:wTPjpyk41pJm1Im9BqHtPLuhxfjxL+qNfSikx9ux0WY= -github.com/pires/go-proxyproto v0.9.1/go.mod h1:ZKAAyp3cgy5Y5Mo4n9AlScrkCZwUy0g3Jf+slqQVcuU= +github.com/pires/go-proxyproto v0.9.2 h1:H1UdHn695zUVVmB0lQ354lOWHOy6TZSpzBl3tgN0s1U= +github.com/pires/go-proxyproto v0.9.2/go.mod h1:ZKAAyp3cgy5Y5Mo4n9AlScrkCZwUy0g3Jf+slqQVcuU= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -447,14 +447,14 @@ golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/api v0.262.0 h1:4B+3u8He2GwyN8St3Jhnd3XRHlIvc//sBmgHSp78oNY= -google.golang.org/api v0.262.0/go.mod h1:jNwmH8BgUBJ/VrUG6/lIl9YiildyLd09r9ZLHiQ6cGI= +google.golang.org/api v0.263.0 h1:UFs7qn8gInIdtk1ZA6eXRXp5JDAnS4x9VRsRVCeKdbk= +google.golang.org/api v0.263.0/go.mod h1:fAU1xtNNisHgOF5JooAs8rRaTkl2rT3uaoNGo9NS3R8= google.golang.org/genproto v0.0.0-20251202230838-ff82c1b0f217 h1:GvESR9BIyHUahIb0NcTum6itIWtdoglGX+rnGxm2934= google.golang.org/genproto v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:yJ2HH4EHEDTd3JiLmhds6NkJ17ITVYOdV3m3VKOnws0= google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 h1:fCvbg86sFXwdrl5LgVcTEvNC+2txB5mgROGmRL5mrls= google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260122232226-8e98ce8d340d h1:xXzuihhT3gL/ntduUZwHECzAn57E8dA6l8SOtYWdD8Q= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260122232226-8e98ce8d340d/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 h1:H86B94AW+VfJWDqFeEbBPhEtHzJwJfTbgE2lZa54ZAQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= diff --git a/goutils b/goutils index 272bc534..52ea531e 160000 --- a/goutils +++ b/goutils @@ -1 +1 @@ -Subproject commit 272bc5343946eb60f7283e8e7a7ff65644b11eca +Subproject commit 52ea531e95bef1b21c4c832f36facb3e509d1191 diff --git a/internal/agentpool/agent.go b/internal/agentpool/agent.go index 59fe1e77..b85aeb0d 100644 --- a/internal/agentpool/agent.go +++ b/internal/agentpool/agent.go @@ -27,6 +27,7 @@ func newAgent(cfg *agent.AgentConfig) *Agent { AgentConfig: cfg, httpClient: &http.Client{ Transport: transport, + Timeout: 5 * time.Second, }, fasthttpHcClient: &fasthttp.Client{ DialTimeout: func(addr string, timeout time.Duration) (net.Conn, error) { diff --git a/internal/api/handler.go b/internal/api/handler.go index b1eb1b00..5878db6b 100644 --- a/internal/api/handler.go +++ b/internal/api/handler.go @@ -86,6 +86,8 @@ func NewHandler(requireAuth bool) *gin.Engine { route.GET("/providers", routeApi.Providers) route.GET("/by_provider", routeApi.ByProvider) route.POST("/playground", routeApi.Playground) + route.GET("/validate", routeApi.Validate) // websocket + route.POST("/validate", routeApi.Validate) } file := v1.Group("/file") diff --git a/internal/api/v1/docs/swagger.json b/internal/api/v1/docs/swagger.json index ff08028c..dd6544c7 100644 --- a/internal/api/v1/docs/swagger.json +++ b/internal/api/v1/docs/swagger.json @@ -1087,7 +1087,7 @@ "post": { "description": "Validate file", "consumes": [ - "text/plain" + "application/yaml" ], "produces": [ "application/json" @@ -3026,6 +3026,122 @@ "operationId": "providers" } }, + "/route/validate": { + "get": { + "description": "Validate route,", + "consumes": [ + "application/yaml" + ], + "produces": [ + "application/json" + ], + "tags": [ + "route", + "websocket" + ], + "summary": "Validate route", + "parameters": [ + { + "description": "Route", + "name": "route", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/Route" + } + } + ], + "responses": { + "200": { + "description": "Route validated", + "schema": { + "$ref": "#/definitions/SuccessResponse" + } + }, + "400": { + "description": "Bad request", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + }, + "417": { + "description": "Validation failed", + "schema": {} + }, + "500": { + "description": "Internal server error", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + }, + "x-id": "validate", + "operationId": "validate" + }, + "post": { + "description": "Validate route,", + "consumes": [ + "application/yaml" + ], + "produces": [ + "application/json" + ], + "tags": [ + "route", + "websocket" + ], + "summary": "Validate route", + "parameters": [ + { + "description": "Route", + "name": "route", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/Route" + } + } + ], + "responses": { + "200": { + "description": "Route validated", + "schema": { + "$ref": "#/definitions/SuccessResponse" + } + }, + "400": { + "description": "Bad request", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + }, + "417": { + "description": "Validation failed", + "schema": {} + }, + "500": { + "description": "Internal server error", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + }, + "x-id": "validate", + "operationId": "validate" + } + }, "/route/{which}": { "get": { "description": "List route", @@ -6745,229 +6861,6 @@ "x-nullable": false, "x-omitempty": false }, - "route.Route": { - "type": "object", - "properties": { - "access_log": { - "allOf": [ - { - "$ref": "#/definitions/RequestLoggerConfig" - } - ], - "x-nullable": true - }, - "agent": { - "type": "string", - "x-nullable": false, - "x-omitempty": false - }, - "alias": { - "type": "string", - "x-nullable": false, - "x-omitempty": false - }, - "bind": { - "description": "for TCP and UDP routes, bind address to listen on", - "type": "string", - "x-nullable": true - }, - "container": { - "description": "Docker only", - "allOf": [ - { - "$ref": "#/definitions/Container" - } - ], - "x-nullable": true - }, - "disable_compression": { - "type": "boolean", - "x-nullable": false, - "x-omitempty": false - }, - "excluded": { - "type": "boolean", - "x-nullable": true - }, - "excluded_reason": { - "type": "string", - "x-nullable": true - }, - "health": { - "description": "for swagger", - "allOf": [ - { - "$ref": "#/definitions/HealthJSON" - } - ], - "x-nullable": false, - "x-omitempty": false - }, - "healthcheck": { - "description": "null on load-balancer routes", - "allOf": [ - { - "$ref": "#/definitions/HealthCheckConfig" - } - ], - "x-nullable": true - }, - "homepage": { - "$ref": "#/definitions/HomepageItemConfig", - "x-nullable": false, - "x-omitempty": false - }, - "host": { - "type": "string", - "x-nullable": false, - "x-omitempty": false - }, - "idlewatcher": { - "allOf": [ - { - "$ref": "#/definitions/IdlewatcherConfig" - } - ], - "x-nullable": true - }, - "index": { - "description": "Index file to serve for single-page app mode", - "type": "string", - "x-nullable": false, - "x-omitempty": false - }, - "load_balance": { - "allOf": [ - { - "$ref": "#/definitions/LoadBalancerConfig" - } - ], - "x-nullable": true - }, - "lurl": { - "description": "private fields", - "type": "string", - "x-nullable": true - }, - "middlewares": { - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/types.LabelMap" - }, - "x-nullable": true - }, - "no_tls_verify": { - "type": "boolean", - "x-nullable": false, - "x-omitempty": false - }, - "path_patterns": { - "type": "array", - "items": { - "type": "string" - }, - "x-nullable": true - }, - "port": { - "$ref": "#/definitions/Port", - "x-nullable": false, - "x-omitempty": false - }, - "provider": { - "description": "for backward compatibility", - "type": "string", - "x-nullable": true - }, - "proxmox": { - "allOf": [ - { - "$ref": "#/definitions/ProxmoxNodeConfig" - } - ], - "x-nullable": true - }, - "purl": { - "type": "string", - "x-nullable": false, - "x-omitempty": false - }, - "response_header_timeout": { - "type": "integer", - "x-nullable": false, - "x-omitempty": false - }, - "root": { - "type": "string", - "x-nullable": false, - "x-omitempty": false - }, - "rule_file": { - "type": "string", - "x-nullable": true - }, - "rules": { - "type": "array", - "items": { - "$ref": "#/definitions/rules.Rule" - }, - "x-nullable": true - }, - "scheme": { - "type": "string", - "enum": [ - "http", - "https", - "h2c", - "tcp", - "udp", - "fileserver" - ], - "x-nullable": false, - "x-omitempty": false - }, - "spa": { - "description": "Single-page app mode: serves index for non-existent paths", - "type": "boolean", - "x-nullable": false, - "x-omitempty": false - }, - "ssl_certificate": { - "description": "Path to client certificate", - "type": "string", - "x-nullable": false, - "x-omitempty": false - }, - "ssl_certificate_key": { - "description": "Path to client certificate key", - "type": "string", - "x-nullable": false, - "x-omitempty": false - }, - "ssl_protocols": { - "description": "Allowed TLS protocols", - "type": "array", - "items": { - "type": "string" - }, - "x-nullable": false, - "x-omitempty": false - }, - "ssl_server_name": { - "description": "SSL/TLS proxy options (nginx-like)", - "type": "string", - "x-nullable": false, - "x-omitempty": false - }, - "ssl_trusted_certificate": { - "description": "Path to trusted CA certificates", - "type": "string", - "x-nullable": false, - "x-omitempty": false - } - }, - "x-nullable": false, - "x-omitempty": false - }, "routeApi.RawRule": { "type": "object", "properties": { @@ -6995,7 +6888,7 @@ "additionalProperties": { "type": "array", "items": { - "$ref": "#/definitions/route.Route" + "$ref": "#/definitions/Route" } }, "x-nullable": false, diff --git a/internal/api/v1/docs/swagger.yaml b/internal/api/v1/docs/swagger.yaml index 5492e1f7..0bacaada 100644 --- a/internal/api/v1/docs/swagger.yaml +++ b/internal/api/v1/docs/swagger.yaml @@ -1807,12 +1807,12 @@ definitions: type: string kernel_version: type: string + load_avg_15m: + type: string load_avg_1m: type: string load_avg_5m: type: string - load_avg_15m: - type: string mem_pct: type: string mem_total: @@ -1830,127 +1830,6 @@ definitions: uptime: type: string type: object - route.Route: - properties: - access_log: - allOf: - - $ref: '#/definitions/RequestLoggerConfig' - x-nullable: true - agent: - type: string - alias: - type: string - bind: - description: for TCP and UDP routes, bind address to listen on - type: string - x-nullable: true - container: - allOf: - - $ref: '#/definitions/Container' - description: Docker only - x-nullable: true - disable_compression: - type: boolean - excluded: - type: boolean - x-nullable: true - excluded_reason: - type: string - x-nullable: true - health: - allOf: - - $ref: '#/definitions/HealthJSON' - description: for swagger - healthcheck: - allOf: - - $ref: '#/definitions/HealthCheckConfig' - description: null on load-balancer routes - x-nullable: true - homepage: - $ref: '#/definitions/HomepageItemConfig' - host: - type: string - idlewatcher: - allOf: - - $ref: '#/definitions/IdlewatcherConfig' - x-nullable: true - index: - description: Index file to serve for single-page app mode - type: string - load_balance: - allOf: - - $ref: '#/definitions/LoadBalancerConfig' - x-nullable: true - lurl: - description: private fields - type: string - x-nullable: true - middlewares: - additionalProperties: - $ref: '#/definitions/types.LabelMap' - type: object - x-nullable: true - no_tls_verify: - type: boolean - path_patterns: - items: - type: string - type: array - x-nullable: true - port: - $ref: '#/definitions/Port' - provider: - description: for backward compatibility - type: string - x-nullable: true - proxmox: - allOf: - - $ref: '#/definitions/ProxmoxNodeConfig' - x-nullable: true - purl: - type: string - response_header_timeout: - type: integer - root: - type: string - rule_file: - type: string - x-nullable: true - rules: - items: - $ref: '#/definitions/rules.Rule' - type: array - x-nullable: true - scheme: - enum: - - http - - https - - h2c - - tcp - - udp - - fileserver - type: string - spa: - description: 'Single-page app mode: serves index for non-existent paths' - type: boolean - ssl_certificate: - description: Path to client certificate - type: string - ssl_certificate_key: - description: Path to client certificate key - type: string - ssl_protocols: - description: Allowed TLS protocols - items: - type: string - type: array - ssl_server_name: - description: SSL/TLS proxy options (nginx-like) - type: string - ssl_trusted_certificate: - description: Path to trusted CA certificates - type: string - type: object routeApi.RawRule: properties: do: @@ -1963,7 +1842,7 @@ definitions: routeApi.RoutesByProvider: additionalProperties: items: - $ref: '#/definitions/route.Route' + $ref: '#/definitions/Route' type: array type: object rules.Rule: @@ -2741,7 +2620,7 @@ paths: /file/validate: post: consumes: - - text/plain + - application/yaml description: Validate file parameters: - description: Type @@ -4079,6 +3958,83 @@ paths: - route - websocket x-id: providers + /route/validate: + get: + consumes: + - application/yaml + description: Validate route, + parameters: + - description: Route + in: body + name: route + required: true + schema: + $ref: '#/definitions/Route' + produces: + - application/json + responses: + "200": + description: Route validated + schema: + $ref: '#/definitions/SuccessResponse' + "400": + description: Bad request + schema: + $ref: '#/definitions/ErrorResponse' + "403": + description: Forbidden + schema: + $ref: '#/definitions/ErrorResponse' + "417": + description: Validation failed + schema: {} + "500": + description: Internal server error + schema: + $ref: '#/definitions/ErrorResponse' + summary: Validate route + tags: + - route + - websocket + x-id: validate + post: + consumes: + - application/yaml + description: Validate route, + parameters: + - description: Route + in: body + name: route + required: true + schema: + $ref: '#/definitions/Route' + produces: + - application/json + responses: + "200": + description: Route validated + schema: + $ref: '#/definitions/SuccessResponse' + "400": + description: Bad request + schema: + $ref: '#/definitions/ErrorResponse' + "403": + description: Forbidden + schema: + $ref: '#/definitions/ErrorResponse' + "417": + description: Validation failed + schema: {} + "500": + description: Internal server error + schema: + $ref: '#/definitions/ErrorResponse' + summary: Validate route + tags: + - route + - websocket + x-id: validate /stats: get: consumes: diff --git a/internal/api/v1/file/validate.go b/internal/api/v1/file/validate.go index 4b0e7d28..e2fa07dc 100644 --- a/internal/api/v1/file/validate.go +++ b/internal/api/v1/file/validate.go @@ -20,7 +20,7 @@ type ValidateFileRequest struct { // @Summary Validate file // @Description Validate file // @Tags file -// @Accept text/plain +// @Accept application/yaml // @Produce json // @Param type query FileType true "Type" // @Param file body string true "File content" @@ -29,7 +29,7 @@ type ValidateFileRequest struct { // @Failure 403 {object} apitypes.ErrorResponse "Forbidden" // @Failure 417 {object} any "Validation failed" // @Failure 500 {object} apitypes.ErrorResponse "Internal server error" -// @Router /file/validate [post] +// @Router /file/validate [post] func Validate(c *gin.Context) { var request ValidateFileRequest if err := c.ShouldBindQuery(&request); err != nil { diff --git a/internal/api/v1/route/validate.go b/internal/api/v1/route/validate.go new file mode 100644 index 00000000..ddda19e0 --- /dev/null +++ b/internal/api/v1/route/validate.go @@ -0,0 +1,69 @@ +package routeApi + +import ( + "net/http" + "time" + + "github.com/gin-gonic/gin" + "github.com/goccy/go-yaml" + "github.com/yusing/godoxy/internal/route" + "github.com/yusing/godoxy/internal/serialization" + apitypes "github.com/yusing/goutils/apitypes" + "github.com/yusing/goutils/http/httpheaders" + "github.com/yusing/goutils/http/websocket" +) + +type _ = route.Route + +// @x-id "validate" +// @BasePath /api/v1 +// @Summary Validate route +// @Description Validate route, +// @Tags route,websocket +// @Accept application/yaml +// @Produce json +// @Param route body route.Route true "Route" +// @Success 200 {object} apitypes.SuccessResponse "Route validated" +// @Failure 400 {object} apitypes.ErrorResponse "Bad request" +// @Failure 403 {object} apitypes.ErrorResponse "Forbidden" +// @Failure 417 {object} any "Validation failed" +// @Failure 500 {object} apitypes.ErrorResponse "Internal server error" +// @Router /route/validate [get] +// @Router /route/validate [post] +func Validate(c *gin.Context) { + if httpheaders.IsWebsocket(c.Request.Header) { + ValidateWS(c) + return + } + var request route.Route + if err := c.ShouldBindWith(&request, serialization.GinYAMLBinding{}); err != nil { + c.JSON(http.StatusExpectationFailed, err) + return + } + c.JSON(http.StatusOK, apitypes.Success("route validated")) +} + +func ValidateWS(c *gin.Context) { + manager, err := websocket.NewManagerWithUpgrade(c) + if err != nil { + c.Error(apitypes.InternalServerError(err, "failed to upgrade to websocket")) + return + } + defer manager.Close() + + const writeTimeout = 5 * time.Second + + for { + select { + case <-manager.Done(): + return + case msg := <-manager.ReadCh(): + var request route.Route + if err := serialization.UnmarshalValidate(msg, &request, yaml.Unmarshal); err != nil { + manager.WriteJSON(gin.H{"error": err}, writeTimeout) + continue + } + manager.WriteJSON(gin.H{"message": "route validated"}, writeTimeout) + } + } +} diff --git a/internal/autocert/config_test.go b/internal/autocert/config_test.go index 6bb53de1..21054633 100644 --- a/internal/autocert/config_test.go +++ b/internal/autocert/config_test.go @@ -4,6 +4,7 @@ import ( "fmt" "testing" + "github.com/goccy/go-yaml" "github.com/stretchr/testify/require" "github.com/yusing/godoxy/internal/autocert" "github.com/yusing/godoxy/internal/dnsproviders" @@ -25,9 +26,9 @@ func TestEABConfigRequired(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - yaml := fmt.Appendf(nil, "eab_kid: %s\neab_hmac: %s", test.cfg.EABKid, test.cfg.EABHmac) + yamlCfg := fmt.Appendf(nil, "eab_kid: %s\neab_hmac: %s", test.cfg.EABKid, test.cfg.EABHmac) cfg := autocert.Config{} - err := serialization.UnmarshalValidateYAML(yaml, &cfg) + err := serialization.UnmarshalValidate(yamlCfg, &cfg, yaml.Unmarshal) if (err != nil) != test.wantErr { t.Errorf("Validate() error = %v, wantErr %v", err, test.wantErr) } diff --git a/internal/autocert/provider_test/multi_cert_test.go b/internal/autocert/provider_test/multi_cert_test.go index d77afe1f..f8ee18c1 100644 --- a/internal/autocert/provider_test/multi_cert_test.go +++ b/internal/autocert/provider_test/multi_cert_test.go @@ -6,6 +6,7 @@ import ( "os" "testing" + "github.com/goccy/go-yaml" "github.com/stretchr/testify/require" "github.com/yusing/godoxy/internal/autocert" "github.com/yusing/godoxy/internal/serialization" @@ -41,7 +42,7 @@ func TestMultipleCertificatesLifecycle(t *testing.T) { cfg.HTTPClient = acmeServer.httpClient() /* unmarshal yaml config with multiple certs */ - err := error(serialization.UnmarshalValidateYAML(yamlConfig, &cfg)) + err := error(serialization.UnmarshalValidate(yamlConfig, &cfg, yaml.Unmarshal)) require.NoError(t, err) require.Equal(t, []string{"main.example.com"}, cfg.Domains) require.Len(t, cfg.Extra, 2) diff --git a/internal/autocert/setup_test.go b/internal/autocert/setup_test.go index 39cb12fb..a124d56a 100644 --- a/internal/autocert/setup_test.go +++ b/internal/autocert/setup_test.go @@ -3,6 +3,7 @@ package autocert_test import ( "testing" + "github.com/goccy/go-yaml" "github.com/stretchr/testify/require" "github.com/yusing/godoxy/internal/autocert" "github.com/yusing/godoxy/internal/dnsproviders" @@ -42,7 +43,7 @@ extra: ` var cfg autocert.Config - err := error(serialization.UnmarshalValidateYAML([]byte(cfgYAML), &cfg)) + err := error(serialization.UnmarshalValidate([]byte(cfgYAML), &cfg, yaml.Unmarshal)) require.NoError(t, err) // Test: extra[0] inherits all fields from main except CertPath and KeyPath. diff --git a/internal/config/state.go b/internal/config/state.go index 57202905..9d5fb259 100644 --- a/internal/config/state.go +++ b/internal/config/state.go @@ -103,7 +103,7 @@ func (state *state) InitFromFile(filename string) error { } func (state *state) Init(data []byte) error { - err := serialization.UnmarshalValidateYAML(data, &state.Config) + err := serialization.UnmarshalValidate(data, &state.Config, yaml.Unmarshal) if err != nil { return err } diff --git a/internal/config/types/config.go b/internal/config/types/config.go index 4ff13a17..f9ce7312 100644 --- a/internal/config/types/config.go +++ b/internal/config/types/config.go @@ -4,6 +4,7 @@ import ( "regexp" "github.com/go-playground/validator/v10" + "github.com/goccy/go-yaml" "github.com/yusing/godoxy/agent/pkg/agent" "github.com/yusing/godoxy/internal/acl" "github.com/yusing/godoxy/internal/autocert" @@ -43,7 +44,7 @@ type ( func Validate(data []byte) gperr.Error { var model Config - return serialization.UnmarshalValidateYAML(data, &model) + return serialization.UnmarshalValidate(data, &model, yaml.Unmarshal) } func DefaultConfig() Config { diff --git a/internal/dnsproviders/go.mod b/internal/dnsproviders/go.mod index 20e83db7..5b541461 100644 --- a/internal/dnsproviders/go.mod +++ b/internal/dnsproviders/go.mod @@ -6,7 +6,7 @@ replace github.com/yusing/godoxy => ../.. require ( github.com/go-acme/lego/v4 v4.31.0 - github.com/yusing/godoxy v0.25.0 + github.com/yusing/godoxy v0.25.2 ) require ( @@ -44,7 +44,7 @@ require ( github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/goccy/go-yaml v1.19.2 // indirect github.com/gofrs/flock v0.13.0 // indirect - github.com/golang-jwt/jwt/v5 v5.3.0 // indirect + github.com/golang-jwt/jwt/v5 v5.3.1 // indirect github.com/google/go-querystring v1.2.0 // indirect github.com/google/s2a-go v0.1.9 // indirect github.com/google/uuid v1.6.0 // indirect @@ -65,8 +65,8 @@ require ( github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/nrdcg/goacmedns v0.2.0 // indirect github.com/nrdcg/goinwx v0.12.0 // indirect - github.com/nrdcg/oci-go-sdk/common/v1065 v1065.106.1 // indirect - github.com/nrdcg/oci-go-sdk/dns/v1065 v1065.106.1 // indirect + github.com/nrdcg/oci-go-sdk/common/v1065 v1065.107.0 // indirect + github.com/nrdcg/oci-go-sdk/dns/v1065 v1065.107.0 // indirect github.com/nrdcg/porkbun v0.4.0 // indirect github.com/ovh/go-ovh v1.9.0 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect @@ -98,8 +98,8 @@ require ( golang.org/x/sys v0.40.0 // indirect golang.org/x/text v0.33.0 // indirect golang.org/x/tools v0.41.0 // indirect - google.golang.org/api v0.262.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260122232226-8e98ce8d340d // indirect + google.golang.org/api v0.263.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 // indirect google.golang.org/grpc v1.78.0 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/ini.v1 v1.67.1 // indirect diff --git a/internal/dnsproviders/go.sum b/internal/dnsproviders/go.sum index 7ea4f067..63cb788e 100644 --- a/internal/dnsproviders/go.sum +++ b/internal/dnsproviders/go.sum @@ -90,8 +90,8 @@ github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7Lk github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw= github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0= -github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= -github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -150,10 +150,10 @@ github.com/nrdcg/goacmedns v0.2.0 h1:ADMbThobzEMnr6kg2ohs4KGa3LFqmgiBA22/6jUWJR0 github.com/nrdcg/goacmedns v0.2.0/go.mod h1:T5o6+xvSLrQpugmwHvrSNkzWht0UGAwj2ACBMhh73Cg= github.com/nrdcg/goinwx v0.12.0 h1:ujdUqDBnaRSFwzVnImvPHYw3w3m9XgmGImNUw1GyMb4= github.com/nrdcg/goinwx v0.12.0/go.mod h1:IrVKd3ZDbFiMjdPgML4CSxZAY9wOoqLvH44zv3NodJ0= -github.com/nrdcg/oci-go-sdk/common/v1065 v1065.106.1 h1:+fx2mbWeR8XX/vidwpRMepJMtRIYQP44Iezm2oeObVM= -github.com/nrdcg/oci-go-sdk/common/v1065 v1065.106.1/go.mod h1:Gcs8GCaZXL3FdiDWgdnMxlOLEdRprJJnPYB22TX1jw8= -github.com/nrdcg/oci-go-sdk/dns/v1065 v1065.106.1 h1:GDhBiaIAm/QXLzHJ0ASDdY/6R/9w60+gk8lY5rgfxEQ= -github.com/nrdcg/oci-go-sdk/dns/v1065 v1065.106.1/go.mod h1:EHScJdbM0gg5Is7e3C0ceRYAFMMsfP4Vf8sBRoxoTgk= +github.com/nrdcg/oci-go-sdk/common/v1065 v1065.107.0 h1:eMzyN+jGJbxG4ut278uwIsUo9XacXc711lFjhKnaUso= +github.com/nrdcg/oci-go-sdk/common/v1065 v1065.107.0/go.mod h1:Gcs8GCaZXL3FdiDWgdnMxlOLEdRprJJnPYB22TX1jw8= +github.com/nrdcg/oci-go-sdk/dns/v1065 v1065.107.0 h1:t34IpOa+8NfmjkU8bdWtYrLrmr346/FGhu8FlpJDQok= +github.com/nrdcg/oci-go-sdk/dns/v1065 v1065.107.0/go.mod h1:p95/OxVsdx71I2Qrck1GtIS87sRxcTRKXzUi5nWm9NY= github.com/nrdcg/porkbun v0.4.0 h1:rWweKlwo1PToQ3H+tEO9gPRW0wzzgmI/Ob3n2Guticw= github.com/nrdcg/porkbun v0.4.0/go.mod h1:/QMskrHEIM0IhC/wY7iTCUgINsxdT2WcOphktJ9+Q54= github.com/ovh/go-ovh v1.9.0 h1:6K8VoL3BYjVV3In9tPJUdT7qMx9h0GExN9EXx1r2kKE= @@ -249,14 +249,14 @@ golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc= golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/api v0.262.0 h1:4B+3u8He2GwyN8St3Jhnd3XRHlIvc//sBmgHSp78oNY= -google.golang.org/api v0.262.0/go.mod h1:jNwmH8BgUBJ/VrUG6/lIl9YiildyLd09r9ZLHiQ6cGI= +google.golang.org/api v0.263.0 h1:UFs7qn8gInIdtk1ZA6eXRXp5JDAnS4x9VRsRVCeKdbk= +google.golang.org/api v0.263.0/go.mod h1:fAU1xtNNisHgOF5JooAs8rRaTkl2rT3uaoNGo9NS3R8= google.golang.org/genproto v0.0.0-20251202230838-ff82c1b0f217 h1:GvESR9BIyHUahIb0NcTum6itIWtdoglGX+rnGxm2934= google.golang.org/genproto v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:yJ2HH4EHEDTd3JiLmhds6NkJ17ITVYOdV3m3VKOnws0= google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 h1:fCvbg86sFXwdrl5LgVcTEvNC+2txB5mgROGmRL5mrls= google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260122232226-8e98ce8d340d h1:xXzuihhT3gL/ntduUZwHECzAn57E8dA6l8SOtYWdD8Q= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260122232226-8e98ce8d340d/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 h1:H86B94AW+VfJWDqFeEbBPhEtHzJwJfTbgE2lZa54ZAQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= diff --git a/internal/health/check/http.go b/internal/health/check/http.go index fb946131..5ee11688 100644 --- a/internal/health/check/http.go +++ b/internal/health/check/http.go @@ -76,8 +76,11 @@ func H2C(ctx context.Context, url *url.URL, method, path string, timeout time.Du setCommonHeaders(req.Header.Set) + client := *h2cClient + client.Timeout = timeout + start := time.Now() - resp, err := h2cClient.Do(req) + resp, err := client.Do(req) lat := time.Since(start) if resp != nil { diff --git a/internal/homepage/icons/list/list_icons.go b/internal/homepage/icons/list/list_icons.go index 36992596..6882ce77 100644 --- a/internal/homepage/icons/list/list_icons.go +++ b/internal/homepage/icons/list/list_icons.go @@ -55,20 +55,20 @@ func init() { func InitCache() { m := make(IconMap) - err := serialization.LoadJSONIfExist(common.IconListCachePath, &m) + err := serialization.LoadFileIfExist(common.IconListCachePath, &m, sonic.Unmarshal) if err != nil { // backward compatible oldFormat := struct { Icons IconMap LastUpdate time.Time }{} - err = serialization.LoadJSONIfExist(common.IconListCachePath, &oldFormat) + err = serialization.LoadFileIfExist(common.IconListCachePath, &oldFormat, sonic.Unmarshal) if err != nil { log.Error().Err(err).Msg("failed to load icons") } else { m = oldFormat.Icons // store it to disk immediately - _ = serialization.SaveJSON(common.IconListCachePath, &m, 0o644) + _ = serialization.SaveFile(common.IconListCachePath, &m, 0o644, sonic.Marshal) } } else if len(m) > 0 { log.Info(). @@ -84,7 +84,7 @@ func InitCache() { task.OnProgramExit("save_icons_cache", func() { icons := iconsCache.Load() - _ = serialization.SaveJSON(common.IconListCachePath, &icons, 0o644) + _ = serialization.SaveFile(common.IconListCachePath, &icons, 0o644, sonic.Marshal) }) go backgroundUpdateIcons() @@ -105,7 +105,7 @@ func backgroundUpdateIcons() { // swap old cache with new cache iconsCache.Store(newCache) // save it to disk - err := serialization.SaveJSON(common.IconListCachePath, &newCache, 0o644) + err := serialization.SaveFile(common.IconListCachePath, &newCache, 0o644, sonic.Marshal) if err != nil { log.Warn().Err(err).Msg("failed to save icons") } diff --git a/internal/jsonstore/jsonstore.go b/internal/jsonstore/jsonstore.go index 598a6eea..780ac32b 100644 --- a/internal/jsonstore/jsonstore.go +++ b/internal/jsonstore/jsonstore.go @@ -83,7 +83,8 @@ func loadNS[T store](ns namespace) T { func save() error { errs := gperr.NewBuilder("failed to save data stores") for ns, store := range stores { - if err := serialization.SaveJSON(filepath.Join(storesPath, string(ns)+".json"), &store, 0o644); err != nil { + path := filepath.Join(storesPath, string(ns)+".json") + if err := serialization.SaveFile(path, &store, 0o644, sonic.Marshal); err != nil { errs.Add(err) } } diff --git a/internal/proxmox/README.md b/internal/proxmox/README.md index 563cb805..0373ff4d 100644 --- a/internal/proxmox/README.md +++ b/internal/proxmox/README.md @@ -70,6 +70,7 @@ type Client struct { *proxmox.Client *proxmox.Cluster Version *proxmox.Version + BaseURL *url.URL // id -> resource; id: lxc/ or qemu/ resources map[string]*VMResource resourcesMu sync.RWMutex @@ -79,6 +80,9 @@ type VMResource struct { *proxmox.ClusterResource IPs []net.IP } + +// NewClient creates a new Proxmox client. +func NewClient(baseUrl string, opts ...proxmox.Option) *Client ``` ### Node @@ -97,10 +101,11 @@ var Nodes = pool.New[*Node]("proxmox_nodes") ```go type NodeConfig struct { - Node string `json:"node" validate:"required"` - VMID int `json:"vmid" validate:"required"` - VMName string `json:"vmname,omitempty"` - Service string `json:"service,omitempty"` + Node string `json:"node" validate:"required"` + VMID *int `json:"vmid"` // nil: auto discover; 0: node-level route; >0: lxc/qemu resource route + VMName string `json:"vmname,omitempty"` + Services []string `json:"services,omitempty" aliases:"service"` + Files []string `json:"files,omitempty" aliases:"file"` } ``` @@ -119,6 +124,9 @@ func (c *Config) Client() *Client ### Client Operations ```go +// NewClient creates a new Proxmox client. +func NewClient(baseUrl string, opts ...proxmox.Option) *Client + // UpdateClusterInfo fetches cluster info and discovers nodes. func (c *Client) UpdateClusterInfo(ctx context.Context) error @@ -136,6 +144,15 @@ func (c *Client) ReverseLookupNode(hostname string, ip net.IP, alias string) str // NumNodes returns the number of nodes in the cluster. func (c *Client) NumNodes() int + +// Key returns the cluster ID. +func (c *Client) Key() string + +// Name returns the cluster name. +func (c *Client) Name() string + +// MarshalJSON returns the cluster info as JSON. +func (c *Client) MarshalJSON() ([]byte, error) ``` ### Node Operations @@ -144,17 +161,29 @@ func (c *Client) NumNodes() int // AvailableNodeNames returns all available node names as a comma-separated string. func AvailableNodeNames() string +// NewNode creates a new node. +func NewNode(client *Client, name, id string) *Node + // Node.Client returns the Proxmox client. func (n *Node) Client() *Client // Node.Get performs a GET request on the node. func (n *Node) Get(ctx context.Context, path string, v any) error +// Node.Key returns the node name. +func (n *Node) Key() string + +// Node.Name returns the node name. +func (n *Node) Name() string + // NodeCommand executes a command on the node and streams output. func (n *Node) NodeCommand(ctx context.Context, command string) (io.ReadCloser, error) // NodeJournalctl streams journalctl output from the node. -func (n *Node) NodeJournalctl(ctx context.Context, service string, limit int) (io.ReadCloser, error) +func (n *Node) NodeJournalctl(ctx context.Context, services []string, limit int) (io.ReadCloser, error) + +// NodeTail streams tail output for the given file. +func (n *Node) NodeTail(ctx context.Context, files []string, limit int) (io.ReadCloser, error) ``` ## Usage @@ -275,7 +304,35 @@ func (node *Node) LXCStats(ctx context.Context, vmid int, stream bool) (io.ReadC func (node *Node) LXCCommand(ctx context.Context, vmid int, command string) (io.ReadCloser, error) // LXCJournalctl streams journalctl output for a container service. -func (node *Node) LXCJournalctl(ctx context.Context, vmid int, service string, limit int) (io.ReadCloser, error) +// On non-systemd systems, it falls back to tailing /var/log/messages. +func (node *Node) LXCJournalctl(ctx context.Context, vmid int, services []string, limit int) (io.ReadCloser, error) + +// LXCTail streams tail output for the given file. +func (node *Node) LXCTail(ctx context.Context, vmid int, files []string, limit int) (io.ReadCloser, error) +``` + +## Node Stats + +```go +type NodeStats struct { + KernelVersion string `json:"kernel_version"` + PVEVersion string `json:"pve_version"` + CPUUsage string `json:"cpu_usage"` + CPUModel string `json:"cpu_model"` + MemUsage string `json:"mem_usage"` + MemTotal string `json:"mem_total"` + MemPct string `json:"mem_pct"` + RootFSUsage string `json:"rootfs_usage"` + RootFSTotal string `json:"rootfs_total"` + RootFSPct string `json:"rootfs_pct"` + Uptime string `json:"uptime"` + LoadAvg1m string `json:"load_avg_1m"` + LoadAvg5m string `json:"load_avg_5m"` + LoadAvg15m string `json:"load_avg_15m"` +} + +// NodeStats streams node statistics like docker stats. +func (n *Node) NodeStats(ctx context.Context, stream bool) (io.ReadCloser, error) ``` ## Data Flow @@ -453,6 +510,12 @@ var ( ) ``` +| Error | Description | +| --------------------- | --------------------------------------------------------------------- | +| `ErrResourceNotFound` | Resource not found in cluster | +| `ErrNoResources` | No resources available | +| `ErrNoSession` | No session for WebSocket operations (requires username/password auth) | + ## Performance Considerations - Cluster info fetched once on init @@ -463,10 +526,26 @@ var ( - Per-operation API calls with 3-second timeout - WebSocket connections properly closed to prevent goroutine leaks +## Command Validation + +Commands executed via WebSocket are validated to prevent command injection. Invalid characters include: + +``` +& | $ ; ' " ` $( ${ < > +``` + +Services and files passed to `journalctl` and `tail` commands are automatically validated. + ## Constants ```go const ResourcePollInterval = 3 * time.Second +const SessionRefreshInterval = 1 * time.Minute +const NodeStatsPollInterval = time.Second ``` -The `ResourcePollInterval` constant controls how often resources are updated in the background loop. +| Constant | Default | Description | +| ------------------------ | ------- | ---------------------------------- | +| `ResourcePollInterval` | 3s | How often VM resources are updated | +| `SessionRefreshInterval` | 1m | How often sessions are refreshed | +| `NodeStatsPollInterval` | 1s | How often node stats are streamed | diff --git a/internal/proxmox/client.go b/internal/proxmox/client.go index 7fe5cc60..674a02b6 100644 --- a/internal/proxmox/client.go +++ b/internal/proxmox/client.go @@ -14,7 +14,6 @@ import ( "github.com/bytedance/sonic" "github.com/luthermonson/go-proxmox" - "github.com/rs/zerolog/log" "golang.org/x/sync/errgroup" ) @@ -120,7 +119,6 @@ func (c *Client) UpdateResources(ctx context.Context) error { c.resources[resource.ID] = vmResources[i] } c.resourcesMu.Unlock() - log.Debug().Str("cluster", c.Cluster.Name).Msgf("[proxmox] updated %d resources", len(c.resources)) return nil } diff --git a/internal/proxmox/command_common.go b/internal/proxmox/command_common.go index f3404ef4..e48549d9 100644 --- a/internal/proxmox/command_common.go +++ b/internal/proxmox/command_common.go @@ -31,14 +31,15 @@ func formatTail(files []string, limit int) (string, error) { } } var command strings.Builder - command.WriteString("tail -f -q --retry ") + command.WriteString("tail -f -q ") for _, file := range files { fmt.Fprintf(&command, " %q ", file) } if limit > 0 { fmt.Fprintf(&command, " -n %d", limit) } - return command.String(), nil + // try --retry first, if it fails, try the command again + return fmt.Sprintf("sh -c '%s --retry 2>/dev/null || %s'", command.String(), command.String()), nil } func formatJournalctl(services []string, limit int) (string, error) { diff --git a/internal/proxmox/config.go b/internal/proxmox/config.go index 35d54445..54269adf 100644 --- a/internal/proxmox/config.go +++ b/internal/proxmox/config.go @@ -162,4 +162,4 @@ func (c *Config) refreshSessionLoop(ctx context.Context) { } } } -} \ No newline at end of file +} diff --git a/internal/proxmox/lxc_command.go b/internal/proxmox/lxc_command.go index f0c82fab..0259eff9 100644 --- a/internal/proxmox/lxc_command.go +++ b/internal/proxmox/lxc_command.go @@ -39,6 +39,8 @@ func (n *Node) LXCCommand(ctx context.Context, vmid int, command string) (io.Rea // LXCJournalctl streams journalctl output for the given service. // +// On non systemd systems, it will tail /var/log/messages as fallback. +// // If services are not empty, it will be used to filter the output by service. // If limit is greater than 0, it will be used to limit the number of lines of output. func (n *Node) LXCJournalctl(ctx context.Context, vmid int, services []string, limit int) (io.ReadCloser, error) { @@ -46,6 +48,11 @@ func (n *Node) LXCJournalctl(ctx context.Context, vmid int, services []string, l if err != nil { return nil, err } + if len(services) == 0 { + // add /var/log/messages fallback for non systemd systems + // in tail command, try --retry first, if it fails, try the command again + command = fmt.Sprintf("sh -c '%s 2>/dev/null || tail -f -q --retry /var/log/messages 2>/dev/null || tail -f -q /var/log/messages'", command) + } return n.LXCCommand(ctx, vmid, command) } diff --git a/internal/proxmox/node.go b/internal/proxmox/node.go index a7d0ca26..f24d8f14 100644 --- a/internal/proxmox/node.go +++ b/internal/proxmox/node.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/bytedance/sonic" + gperr "github.com/yusing/goutils/errs" "github.com/yusing/goutils/pool" ) @@ -25,6 +26,22 @@ type Node struct { // statsScriptInitErrs *xsync.Map[int, error] } +// Validate implements the serialization.CustomValidator interface. +func (n *NodeConfig) Validate() gperr.Error { + var errs gperr.Builder + for i, service := range n.Services { + if err := checkValidInput(service); err != nil { + errs.AddSubjectf(err, "services[%d]", i) + } + } + for i, file := range n.Files { + if err := checkValidInput(file); err != nil { + errs.AddSubjectf(err, "files[%d]", i) + } + } + return errs.Error() +} + var Nodes = pool.New[*Node]("proxmox_nodes") func NewNode(client *Client, name, id string) *Node { diff --git a/internal/proxmox/validation_test.go b/internal/proxmox/validation_test.go new file mode 100644 index 00000000..b944bc2f --- /dev/null +++ b/internal/proxmox/validation_test.go @@ -0,0 +1,56 @@ +package proxmox + +import ( + "testing" + + "github.com/goccy/go-yaml" + "github.com/stretchr/testify/require" + "github.com/yusing/godoxy/internal/serialization" +) + +func TestValidateCommandArgs(t *testing.T) { + tests := []struct { + name string + yamlCfg string + wantErr bool + }{ + { + name: "valid_services", + yamlCfg: `services: ["foo", "bar"]`, + wantErr: false, + }, + { + name: "invalid_services", + yamlCfg: `services: ["foo", "bar & baz"]`, + wantErr: true, + }, + { + name: "invalid_services_with_$(", + yamlCfg: `services: ["foo", "bar & $(echo 'hello')"]`, + wantErr: true, + }, + { + name: "valid_files", + yamlCfg: `files: ["foo", "bar"]`, + wantErr: false, + }, + { + name: "invalid_files", + yamlCfg: `files: ["foo", "bar & baz"]`, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var cfg NodeConfig + err := serialization.UnmarshalValidate([]byte(tt.yamlCfg), &cfg, yaml.Unmarshal) + if tt.wantErr { + require.Error(t, err) + require.ErrorContains(t, err, "input contains invalid characters") + } else { + require.NoError(t, err) + } + }) + } +} diff --git a/internal/route/provider/file.go b/internal/route/provider/file.go index bc4b1d8d..82860581 100644 --- a/internal/route/provider/file.go +++ b/internal/route/provider/file.go @@ -5,6 +5,7 @@ import ( "path" "strings" + "github.com/goccy/go-yaml" "github.com/rs/zerolog" "github.com/rs/zerolog/log" "github.com/yusing/godoxy/internal/common" @@ -43,7 +44,7 @@ func removeXPrefix(m map[string]any) gperr.Error { } func validate(data []byte) (routes route.Routes, err gperr.Error) { - err = serialization.UnmarshalValidateYAMLIntercept(data, &routes, removeXPrefix) + err = serialization.UnmarshalValidate(data, &routes, yaml.Unmarshal, removeXPrefix) return routes, err } diff --git a/internal/route/route.go b/internal/route/route.go index 6152ef5c..8f1aa03c 100644 --- a/internal/route/route.go +++ b/internal/route/route.go @@ -69,7 +69,7 @@ type ( Idlewatcher *types.IdlewatcherConfig `json:"idlewatcher,omitempty" extensions:"x-nullable"` Metadata `deserialize:"-"` - } + } // @name Route Metadata struct { /* Docker only */ @@ -199,6 +199,7 @@ func (r *Route) validate() gperr.Error { } if (r.Proxmox == nil || r.Proxmox.Node == "" || r.Proxmox.VMID == nil) && r.Container == nil { + wasNotNil := r.Proxmox != nil proxmoxProviders := config.WorkingState.Load().Value().Providers.Proxmox if len(proxmoxProviders) > 0 { // it's fine if ip is nil @@ -239,88 +240,13 @@ func (r *Route) validate() gperr.Error { } } } + if wasNotNil && (r.Proxmox.Node == "" || r.Proxmox.VMID == nil) { + log.Warn().Msgf("no proxmox node / resource found for route %q", r.Alias) + } } if r.Proxmox != nil { - nodeName := r.Proxmox.Node - vmid := r.Proxmox.VMID - if nodeName == "" || vmid == nil { - return gperr.Errorf("node (proxmox node name) is required") - } - - node, ok := proxmox.Nodes.Get(nodeName) - if !ok { - return gperr.Errorf("proxmox node %s not found in pool", nodeName) - } - - // Node-level route (VMID = 0) - if *vmid == 0 { - r.Scheme = route.SchemeHTTPS - if r.Host == DefaultHost { - r.Host = node.Client().BaseURL.Hostname() - } - port, _ := strconv.Atoi(node.Client().BaseURL.Port()) - if port == 0 { - port = 8006 - } - r.Port.Proxy = port - } else { - res, err := node.Client().GetResource("lxc", *vmid) - if err != nil { - return gperr.Wrap(err) // ErrResourceNotFound - } - - r.Proxmox.VMName = res.Name - - if r.Host == DefaultHost { - containerName := res.Name - // get ip addresses of the vmid - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - ips := res.IPs - if len(ips) == 0 { - return gperr.Multiline(). - Addf("no ip addresses found for %s", containerName). - Adds("make sure you have set static ip address for container instead of dhcp"). - Subject(containerName) - } - - l := log.With().Str("container", containerName).Logger() - - l.Info().Msg("checking if container is running") - running, err := node.LXCIsRunning(ctx, *vmid) - if err != nil { - return gperr.New("failed to check container state").With(err) - } - - if !running { - l.Info().Msg("starting container") - if err := node.LXCAction(ctx, *vmid, proxmox.LXCStart); err != nil { - return gperr.New("failed to start container").With(err) - } - } - - l.Info().Msgf("finding reachable ip addresses") - errs := gperr.NewBuilder("failed to find reachable ip addresses") - for _, ip := range ips { - if err := netutils.PingTCP(ctx, ip, r.Port.Proxy); err != nil { - errs.Add(gperr.Unwrap(err).Subjectf("%s:%d", ip, r.Port.Proxy)) - } else { - r.Host = ip.String() - l.Info().Msgf("using ip %s", r.Host) - break - } - } - if r.Host == DefaultHost { - return gperr.Multiline(). - Addf("no reachable ip addresses found, tried %d IPs", len(ips)). - With(errs.Error()). - Subject(containerName) - } - } - } + r.validateProxmox() } if r.Container != nil && r.Container.IdlewatcherConfig != nil { @@ -470,6 +396,90 @@ func (r *Route) validateRules() error { return nil } +func (r *Route) validateProxmox() { + l := log.With().Str("route", r.Alias).Logger() + + nodeName := r.Proxmox.Node + vmid := r.Proxmox.VMID + if nodeName == "" || vmid == nil { + l.Error().Msg("node (proxmox node name) is required") + return + } + + node, ok := proxmox.Nodes.Get(nodeName) + if !ok { + l.Error().Msgf("proxmox node %s not found in pool", nodeName) + return + } + + // Node-level route (VMID = 0) + if *vmid == 0 { + r.Scheme = route.SchemeHTTPS + if r.Host == DefaultHost { + r.Host = node.Client().BaseURL.Hostname() + } + port, _ := strconv.Atoi(node.Client().BaseURL.Port()) + if port == 0 { + port = 8006 + } + r.Port.Proxy = port + } else { + res, err := node.Client().GetResource("lxc", *vmid) + if err != nil { // ErrResourceNotFound + l.Err(err).Msgf("failed to get resource %d", *vmid) + return + } + + r.Proxmox.VMName = res.Name + + if r.Host == DefaultHost { + containerName := res.Name + // get ip addresses of the vmid + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + ips := res.IPs + if len(ips) == 0 { + l.Warn().Msgf("no ip addresses found for %s, make sure you have set static ip address for container instead of dhcp", containerName) + return + } + + l = l.With().Str("container", containerName).Logger() + + l.Info().Msgf("checking if container is running") + running, err := node.LXCIsRunning(ctx, *vmid) + if err != nil { + l.Err(err).Msgf("failed to check container state") + return + } + + if !running { + l.Info().Msgf("starting container") + if err := node.LXCAction(ctx, *vmid, proxmox.LXCStart); err != nil { + l.Err(err).Msgf("failed to start container") + return + } + } + + l.Info().Msgf("finding reachable ip addresses") + errs := gperr.NewBuilder("failed to find reachable ip addresses") + for _, ip := range ips { + if err := netutils.PingTCP(ctx, ip, r.Port.Proxy); err != nil { + errs.Add(gperr.Unwrap(err).Subjectf("%s:%d", ip, r.Port.Proxy)) + } else { + r.Host = ip.String() + l.Info().Msgf("using ip %s", r.Host) + break + } + } + if r.Host == DefaultHost { + l.Warn().Err(errs.Error()).Msgf("no reachable ip addresses found, tried %d IPs", len(ips)) + } + } + } +} + func (r *Route) Impl() types.Route { return r.impl } diff --git a/internal/route/rules/rules.go b/internal/route/rules/rules.go index 0fbf9b88..ebaf5fab 100644 --- a/internal/route/rules/rules.go +++ b/internal/route/rules/rules.go @@ -244,7 +244,7 @@ func (rules Rules) BuildHandler(up http.HandlerFunc) http.HandlerFunc { } func appendRuleError(rm *httputils.ResponseModifier, rule *Rule, err error) { - rm.AppendError("rule: %s, error: %w", rule.Name, err) + // rm.AppendError("rule: %s, error: %w", rule.Name, err) } func isTerminatingHandler(handler CommandHandler) bool { diff --git a/internal/route/rules/validate.go b/internal/route/rules/validate.go index 6b2e2062..14c71a3f 100644 --- a/internal/route/rules/validate.go +++ b/internal/route/rules/validate.go @@ -115,24 +115,6 @@ func validateURL(args []string) (any, gperr.Error) { return u, nil } -// validateAbsoluteURL returns types.URL with the URL validated. -func validateAbsoluteURL(args []string) (any, gperr.Error) { - if len(args) != 1 { - return nil, ErrExpectOneArg - } - u, err := nettypes.ParseURL(args[0]) - if err != nil { - return nil, ErrInvalidArguments.With(err) - } - if u.Scheme == "" { - u.Scheme = "http" - } - if u.Host == "" { - return nil, ErrInvalidArguments.Withf("missing host") - } - return u, nil -} - // validateCIDR returns types.CIDR with the CIDR validated. func validateCIDR(args []string) (any, gperr.Error) { if len(args) != 1 { diff --git a/internal/serialization/README.md b/internal/serialization/README.md index 21165794..89cedf34 100644 --- a/internal/serialization/README.md +++ b/internal/serialization/README.md @@ -11,7 +11,7 @@ This package provides robust YAML/JSON serialization with: - Case-insensitive field matching using FNV-1a hashing - Environment variable substitution (`${VAR}` syntax) - Field-level validation with go-playground/validator tags -- Custom type conversion with alias support +- Custom type conversion with pluggable format handlers ### Primary Consumers @@ -55,21 +55,27 @@ type CustomValidator interface { ### Deserialization Functions ```go -// YAML with full validation -func UnmarshalValidateYAML[T any](data []byte, target *T) gperr.Error +// Generic unmarshal with pluggable format handler +func UnmarshalValidate[T any](data []byte, target *T, unmarshaler unmarshalFunc, interceptFns ...interceptFunc) gperr.Error -// YAML with interceptor for preprocessing -func UnmarshalValidateYAMLIntercept[T any]( - data []byte, - target *T, - intercept func(m map[string]any) gperr.Error, -) gperr.Error +// Read from io.Reader with format decoder +func UnmarshalValidateReader[T any](reader io.Reader, target *T, newDecoder newDecoderFunc, interceptFns ...interceptFunc) gperr.Error // Direct map deserialization func MapUnmarshalValidate(src SerializedObject, dst any) gperr.Error -// To xsync.Map -func UnmarshalValidateYAMLXSync[V any](data []byte) (*xsync.Map[string, V], gperr.Error) +// To xsync.Map with pluggable format handler +func UnmarshalValidateXSync[V any](data []byte, unmarshaler unmarshalFunc, interceptFns ...interceptFunc) (*xsync.Map[string, V], gperr.Error) +``` + +### File I/O Functions + +```go +// Write marshaled data to file +func SaveFile[T any](path string, src *T, perm os.FileMode, marshaler marshalFunc) error + +// Read and unmarshal file if it exists +func LoadFileIfExist[T any](path string, dst *T, unmarshaler unmarshalFunc) error ``` ### Conversion Functions @@ -115,19 +121,19 @@ func ToSerializedObject[VT any](m map[string]VT) SerializedObject ```mermaid sequenceDiagram participant C as Caller - participant U as UnmarshalValidateYAML + participant U as UnmarshalValidate participant E as Env Substitution - participant Y as YAML Parser + participant F as Format Parser participant M as MapUnmarshalValidate participant T as Type Info Cache participant CV as Convert participant V as Validator - C->>U: YAML bytes + target struct + C->>U: Data bytes + target struct + format handler U->>E: Substitute ${ENV} vars E-->>U: Substituted bytes - U->>Y: Parse YAML - Y-->>U: map[string]any + U->>F: Parse with format handler (YAML/JSON) + F-->>U: map[string]any U->>M: Map + target M->>T: Get type info loop For each field in map @@ -147,9 +153,9 @@ sequenceDiagram ```mermaid flowchart TB subgraph Input Processing - YAML[YAML Bytes] --> EnvSub[Env Substitution] - EnvSub --> YAMLParse[YAML Parse] - YAMLParse --> Map[map] + Bytes[Data Bytes] --> EnvSub[Env Substitution] + EnvSub --> FormatParse[Format Parse] + FormatParse --> Map[map] end subgraph Type Inspection @@ -221,6 +227,7 @@ autocert: ### Internal Dependencies - `github.com/yusing/goutils/errs` - Error handling +- `github.com/yusing/gointernals` - Reflection utilities ## Observability @@ -251,11 +258,11 @@ ErrUnsupportedConversion.Subjectf("string to int") | Validation failure | Structured error | Fix field value | | Type mismatch | Error | Check field type | | Missing env var | Error | Set environment variable | -| Invalid YAML | Error | Fix YAML syntax | +| Invalid format | Error | Fix YAML/JSON syntax | ## Usage Examples -### Basic Struct Deserialization +### YAML Deserialization ```go type ServerConfig struct { @@ -273,7 +280,16 @@ tls_enabled: true `) var config ServerConfig -if err := serialization.UnmarshalValidateYAML(yamlData, &config); err != nil { +if err := serialization.UnmarshalValidate(yamlData, &config, yaml.Unmarshal); err != nil { + panic(err) +} +``` + +### JSON Deserialization + +```go +var config ServerConfig +if err := serialization.UnmarshalValidate(jsonData, &config, json.Unmarshal); err != nil { panic(err) } ``` @@ -293,7 +309,7 @@ func (c *Config) Validate() gperr.Error { } ``` -### Custom Type with Parse Method +### Custom Type with Parser Interface ```go type Duration struct { @@ -307,6 +323,31 @@ func (d *Duration) Parse(v string) error { } ``` +### Reading from File + +```go +var config ServerConfig +if err := serialization.LoadFileIfExist("config.yml", &config, yaml.Unmarshal); err != nil { + panic(err) +} + +// Save back to file +if err := serialization.SaveFile("config.yml", &config, 0644, yaml.Marshal); err != nil { + panic(err) +} +``` + +### Reading from io.Reader + +```go +var config ServerConfig +file, _ := os.Open("config.yml") +defer file.Close() +if err := serialization.UnmarshalValidateReader(file, &config, yaml.NewDecoder); err != nil { + panic(err) +} +``` + ## Testing Notes - `serialization_test.go` - Core functionality tests @@ -319,3 +360,4 @@ func (d *Duration) Parse(v string) error { - String conversions - Environment substitution - Custom validators + - Multiple format handlers (YAML/JSON) diff --git a/internal/serialization/gin_binding.go b/internal/serialization/gin_binding.go new file mode 100644 index 00000000..6631792c --- /dev/null +++ b/internal/serialization/gin_binding.go @@ -0,0 +1,37 @@ +package serialization + +import ( + "net/http" + + "github.com/bytedance/sonic" + "github.com/goccy/go-yaml" +) + +type ( + GinJSONBinding struct{} + GinYAMLBinding struct{} +) + +func (b GinJSONBinding) Name() string { + return "json" +} + +func (b GinJSONBinding) Bind(req *http.Request, obj any) error { + m := make(map[string]any) + if err := sonic.ConfigDefault.NewDecoder(NewSubstituteEnvReader(req.Body)).Decode(&m); err != nil { + return err + } + return MapUnmarshalValidate(m, obj) +} + +func (b GinYAMLBinding) Name() string { + return "yaml" +} + +func (b GinYAMLBinding) Bind(req *http.Request, obj any) error { + m := make(map[string]any) + if err := yaml.NewDecoder(NewSubstituteEnvReader(req.Body)).Decode(&m); err != nil { + return err + } + return MapUnmarshalValidate(m, obj) +} diff --git a/internal/serialization/gin_binding_test.go b/internal/serialization/gin_binding_test.go new file mode 100644 index 00000000..d7f8830e --- /dev/null +++ b/internal/serialization/gin_binding_test.go @@ -0,0 +1,50 @@ +package serialization_test + +import ( + "bytes" + "net/http/httptest" + "testing" + + "github.com/yusing/godoxy/internal/serialization" + gperr "github.com/yusing/goutils/errs" +) + +type TestStruct struct { + Value string `json:"value"` + Value2 int `json:"value2"` +} + +func (t *TestStruct) Validate() gperr.Error { + if t.Value == "" { + return gperr.New("value is required") + } + if t.Value2 != 0 && (t.Value2 < 5 || t.Value2 > 10) { + return gperr.New("value2 must be between 5 and 10") + } + return nil +} + +func TestGinBinding(t *testing.T) { + + tests := []struct { + name string + input string + wantErr bool + }{ + {"valid1", `{"value": "test", "value2": 7}`, false}, + {"valid2", `{"value": "test"}`, false}, + {"invalid1", `{"value2": 7}`, true}, + {"invalid2", `{"value": "test", "value2": 3}`, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var dst TestStruct + body := bytes.NewBufferString(tt.input) + req := httptest.NewRequest("POST", "/", body) + err := serialization.GinJSONBinding{}.Bind(req, &dst) + if (err != nil) != tt.wantErr { + t.Errorf("%s: Bind() error = %v, wantErr %v", tt.name, err, tt.wantErr) + } + }) + } +} diff --git a/internal/serialization/reader.go b/internal/serialization/reader.go new file mode 100644 index 00000000..b44ee3cd --- /dev/null +++ b/internal/serialization/reader.go @@ -0,0 +1,146 @@ +package serialization + +import ( + "bytes" + "io" +) + +type SubstituteEnvReader struct { + reader io.Reader + buf []byte // buffered data with substitutions applied + err error // sticky error +} + +func NewSubstituteEnvReader(reader io.Reader) *SubstituteEnvReader { + return &SubstituteEnvReader{reader: reader} +} + +const peekSize = 4096 +const maxVarNameLength = 256 + +func (r *SubstituteEnvReader) Read(p []byte) (n int, err error) { + // Return buffered data first + if len(r.buf) > 0 { + n = copy(p, r.buf) + r.buf = r.buf[n:] + return n, nil + } + + // Return sticky error if we have one + if r.err != nil { + return 0, r.err + } + + var buf [2 * peekSize]byte + + // Read a chunk from the underlying reader + chunk, more := buf[:peekSize], buf[peekSize:] + nRead, readErr := r.reader.Read(chunk) + if nRead == 0 { + if readErr != nil { + return 0, readErr + } + return 0, io.EOF + } + chunk = chunk[:nRead] + + // Check if there's a potential incomplete pattern at the end + // Pattern: ${VAR_NAME} + // We need to check if chunk ends with a partial pattern like "$", "${", "${VAR", etc. + incompleteStart := findIncompletePatternStart(chunk) + + if incompleteStart >= 0 && readErr == nil { + // There might be an incomplete pattern, read more to complete it + incomplete := chunk[incompleteStart:] + chunk = chunk[:incompleteStart] + + // Keep reading until we complete the pattern or hit EOF/error + for { + // Limit how much we buffer to prevent memory exhaustion + if len(incomplete) > maxVarNameLength+3 { // ${} + var name + // Pattern too long to be valid, give up and process as-is + chunk = append(chunk, incomplete...) + break + } + nMore, moreErr := r.reader.Read(more) + if nMore > 0 { + incomplete = append(incomplete, more[:nMore]...) + // Check if pattern is now complete + if idx := bytes.IndexByte(incomplete, '}'); idx >= 0 { + // Pattern complete, append the rest back to chunk + chunk = append(chunk, incomplete...) + break + } + } + if moreErr != nil { + // No more data, append whatever we have + chunk = append(chunk, incomplete...) + readErr = moreErr + break + } + } + } + + substituted, subErr := substituteEnv(chunk) + if subErr != nil { + r.err = subErr + return 0, subErr + } + + n = copy(p, substituted) + if n < len(substituted) { + // Buffer the rest + r.buf = substituted[n:] + } + + // Store sticky error for next read + if readErr != nil && readErr != io.EOF { + r.err = readErr + } else { + if readErr == io.EOF && n > 0 { + return n, nil + } + if readErr == io.EOF { + return n, io.EOF + } + } + + return n, nil +} + +// findIncompletePatternStart returns the index where an incomplete ${...} pattern starts, +// or -1 if there's no incomplete pattern at the end. +func findIncompletePatternStart(data []byte) int { + // Look for '$' near the end that might be start of ${VAR} + // Maximum var name we reasonably expect + "${}" = ~256 chars + searchStart := max(0, len(data)-maxVarNameLength) + + for i := len(data) - 1; i >= searchStart; i-- { + if data[i] == '$' { + // Check if this is a complete pattern or incomplete + if i+1 >= len(data) { + // Just "$" at end + return i + } + if data[i+1] == '{' { + // Check if there's anything after "${" + if i+2 >= len(data) { + // Just "${" at end + return i + } + // Check if pattern is complete by looking for '}' + for j := i + 2; j < len(data); j++ { + if data[j] == '}' { + // This pattern is complete, continue searching for another + break + } + if j == len(data)-1 { + // Reached end without finding '}', incomplete pattern + return i + } + } + } + } + } + return -1 +} diff --git a/internal/serialization/reader_bench_test.go b/internal/serialization/reader_bench_test.go new file mode 100644 index 00000000..7a415b6d --- /dev/null +++ b/internal/serialization/reader_bench_test.go @@ -0,0 +1,286 @@ +package serialization + +import ( + "bytes" + "io" + "os" + "strings" + "testing" +) + +// setupEnv sets up environment variables for benchmarks +func setupEnv(b *testing.B) { + b.Helper() + os.Setenv("BENCH_VAR", "benchmark_value") + os.Setenv("BENCH_VAR_2", "second_value") + os.Setenv("BENCH_VAR_3", "third_value") +} + +// cleanupEnv cleans up environment variables after benchmarks +func cleanupEnv(b *testing.B) { + b.Helper() + os.Unsetenv("BENCH_VAR") + os.Unsetenv("BENCH_VAR_2") + os.Unsetenv("BENCH_VAR_3") +} + +// BenchmarkSubstituteEnvReader_NoSubstitution benchmarks reading without any env substitutions +func BenchmarkSubstituteEnvReader_NoSubstitution(b *testing.B) { + r := strings.NewReader(`key: value +name: test +data: some content here +`) + + for b.Loop() { + reader := NewSubstituteEnvReader(r) + _, err := io.ReadAll(reader) + if err != nil { + b.Fatal(err) + } + r.Seek(0, io.SeekStart) + } +} + +// BenchmarkSubstituteEnvReader_SingleSubstitution benchmarks reading with a single env substitution +func BenchmarkSubstituteEnvReader_SingleSubstitution(b *testing.B) { + setupEnv(b) + defer cleanupEnv(b) + + r := strings.NewReader(`key: ${BENCH_VAR} +`) + + for b.Loop() { + reader := NewSubstituteEnvReader(r) + _, err := io.ReadAll(reader) + if err != nil { + b.Fatal(err) + } + r.Seek(0, io.SeekStart) + } +} + +// BenchmarkSubstituteEnvReader_MultipleSubstitutions benchmarks reading with multiple env substitutions +func BenchmarkSubstituteEnvReader_MultipleSubstitutions(b *testing.B) { + setupEnv(b) + defer cleanupEnv(b) + + r := strings.NewReader(`key1: ${BENCH_VAR} +key2: ${BENCH_VAR_2} +key3: ${BENCH_VAR_3} +`) + + for b.Loop() { + reader := NewSubstituteEnvReader(r) + _, err := io.ReadAll(reader) + if err != nil { + b.Fatal(err) + } + r.Seek(0, io.SeekStart) + } +} + +// BenchmarkSubstituteEnvReader_LargeInput_NoSubstitution benchmarks large input without substitutions +func BenchmarkSubstituteEnvReader_LargeInput_NoSubstitution(b *testing.B) { + r := strings.NewReader(strings.Repeat("x", 100000)) + + for b.Loop() { + reader := NewSubstituteEnvReader(r) + _, err := io.ReadAll(reader) + if err != nil { + b.Fatal(err) + } + r.Seek(0, io.SeekStart) + } +} + +// BenchmarkSubstituteEnvReader_LargeInput_WithSubstitutions benchmarks large input with scattered substitutions +func BenchmarkSubstituteEnvReader_LargeInput_WithSubstitutions(b *testing.B) { + setupEnv(b) + defer cleanupEnv(b) + + var builder bytes.Buffer + for range 100 { + builder.WriteString(strings.Repeat("x", 1000)) + builder.WriteString("${BENCH_VAR}") + } + r := bytes.NewReader(builder.Bytes()) + + for b.Loop() { + reader := NewSubstituteEnvReader(r) + _, err := io.ReadAll(reader) + if err != nil { + b.Fatal(err) + } + r.Seek(0, io.SeekStart) + } +} + +// BenchmarkSubstituteEnvReader_SmallBuffer benchmarks reading with a small buffer size +func BenchmarkSubstituteEnvReader_SmallBuffer(b *testing.B) { + setupEnv(b) + defer cleanupEnv(b) + + r := strings.NewReader(`key: ${BENCH_VAR} and some more content here`) + buf := make([]byte, 16) + + for b.Loop() { + reader := NewSubstituteEnvReader(r) + for { + _, err := reader.Read(buf) + if err == io.EOF { + break + } + if err != nil { + b.Fatal(err) + } + } + r.Seek(0, io.SeekStart) + } +} + +// BenchmarkSubstituteEnvReader_YAMLConfig benchmarks a realistic YAML config scenario +func BenchmarkSubstituteEnvReader_YAMLConfig(b *testing.B) { + setupEnv(b) + defer cleanupEnv(b) + + r := strings.NewReader(`database: + host: ${BENCH_VAR} + port: ${BENCH_VAR_2} + username: ${BENCH_VAR_3} + password: ${BENCH_VAR} +cache: + enabled: true + ttl: ${BENCH_VAR_2} +server: + host: ${BENCH_VAR} + port: 8080 +`) + + b.ResetTimer() + for b.Loop() { + reader := NewSubstituteEnvReader(r) + _, err := io.ReadAll(reader) + if err != nil { + b.Fatal(err) + } + r.Seek(0, io.SeekStart) + } +} + +// BenchmarkSubstituteEnvReader_BoundaryPattern benchmarks patterns at buffer boundaries (4096 bytes) +func BenchmarkSubstituteEnvReader_BoundaryPattern(b *testing.B) { + setupEnv(b) + defer cleanupEnv(b) + + // Pattern exactly at 4090 bytes, with ${VAR} crossing the 4096 boundary + prefix := strings.Repeat("x", 4090) + r := strings.NewReader(prefix + "${BENCH_VAR}") + + for b.Loop() { + reader := NewSubstituteEnvReader(r) + _, err := io.ReadAll(reader) + if err != nil { + b.Fatal(err) + } + r.Seek(0, io.SeekStart) + } +} + +// BenchmarkSubstituteEnvReader_MultipleBoundaries benchmarks multiple patterns crossing boundaries +func BenchmarkSubstituteEnvReader_MultipleBoundaries(b *testing.B) { + setupEnv(b) + defer cleanupEnv(b) + + var builder bytes.Buffer + for range 10 { + builder.WriteString(strings.Repeat("x", 4000)) + builder.WriteString("${BENCH_VAR}") + } + r := bytes.NewReader(builder.Bytes()) + + for b.Loop() { + reader := NewSubstituteEnvReader(r) + _, err := io.ReadAll(reader) + if err != nil { + b.Fatal(err) + } + r.Seek(0, io.SeekStart) + } +} + +// BenchmarkSubstituteEnvReader_SpecialChars benchmarks substitution with special characters +func BenchmarkSubstituteEnvReader_SpecialChars(b *testing.B) { + os.Setenv("SPECIAL_BENCH_VAR", `value with "quotes" and \backslash\`) + defer os.Unsetenv("SPECIAL_BENCH_VAR") + + r := strings.NewReader(`key: ${SPECIAL_BENCH_VAR} +`) + + for b.Loop() { + reader := NewSubstituteEnvReader(r) + _, err := io.ReadAll(reader) + if err != nil { + b.Fatal(err) + } + r.Seek(0, io.SeekStart) + } +} + +// BenchmarkSubstituteEnvReader_EmptyValue benchmarks substitution with empty value +func BenchmarkSubstituteEnvReader_EmptyValue(b *testing.B) { + os.Setenv("EMPTY_BENCH_VAR", "") + defer os.Unsetenv("EMPTY_BENCH_VAR") + + r := strings.NewReader(`key: ${EMPTY_BENCH_VAR} +`) + + for b.Loop() { + reader := NewSubstituteEnvReader(r) + _, err := io.ReadAll(reader) + if err != nil { + b.Fatal(err) + } + r.Seek(0, io.SeekStart) + } +} + +// BenchmarkSubstituteEnvReader_DollarWithoutBrace benchmarks $ without following { +func BenchmarkSubstituteEnvReader_DollarWithoutBrace(b *testing.B) { + os.Setenv("BENCH_VAR", "benchmark_value") + defer os.Unsetenv("BENCH_VAR") + + r := strings.NewReader(`price: $100 and $200 for ${BENCH_VAR}`) + + for b.Loop() { + reader := NewSubstituteEnvReader(r) + _, err := io.ReadAll(reader) + if err != nil { + b.Fatal(err) + } + r.Seek(0, io.SeekStart) + } +} + +// BenchmarkFindIncompletePatternStart benchmarks the findIncompletePatternStart function +func BenchmarkFindIncompletePatternStart(b *testing.B) { + testCases := []struct { + name string + input string + }{ + {"no pattern", strings.Repeat("hello world ", 100)}, + {"complete pattern", strings.Repeat("hello ${VAR} world ", 50)}, + {"dollar at end", strings.Repeat("hello ", 100) + "$"}, + {"incomplete at end", strings.Repeat("hello ", 100) + "${VAR"}, + {"large input no pattern", strings.Repeat("x", 5000)}, + {"large input with pattern", strings.Repeat("x", 4000) + "${VAR}"}, + } + + for _, tc := range testCases { + b.Run(tc.name, func(b *testing.B) { + data := []byte(tc.input) + for b.Loop() { + findIncompletePatternStart(data) + } + }) + } +} diff --git a/internal/serialization/reader_test.go b/internal/serialization/reader_test.go new file mode 100644 index 00000000..2d9f6961 --- /dev/null +++ b/internal/serialization/reader_test.go @@ -0,0 +1,217 @@ +package serialization + +import ( + "bytes" + "io" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSubstituteEnvReader_Basic(t *testing.T) { + os.Setenv("TEST_VAR", "hello") + defer os.Unsetenv("TEST_VAR") + + input := []byte(`key: ${TEST_VAR}`) + reader := NewSubstituteEnvReader(bytes.NewReader(input)) + + output, err := io.ReadAll(reader) + require.NoError(t, err) + require.Equal(t, `key: "hello"`, string(output)) +} + +func TestSubstituteEnvReader_Multiple(t *testing.T) { + os.Setenv("VAR1", "first") + os.Setenv("VAR2", "second") + defer os.Unsetenv("VAR1") + defer os.Unsetenv("VAR2") + + input := []byte(`a: ${VAR1}, b: ${VAR2}`) + reader := NewSubstituteEnvReader(bytes.NewReader(input)) + + output, err := io.ReadAll(reader) + require.NoError(t, err) + require.Equal(t, `a: "first", b: "second"`, string(output)) +} + +func TestSubstituteEnvReader_NoSubstitution(t *testing.T) { + input := []byte(`key: value`) + reader := NewSubstituteEnvReader(bytes.NewReader(input)) + + output, err := io.ReadAll(reader) + require.NoError(t, err) + require.Equal(t, `key: value`, string(output)) +} + +func TestSubstituteEnvReader_UnsetEnvError(t *testing.T) { + os.Unsetenv("UNSET_VAR_FOR_TEST") + + input := []byte(`key: ${UNSET_VAR_FOR_TEST}`) + reader := NewSubstituteEnvReader(bytes.NewReader(input)) + + _, err := io.ReadAll(reader) + require.Error(t, err) + require.Contains(t, err.Error(), "UNSET_VAR_FOR_TEST is not set") +} + +func TestSubstituteEnvReader_SmallBuffer(t *testing.T) { + os.Setenv("SMALL_BUF_VAR", "value") + defer os.Unsetenv("SMALL_BUF_VAR") + + input := []byte(`key: ${SMALL_BUF_VAR}`) + reader := NewSubstituteEnvReader(bytes.NewReader(input)) + + var result []byte + buf := make([]byte, 3) + for { + n, err := reader.Read(buf) + if n > 0 { + result = append(result, buf[:n]...) + } + if err == io.EOF { + break + } + require.NoError(t, err) + } + require.Equal(t, `key: "value"`, string(result)) +} + +func TestSubstituteEnvReader_SpecialChars(t *testing.T) { + os.Setenv("SPECIAL_VAR", `hello "world" \n`) + defer os.Unsetenv("SPECIAL_VAR") + + input := []byte(`key: ${SPECIAL_VAR}`) + reader := NewSubstituteEnvReader(bytes.NewReader(input)) + + output, err := io.ReadAll(reader) + require.NoError(t, err) + require.Equal(t, `key: "hello \"world\" \\n"`, string(output)) +} + +func TestSubstituteEnvReader_EmptyValue(t *testing.T) { + os.Setenv("EMPTY_VAR", "") + defer os.Unsetenv("EMPTY_VAR") + + input := []byte(`key: ${EMPTY_VAR}`) + reader := NewSubstituteEnvReader(bytes.NewReader(input)) + + output, err := io.ReadAll(reader) + require.NoError(t, err) + require.Equal(t, `key: ""`, string(output)) +} + +func TestSubstituteEnvReader_LargeInput(t *testing.T) { + os.Setenv("LARGE_VAR", "replaced") + defer os.Unsetenv("LARGE_VAR") + + prefix := strings.Repeat("x", 5000) + suffix := strings.Repeat("y", 5000) + input := []byte(prefix + "${LARGE_VAR}" + suffix) + + reader := NewSubstituteEnvReader(bytes.NewReader(input)) + + output, err := io.ReadAll(reader) + require.NoError(t, err) + expected := prefix + `"replaced"` + suffix + require.Equal(t, expected, string(output)) +} + +func TestSubstituteEnvReader_PatternAtBoundary(t *testing.T) { + os.Setenv("BOUNDARY_VAR", "boundary_value") + defer os.Unsetenv("BOUNDARY_VAR") + + prefix := strings.Repeat("a", 4090) + input := []byte(prefix + "${BOUNDARY_VAR}") + + reader := NewSubstituteEnvReader(bytes.NewReader(input)) + + output, err := io.ReadAll(reader) + require.NoError(t, err) + expected := prefix + `"boundary_value"` + require.Equal(t, expected, string(output)) +} + +func TestSubstituteEnvReader_MultiplePatternsBoundary(t *testing.T) { + os.Setenv("VAR_A", "aaa") + os.Setenv("VAR_B", "bbb") + defer os.Unsetenv("VAR_A") + defer os.Unsetenv("VAR_B") + + prefix := strings.Repeat("x", 4090) + input := []byte(prefix + "${VAR_A} middle ${VAR_B}") + + reader := NewSubstituteEnvReader(bytes.NewReader(input)) + + output, err := io.ReadAll(reader) + require.NoError(t, err) + expected := prefix + `"aaa" middle "bbb"` + require.Equal(t, expected, string(output)) +} + +func TestSubstituteEnvReader_YAMLConfig(t *testing.T) { + os.Setenv("DB_HOST", "localhost") + os.Setenv("DB_PORT", "5432") + os.Setenv("DB_PASSWORD", "secret123") + defer os.Unsetenv("DB_HOST") + defer os.Unsetenv("DB_PORT") + defer os.Unsetenv("DB_PASSWORD") + + input := []byte(`database: + host: ${DB_HOST} + port: ${DB_PORT} + password: ${DB_PASSWORD} +`) + reader := NewSubstituteEnvReader(bytes.NewReader(input)) + + output, err := io.ReadAll(reader) + require.NoError(t, err) + expected := `database: + host: "localhost" + port: "5432" + password: "secret123" +` + require.Equal(t, expected, string(output)) +} + +func TestSubstituteEnvReader_DollarWithoutBrace(t *testing.T) { + input := []byte(`key: $NOT_A_PATTERN`) + reader := NewSubstituteEnvReader(bytes.NewReader(input)) + + output, err := io.ReadAll(reader) + require.NoError(t, err) + require.Equal(t, `key: $NOT_A_PATTERN`, string(output)) +} + +func TestSubstituteEnvReader_EmptyInput(t *testing.T) { + input := []byte(``) + reader := NewSubstituteEnvReader(bytes.NewReader(input)) + + output, err := io.ReadAll(reader) + require.NoError(t, err) + require.Equal(t, ``, string(output)) +} + +func TestFindIncompletePatternStart(t *testing.T) { + tests := []struct { + name string + input string + expected int + }{ + {"no pattern", "hello world", -1}, + {"complete pattern", "hello ${VAR} world", -1}, + {"dollar at end", "hello $", 6}, + {"dollar brace at end", "hello ${", 6}, + {"incomplete var at end", "hello ${VAR", 6}, + {"complete then incomplete", "hello ${VAR} ${INCOMPLETE", 13}, + {"multiple complete", "${A} ${B} ${C}", -1}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := findIncompletePatternStart([]byte(tt.input)) + require.Equal(t, tt.expected, result) + }) + } +} diff --git a/internal/serialization/serialization.go b/internal/serialization/serialization.go index bd16526e..920abbb9 100644 --- a/internal/serialization/serialization.go +++ b/internal/serialization/serialization.go @@ -2,6 +2,7 @@ package serialization import ( "errors" + "io" "os" "reflect" "regexp" @@ -85,6 +86,10 @@ func initPtr(dst reflect.Value) { } } +// Validate performs struct validation using go-playground/validator tags. +// +// It collects all validation errors and returns them as a single error. +// Field names in errors are prefixed with their namespace (e.g., "User.Email"). func ValidateWithFieldTags(s any) gperr.Error { var errs gperr.Builder err := validate.Struct(s) @@ -257,10 +262,11 @@ func initTypeKeyFieldIndexesMap(t reflect.Type) typeInfo { } } -// MapUnmarshalValidate takes a SerializedObject and a target value, and assigns the values in the SerializedObject to the target value. -// MapUnmarshalValidate ignores case differences between the field names in the SerializedObject and the target. +// MapUnmarshalValidate takes a SerializedObject and a target value, +// and assigns the values in the SerializedObject to the target value. +// +// It ignores case differences between the field names in the SerializedObject and the target. // -// The target value must be a struct or a map[string]any. // If the target value is a struct , and implements the MapUnmarshaller interface, // the UnmarshalMap method will be called. // @@ -309,7 +315,7 @@ func mapUnmarshalValidate(src SerializedObject, dstV reflect.Value, checkValidat info := getTypeInfo(dstT) for k, v := range src { if field, ok := info.getField(dstV, k); ok { - err := Convert(reflect.ValueOf(v), field, !info.hasValidateTag) + err := Convert(reflect.ValueOf(v), field, checkValidateTag) if err != nil { errs.Add(err.Subject(k)) } @@ -455,6 +461,13 @@ func Convert(src reflect.Value, dst reflect.Value, checkValidateTag bool) gperr. return ErrUnsupportedConversion.Subjectf("%s to %s", srcT, dstT) } +// ConvertSlice converts a source slice to a destination slice. +// +// - Elements are converted one by one using the Convert function. +// - Validation is performed on each element if checkValidateTag is true. +// - The destination slice is initialized with the source length. +// - On error, the destination slice is truncated to the number of +// successfully converted elements. func ConvertSlice(src reflect.Value, dst reflect.Value, checkValidateTag bool) gperr.Error { if dst.Kind() == reflect.Pointer { if dst.IsNil() && !dst.CanSet() { @@ -507,6 +520,12 @@ func ConvertSlice(src reflect.Value, dst reflect.Value, checkValidateTag bool) g return nil } +// ConvertString converts a string value to the destination reflect.Value. +// - It handles various types including numeric types, booleans, time.Duration, +// slices (comma-separated or YAML), maps, and structs (YAML). +// - If the destination implements the Parser interface, it is used for conversion. +// - Returns true if conversion was handled (even with error), false if +// conversion is unsupported. func ConvertString(src string, dst reflect.Value) (convertible bool, convErr gperr.Error) { convertible = true dstT := dst.Type() @@ -618,48 +637,80 @@ func substituteEnv(data []byte) ([]byte, gperr.Error) { return data, nil } -func UnmarshalValidateYAML[T any](data []byte, target *T) gperr.Error { +type ( + marshalFunc func(src any) ([]byte, error) + unmarshalFunc func(data []byte, target any) error + newDecoderFunc func(r io.Reader) interface { + Decode(v any) error + } + interceptFunc func(m map[string]any) gperr.Error +) + +// UnmarshalValidate unmarshals data into a map, applies optional intercept +// functions, and validates the result against the target struct using field tags. +// - Environment variables in the data are substituted using ${VAR} syntax. +// - The unmarshaler function converts data to a map[string]any. +// - Intercept functions can modify or validate the map before unmarshaling. +func UnmarshalValidate[T any](data []byte, target *T, unmarshaler unmarshalFunc, interceptFns ...interceptFunc) gperr.Error { data, err := substituteEnv(data) if err != nil { return err } m := make(map[string]any) - if err := yaml.Unmarshal(data, &m); err != nil { + if err := unmarshaler(data, &m); err != nil { return gperr.Wrap(err) } + for _, intercept := range interceptFns { + if err := intercept(m); err != nil { + return err + } + } return MapUnmarshalValidate(m, target) } -func UnmarshalValidateYAMLIntercept[T any](data []byte, target *T, intercept func(m map[string]any) gperr.Error) gperr.Error { +// UnmarshalValidateReader reads from an io.Reader, unmarshals to a map, +// - Applies optional intercept functions, and validates against the target struct. +// - Environment variables are substituted during reading using ${VAR} syntax. +// - The newDecoder function creates a decoder for the reader (e.g., +// json.NewDecoder). +func UnmarshalValidateReader[T any](reader io.Reader, target *T, newDecoder newDecoderFunc, interceptFns ...interceptFunc) gperr.Error { + m := make(map[string]any) + if err := newDecoder(NewSubstituteEnvReader(reader)).Decode(&m); err != nil { + return gperr.Wrap(err) + } + for _, intercept := range interceptFns { + if err := intercept(m); err != nil { + return err + } + } + return MapUnmarshalValidate(m, target) +} + +// UnmarshalValidateXSync unmarshals data into an xsync.Map[string, V]. +// - Environment variables in the data are substituted using ${VAR} syntax. +// - The unmarshaler function converts data to a map[string]any. +// - Intercept functions can modify or validate the map before unmarshaling. +// - Returns a thread-safe concurrent map with the unmarshaled values. +func UnmarshalValidateXSync[V any](data []byte, unmarshaler unmarshalFunc, interceptFns ...interceptFunc) (*xsync.Map[string, V], gperr.Error) { data, err := substituteEnv(data) if err != nil { - return err + return nil, err } m := make(map[string]any) - if err := yaml.Unmarshal(data, &m); err != nil { - return gperr.Wrap(err) + if err := unmarshaler(data, &m); err != nil { + return nil, gperr.Wrap(err) } - if err := intercept(m); err != nil { - return err - } - return MapUnmarshalValidate(m, target) -} - -func UnmarshalValidateYAMLXSync[V any](data []byte) (_ *xsync.Map[string, V], err gperr.Error) { - data, err = substituteEnv(data) - if err != nil { - return + for _, intercept := range interceptFns { + if err := intercept(m); err != nil { + return nil, err + } } - m := make(map[string]any) - if err = gperr.Wrap(yaml.Unmarshal(data, &m)); err != nil { - return - } m2 := make(map[string]V, len(m)) if err = MapUnmarshalValidate(m, m2); err != nil { - return + return nil, err } ret := xsync.NewMap[string, V](xsync.WithPresize(len(m))) for k, v := range m2 { @@ -668,26 +719,27 @@ func UnmarshalValidateYAMLXSync[V any](data []byte) (_ *xsync.Map[string, V], er return ret, nil } -func loadSerialized[T any](path string, dst *T, deserialize func(data []byte, dst any) error) error { - data, err := os.ReadFile(path) - if err != nil { - return err - } - return deserialize(data, dst) -} - -func SaveJSON[T any](path string, src *T, perm os.FileMode) error { - data, err := sonic.Marshal(src) +// SaveFile marshals a value to bytes and writes it to a file. +// - The marshaler function converts the value to bytes. +// - The file is written with the specified permissions. +func SaveFile[T any](path string, src *T, perm os.FileMode, marshaler marshalFunc) error { + data, err := marshaler(src) if err != nil { return err } return os.WriteFile(path, data, perm) } -func LoadJSONIfExist[T any](path string, dst *T) error { - err := loadSerialized(path, dst, sonic.Unmarshal) - if os.IsNotExist(err) { - return nil +// LoadFileIfExist reads a file and unmarshals its contents to a value. +// - The unmarshaler function converts the bytes to a value. +// - If the file does not exist, nil is returned and dst remains unchanged. +func LoadFileIfExist[T any](path string, dst *T, unmarshaler unmarshalFunc) error { + data, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err } - return err + return unmarshaler(data, dst) } diff --git a/internal/serialization/serialization_test.go b/internal/serialization/serialization_test.go index 1b0a6c27..9d2ae6d5 100644 --- a/internal/serialization/serialization_test.go +++ b/internal/serialization/serialization_test.go @@ -6,6 +6,7 @@ import ( "strconv" "testing" + "github.com/goccy/go-yaml" "github.com/stretchr/testify/require" expect "github.com/yusing/goutils/testing" ) @@ -303,6 +304,6 @@ autocert: } `yaml:"options"` } `yaml:"autocert"` } - require.NoError(t, UnmarshalValidateYAML(data, &cfg)) + require.NoError(t, UnmarshalValidate(data, &cfg, yaml.Unmarshal)) require.Equal(t, "test", cfg.Autocert.Options.AuthToken) } diff --git a/internal/serialization/validation.go b/internal/serialization/validation.go index c0040dc2..bebd9370 100644 --- a/internal/serialization/validation.go +++ b/internal/serialization/validation.go @@ -29,17 +29,19 @@ type CustomValidator interface { var validatorType = reflect.TypeFor[CustomValidator]() func ValidateWithCustomValidator(v reflect.Value) gperr.Error { + vt := v.Type() if v.Kind() == reflect.Pointer { - if v.IsNil() { - // return nil - return validateWithValidator(reflect.New(v.Type().Elem())) - } - if v.Type().Implements(validatorType) { + elemType := vt.Elem() + if vt.Implements(validatorType) { + if v.IsNil() { + return reflect.New(elemType).Interface().(CustomValidator).Validate() + } return v.Interface().(CustomValidator).Validate() } - return validateWithValidator(v.Elem()) + if elemType.Implements(validatorType) { + return v.Elem().Interface().(CustomValidator).Validate() + } } else { - vt := v.Type() if vt.PkgPath() != "" { // not a builtin type // prioritize pointer method if v.CanAddr() { @@ -56,10 +58,3 @@ func ValidateWithCustomValidator(v reflect.Value) gperr.Error { } return nil } - -func validateWithValidator(v reflect.Value) gperr.Error { - if v.Type().Implements(validatorType) { - return v.Interface().(CustomValidator).Validate() - } - return nil -} diff --git a/internal/types/docker_provider_config_test.go b/internal/types/docker_provider_config_test.go index 093c7b32..bae5b49d 100644 --- a/internal/types/docker_provider_config_test.go +++ b/internal/types/docker_provider_config_test.go @@ -3,6 +3,7 @@ package types import ( "testing" + "github.com/goccy/go-yaml" "github.com/stretchr/testify/assert" "github.com/yusing/godoxy/internal/serialization" ) @@ -10,14 +11,14 @@ import ( func TestDockerProviderConfigUnmarshalMap(t *testing.T) { t.Run("string", func(t *testing.T) { var cfg map[string]*DockerProviderConfig - err := serialization.UnmarshalValidateYAML([]byte("test: http://localhost:2375"), &cfg) + err := serialization.UnmarshalValidate([]byte("test: http://localhost:2375"), &cfg, yaml.Unmarshal) assert.NoError(t, err) assert.Equal(t, &DockerProviderConfig{URL: "http://localhost:2375"}, cfg["test"]) }) t.Run("detailed", func(t *testing.T) { var cfg map[string]*DockerProviderConfig - err := serialization.UnmarshalValidateYAML([]byte(` + err := serialization.UnmarshalValidate([]byte(` test: scheme: http host: localhost @@ -25,7 +26,7 @@ test: tls: ca_file: /etc/ssl/ca.crt cert_file: /etc/ssl/cert.crt - key_file: /etc/ssl/key.crt`), &cfg) + key_file: /etc/ssl/key.crt`), &cfg, yaml.Unmarshal) assert.NoError(t, err) assert.Equal(t, &DockerProviderConfig{URL: "http://localhost:2375", TLS: &DockerTLSConfig{CAFile: "/etc/ssl/ca.crt", CertFile: "/etc/ssl/cert.crt", KeyFile: "/etc/ssl/key.crt"}}, cfg["test"]) }) @@ -131,7 +132,7 @@ func TestDockerProviderConfigValidation(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { var cfg map[string]*DockerProviderConfig - err := serialization.UnmarshalValidateYAML([]byte(test.yamlStr), &cfg) + err := serialization.UnmarshalValidate([]byte(test.yamlStr), &cfg, yaml.Unmarshal) if test.wantErr { assert.Error(t, err) } else {