refactor(serialization): generalize unmarshal/load functions with pluggable format handlers

Replace YAML-specific functions with generic ones accepting unmarshaler/marshaler
function parameters. This enables future support for JSON and other formats
while maintaining current YAML behavior.

- UnmarshalValidateYAML -> UnmarshalValidate(unmarshalFunc)
- UnmarshalValidateYAMLXSync -> UnmarshalValidateXSync(unmarshalFunc)
- SaveJSON -> SaveFile(marshalFunc)
- LoadJSONIfExist -> LoadFileIfExist(unmarshalFunc)
- Add UnmarshalValidateReader for reader-based decoding

Testing: all 12 staged test files updated to use new API

internal/autocert/config_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"testing"
 
+	"github.com/goccy/go-yaml"
 	"github.com/stretchr/testify/require"
 	"github.com/yusing/godoxy/internal/autocert"
 	"github.com/yusing/godoxy/internal/dnsproviders"
@@ -25,9 +26,9 @@ func TestEABConfigRequired(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			yaml := fmt.Appendf(nil, "eab_kid: %s\neab_hmac: %s", test.cfg.EABKid, test.cfg.EABHmac)
+			yamlCfg := fmt.Appendf(nil, "eab_kid: %s\neab_hmac: %s", test.cfg.EABKid, test.cfg.EABHmac)
 			cfg := autocert.Config{}
-			err := serialization.UnmarshalValidateYAML(yaml, &cfg)
+			err := serialization.UnmarshalValidate(yamlCfg, &cfg, yaml.Unmarshal)
 			if (err != nil) != test.wantErr {
 				t.Errorf("Validate() error = %v, wantErr %v", err, test.wantErr)
 			}

internal/autocert/provider_test/multi_cert_test.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"testing"
 
+	"github.com/goccy/go-yaml"
 	"github.com/stretchr/testify/require"
 	"github.com/yusing/godoxy/internal/autocert"
 	"github.com/yusing/godoxy/internal/serialization"
@@ -41,7 +42,7 @@ func TestMultipleCertificatesLifecycle(t *testing.T) {
 	cfg.HTTPClient = acmeServer.httpClient()
 
 	/* unmarshal yaml config with multiple certs */
-	err := error(serialization.UnmarshalValidateYAML(yamlConfig, &cfg))
+	err := error(serialization.UnmarshalValidate(yamlConfig, &cfg, yaml.Unmarshal))
 	require.NoError(t, err)
 	require.Equal(t, []string{"main.example.com"}, cfg.Domains)
 	require.Len(t, cfg.Extra, 2)

internal/autocert/setup_test.go

@@ -3,6 +3,7 @@ package autocert_test
 import (
 	"testing"
 
+	"github.com/goccy/go-yaml"
 	"github.com/stretchr/testify/require"
 	"github.com/yusing/godoxy/internal/autocert"
 	"github.com/yusing/godoxy/internal/dnsproviders"
@@ -42,7 +43,7 @@ extra:
 `
 
 	var cfg autocert.Config
-	err := error(serialization.UnmarshalValidateYAML([]byte(cfgYAML), &cfg))
+	err := error(serialization.UnmarshalValidate([]byte(cfgYAML), &cfg, yaml.Unmarshal))
 	require.NoError(t, err)
 
 	// Test: extra[0] inherits all fields from main except CertPath and KeyPath.

internal/config/state.go

@@ -103,7 +103,7 @@ func (state *state) InitFromFile(filename string) error {
 }
 
 func (state *state) Init(data []byte) error {
-	err := serialization.UnmarshalValidateYAML(data, &state.Config)
+	err := serialization.UnmarshalValidate(data, &state.Config, yaml.Unmarshal)
 	if err != nil {
 		return err
 	}

internal/config/types/config.go

@@ -4,6 +4,7 @@ import (
 	"regexp"
 
 	"github.com/go-playground/validator/v10"
+	"github.com/goccy/go-yaml"
 	"github.com/yusing/godoxy/agent/pkg/agent"
 	"github.com/yusing/godoxy/internal/acl"
 	"github.com/yusing/godoxy/internal/autocert"
@@ -43,7 +44,7 @@ type (
 
 func Validate(data []byte) gperr.Error {
 	var model Config
-	return serialization.UnmarshalValidateYAML(data, &model)
+	return serialization.UnmarshalValidate(data, &model, yaml.Unmarshal)
 }
 
 func DefaultConfig() Config {

internal/homepage/icons/list/list_icons.go

@@ -55,20 +55,20 @@ func init() {
 
 func InitCache() {
 	m := make(IconMap)
-	err := serialization.LoadJSONIfExist(common.IconListCachePath, &m)
+	err := serialization.LoadFileIfExist(common.IconListCachePath, &m, sonic.Unmarshal)
 	if err != nil {
 		// backward compatible
 		oldFormat := struct {
 			Icons      IconMap
 			LastUpdate time.Time
 		}{}
-		err = serialization.LoadJSONIfExist(common.IconListCachePath, &oldFormat)
+		err = serialization.LoadFileIfExist(common.IconListCachePath, &oldFormat, sonic.Unmarshal)
 		if err != nil {
 			log.Error().Err(err).Msg("failed to load icons")
 		} else {
 			m = oldFormat.Icons
 			// store it to disk immediately
-			_ = serialization.SaveJSON(common.IconListCachePath, &m, 0o644)
+			_ = serialization.SaveFile(common.IconListCachePath, &m, 0o644, sonic.Marshal)
 		}
 	} else if len(m) > 0 {
 		log.Info().
@@ -84,7 +84,7 @@ func InitCache() {
 
 	task.OnProgramExit("save_icons_cache", func() {
 		icons := iconsCache.Load()
-		_ = serialization.SaveJSON(common.IconListCachePath, &icons, 0o644)
+		_ = serialization.SaveFile(common.IconListCachePath, &icons, 0o644, sonic.Marshal)
 	})
 
 	go backgroundUpdateIcons()
@@ -105,7 +105,7 @@ func backgroundUpdateIcons() {
 				// swap old cache with new cache
 				iconsCache.Store(newCache)
 				// save it to disk
-				err := serialization.SaveJSON(common.IconListCachePath, &newCache, 0o644)
+				err := serialization.SaveFile(common.IconListCachePath, &newCache, 0o644, sonic.Marshal)
 				if err != nil {
 					log.Warn().Err(err).Msg("failed to save icons")
 				}

internal/jsonstore/jsonstore.go

@@ -83,7 +83,8 @@ func loadNS[T store](ns namespace) T {
 func save() error {
 	errs := gperr.NewBuilder("failed to save data stores")
 	for ns, store := range stores {
-		if err := serialization.SaveJSON(filepath.Join(storesPath, string(ns)+".json"), &store, 0o644); err != nil {
+		path := filepath.Join(storesPath, string(ns)+".json")
+		if err := serialization.SaveFile(path, &store, 0o644, sonic.Marshal); err != nil {
 			errs.Add(err)
 		}
 	}

internal/proxmox/validation_test.go

@@ -3,6 +3,7 @@ package proxmox
 import (
 	"testing"
 
+	"github.com/goccy/go-yaml"
 	"github.com/stretchr/testify/require"
 	"github.com/yusing/godoxy/internal/serialization"
 )
@@ -43,7 +44,7 @@ func TestValidateCommandArgs(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			var cfg NodeConfig
-			err := serialization.UnmarshalValidateYAML([]byte(tt.yamlCfg), &cfg)
+			err := serialization.UnmarshalValidate([]byte(tt.yamlCfg), &cfg, yaml.Unmarshal)
 			if tt.wantErr {
 				require.Error(t, err)
 				require.ErrorContains(t, err, "input contains invalid characters")

internal/route/provider/file.go

@@ -5,6 +5,7 @@ import (
 	"path"
 	"strings"
 
+	"github.com/goccy/go-yaml"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 	"github.com/yusing/godoxy/internal/common"
@@ -43,7 +44,7 @@ func removeXPrefix(m map[string]any) gperr.Error {
 }
 
 func validate(data []byte) (routes route.Routes, err gperr.Error) {
-	err = serialization.UnmarshalValidateYAMLIntercept(data, &routes, removeXPrefix)
+	err = serialization.UnmarshalValidate(data, &routes, yaml.Unmarshal, removeXPrefix)
 	return routes, err
 }
 

internal/serialization/serialization.go

@@ -2,6 +2,7 @@ package serialization
 
 import (
 	"errors"
+	"io"
 	"os"
 	"reflect"
 	"regexp"
@@ -85,6 +86,10 @@ func initPtr(dst reflect.Value) {
 	}
 }
 
+// Validate performs struct validation using go-playground/validator tags.
+//
+// It collects all validation errors and returns them as a single error.
+// Field names in errors are prefixed with their namespace (e.g., "User.Email").
 func ValidateWithFieldTags(s any) gperr.Error {
 	var errs gperr.Builder
 	err := validate.Struct(s)
@@ -257,10 +262,11 @@ func initTypeKeyFieldIndexesMap(t reflect.Type) typeInfo {
 	}
 }
 
-// MapUnmarshalValidate takes a SerializedObject and a target value, and assigns the values in the SerializedObject to the target value.
-// MapUnmarshalValidate ignores case differences between the field names in the SerializedObject and the target.
+// MapUnmarshalValidate takes a SerializedObject and a target value,
+// and assigns the values in the SerializedObject to the target value.
+//
+// It ignores case differences between the field names in the SerializedObject and the target.
 //
-// The target value must be a struct or a map[string]any.
 // If the target value is a struct , and implements the MapUnmarshaller interface,
 // the UnmarshalMap method will be called.
 //
@@ -455,6 +461,13 @@ func Convert(src reflect.Value, dst reflect.Value, checkValidateTag bool) gperr.
 	return ErrUnsupportedConversion.Subjectf("%s to %s", srcT, dstT)
 }
 
+// ConvertSlice converts a source slice to a destination slice.
+//
+//   - Elements are converted one by one using the Convert function.
+//   - Validation is performed on each element if checkValidateTag is true.
+//   - The destination slice is initialized with the source length.
+//   - On error, the destination slice is truncated to the number of
+//     successfully converted elements.
 func ConvertSlice(src reflect.Value, dst reflect.Value, checkValidateTag bool) gperr.Error {
 	if dst.Kind() == reflect.Pointer {
 		if dst.IsNil() && !dst.CanSet() {
@@ -507,6 +520,12 @@ func ConvertSlice(src reflect.Value, dst reflect.Value, checkValidateTag bool) g
 	return nil
 }
 
+// ConvertString converts a string value to the destination reflect.Value.
+//   - It handles various types including numeric types, booleans, time.Duration,
+//     slices (comma-separated or YAML), maps, and structs (YAML).
+//   - If the destination implements the Parser interface, it is used for conversion.
+//   - Returns true if conversion was handled (even with error), false if
+//     conversion is unsupported.
 func ConvertString(src string, dst reflect.Value) (convertible bool, convErr gperr.Error) {
 	convertible = true
 	dstT := dst.Type()
@@ -618,48 +637,80 @@ func substituteEnv(data []byte) ([]byte, gperr.Error) {
 	return data, nil
 }
 
-func UnmarshalValidateYAML[T any](data []byte, target *T) gperr.Error {
+type (
+	marshalFunc    func(src any) ([]byte, error)
+	unmarshalFunc  func(data []byte, target any) error
+	newDecoderFunc func(r io.Reader) interface {
+		Decode(v any) error
+	}
+	interceptFunc func(m map[string]any) gperr.Error
+)
+
+// UnmarshalValidate unmarshals data into a map, applies optional intercept
+// functions, and validates the result against the target struct using field tags.
+//   - Environment variables in the data are substituted using ${VAR} syntax.
+//   - The unmarshaler function converts data to a map[string]any.
+//   - Intercept functions can modify or validate the map before unmarshaling.
+func UnmarshalValidate[T any](data []byte, target *T, unmarshaler unmarshalFunc, interceptFns ...interceptFunc) gperr.Error {
 	data, err := substituteEnv(data)
 	if err != nil {
 		return err
 	}
 
 	m := make(map[string]any)
-	if err := yaml.Unmarshal(data, &m); err != nil {
+	if err := unmarshaler(data, &m); err != nil {
 		return gperr.Wrap(err)
 	}
+	for _, intercept := range interceptFns {
+		if err := intercept(m); err != nil {
+			return err
+		}
+	}
 	return MapUnmarshalValidate(m, target)
 }
 
-func UnmarshalValidateYAMLIntercept[T any](data []byte, target *T, intercept func(m map[string]any) gperr.Error) gperr.Error {
+// UnmarshalValidateReader reads from an io.Reader, unmarshals to a map,
+//   - Applies optional intercept functions, and validates against the target struct.
+//   - Environment variables are substituted during reading using ${VAR} syntax.
+//   - The newDecoder function creates a decoder for the reader (e.g.,
+//     json.NewDecoder).
+func UnmarshalValidateReader[T any](reader io.Reader, target *T, newDecoder newDecoderFunc, interceptFns ...interceptFunc) gperr.Error {
+	m := make(map[string]any)
+	if err := newDecoder(NewSubstituteEnvReader(reader)).Decode(&m); err != nil {
+		return gperr.Wrap(err)
+	}
+	for _, intercept := range interceptFns {
+		if err := intercept(m); err != nil {
+			return err
+		}
+	}
+	return MapUnmarshalValidate(m, target)
+}
+
+// UnmarshalValidateXSync unmarshals data into an xsync.Map[string, V].
+//   - Environment variables in the data are substituted using ${VAR} syntax.
+//   - The unmarshaler function converts data to a map[string]any.
+//   - Intercept functions can modify or validate the map before unmarshaling.
+//   - Returns a thread-safe concurrent map with the unmarshaled values.
+func UnmarshalValidateXSync[V any](data []byte, unmarshaler unmarshalFunc, interceptFns ...interceptFunc) (*xsync.Map[string, V], gperr.Error) {
 	data, err := substituteEnv(data)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	m := make(map[string]any)
-	if err := yaml.Unmarshal(data, &m); err != nil {
-		return gperr.Wrap(err)
+	if err := unmarshaler(data, &m); err != nil {
+		return nil, gperr.Wrap(err)
 	}
-	if err := intercept(m); err != nil {
-		return err
-	}
-	return MapUnmarshalValidate(m, target)
-}
-
-func UnmarshalValidateYAMLXSync[V any](data []byte) (_ *xsync.Map[string, V], err gperr.Error) {
-	data, err = substituteEnv(data)
-	if err != nil {
-		return
+	for _, intercept := range interceptFns {
+		if err := intercept(m); err != nil {
+			return nil, err
+		}
 	}
 
-	m := make(map[string]any)
-	if err = gperr.Wrap(yaml.Unmarshal(data, &m)); err != nil {
-		return
-	}
 	m2 := make(map[string]V, len(m))
 	if err = MapUnmarshalValidate(m, m2); err != nil {
-		return
+		return nil, err
 	}
 	ret := xsync.NewMap[string, V](xsync.WithPresize(len(m)))
 	for k, v := range m2 {
@@ -668,26 +719,27 @@ func UnmarshalValidateYAMLXSync[V any](data []byte) (_ *xsync.Map[string, V], er
 	return ret, nil
 }
 
-func loadSerialized[T any](path string, dst *T, deserialize func(data []byte, dst any) error) error {
-	data, err := os.ReadFile(path)
-	if err != nil {
-		return err
-	}
-	return deserialize(data, dst)
-}
-
-func SaveJSON[T any](path string, src *T, perm os.FileMode) error {
-	data, err := sonic.Marshal(src)
+// SaveFile marshals a value to bytes and writes it to a file.
+//   - The marshaler function converts the value to bytes.
+//   - The file is written with the specified permissions.
+func SaveFile[T any](path string, src *T, perm os.FileMode, marshaler marshalFunc) error {
+	data, err := marshaler(src)
 	if err != nil {
 		return err
 	}
 	return os.WriteFile(path, data, perm)
 }
 
-func LoadJSONIfExist[T any](path string, dst *T) error {
-	err := loadSerialized(path, dst, sonic.Unmarshal)
-	if os.IsNotExist(err) {
-		return nil
+// LoadFileIfExist reads a file and unmarshals its contents to a value.
+//   - The unmarshaler function converts the bytes to a value.
+//   - If the file does not exist, nil is returned and dst remains unchanged.
+func LoadFileIfExist[T any](path string, dst *T, unmarshaler unmarshalFunc) error {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return err
 	}
-	return err
+	return unmarshaler(data, dst)
 }

internal/serialization/serialization_test.go

@@ -6,6 +6,7 @@ import (
 	"strconv"
 	"testing"
 
+	"github.com/goccy/go-yaml"
 	"github.com/stretchr/testify/require"
 	expect "github.com/yusing/goutils/testing"
 )
@@ -303,6 +304,6 @@ autocert:
 			} `yaml:"options"`
 		} `yaml:"autocert"`
 	}
-	require.NoError(t, UnmarshalValidateYAML(data, &cfg))
+	require.NoError(t, UnmarshalValidate(data, &cfg, yaml.Unmarshal))
 	require.Equal(t, "test", cfg.Autocert.Options.AuthToken)
 }

internal/types/docker_provider_config_test.go

@@ -3,6 +3,7 @@ package types
 import (
 	"testing"
 
+	"github.com/goccy/go-yaml"
 	"github.com/stretchr/testify/assert"
 	"github.com/yusing/godoxy/internal/serialization"
 )
@@ -10,14 +11,14 @@ import (
 func TestDockerProviderConfigUnmarshalMap(t *testing.T) {
 	t.Run("string", func(t *testing.T) {
 		var cfg map[string]*DockerProviderConfig
-		err := serialization.UnmarshalValidateYAML([]byte("test: http://localhost:2375"), &cfg)
+		err := serialization.UnmarshalValidate([]byte("test: http://localhost:2375"), &cfg, yaml.Unmarshal)
 		assert.NoError(t, err)
 		assert.Equal(t, &DockerProviderConfig{URL: "http://localhost:2375"}, cfg["test"])
 	})
 
 	t.Run("detailed", func(t *testing.T) {
 		var cfg map[string]*DockerProviderConfig
-		err := serialization.UnmarshalValidateYAML([]byte(`
+		err := serialization.UnmarshalValidate([]byte(`
 test:
   scheme: http
   host: localhost
@@ -25,7 +26,7 @@ test:
   tls:
     ca_file: /etc/ssl/ca.crt
     cert_file: /etc/ssl/cert.crt
-    key_file: /etc/ssl/key.crt`), &cfg)
+    key_file: /etc/ssl/key.crt`), &cfg, yaml.Unmarshal)
 		assert.NoError(t, err)
 		assert.Equal(t, &DockerProviderConfig{URL: "http://localhost:2375", TLS: &DockerTLSConfig{CAFile: "/etc/ssl/ca.crt", CertFile: "/etc/ssl/cert.crt", KeyFile: "/etc/ssl/key.crt"}}, cfg["test"])
 	})
@@ -131,7 +132,7 @@ func TestDockerProviderConfigValidation(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			var cfg map[string]*DockerProviderConfig
-			err := serialization.UnmarshalValidateYAML([]byte(test.yamlStr), &cfg)
+			err := serialization.UnmarshalValidate([]byte(test.yamlStr), &cfg, yaml.Unmarshal)
 			if test.wantErr {
 				assert.Error(t, err)
 			} else {