From 4852efcf9c3d8dd4abd4f6f1a646981ba2b5480e Mon Sep 17 00:00:00 2001 From: Yuzerion Date: Thu, 2 Oct 2025 20:30:31 +0800 Subject: [PATCH] feat: faster serialization (#157) * refactor: improve deserialization performance * refactor(serialization): simplify string conversion logic in Convert function * fix(serialization): default value lookup * refactor: add comment about concurrency in RegisterDefaultValueFactory --------- Co-authored-by: yusing --- agent/go.mod | 1 + agent/go.sum | 2 + go.mod | 1 + go.sum | 2 + internal/dnsproviders/go.mod | 1 + internal/dnsproviders/go.sum | 2 + internal/logging/accesslog/fields.go | 2 +- internal/serialization/serialization.go | 392 ++++++++++-------- .../serialization_benchmark_test.go | 95 +++++ internal/serialization/serialization_test.go | 49 --- 10 files changed, 326 insertions(+), 221 deletions(-) create mode 100644 internal/serialization/serialization_benchmark_test.go diff --git a/agent/go.mod b/agent/go.mod index b9a3fc66..7c464ba9 100644 --- a/agent/go.mod +++ b/agent/go.mod @@ -90,6 +90,7 @@ require ( github.com/vincent-petithory/dataurl v1.0.0 // indirect github.com/yusing/ds v0.2.0 // indirect github.com/yusing/godoxy/internal/utils v0.1.0 // indirect + github.com/yusing/gointernals v0.1.16 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect diff --git a/agent/go.sum b/agent/go.sum index aa397377..42f039a8 100644 --- a/agent/go.sum +++ b/agent/go.sum @@ -208,6 +208,8 @@ github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusing/ds v0.2.0 h1:lPhDU5eA2uvquVrBrzLCrQXRJJgSXlUYA53TbuK2sQY= github.com/yusing/ds v0.2.0/go.mod h1:XhKV4l7cZwBbbl7lRzNC9zX27zvCM0frIwiuD40ULRk= +github.com/yusing/gointernals v0.1.16 h1:GrhZZdxzA+jojLEqankctJrOuAYDb7kY1C93S1pVR34= +github.com/yusing/gointernals v0.1.16/go.mod h1:B/0FVXt4WPmgzVy3ynzkqKi+BSGaJVmwCJBRXYapo34= github.com/yusing/goutils v0.4.1 h1:80uFNxXfm4zXMYDku0rWMLyqEiXO0UOMFOaUC4b/6fI= github.com/yusing/goutils v0.4.1/go.mod h1:xsoLWLtIiu7k+9Bn6azERDs5o3Djb3b2/DW1htHrOjg= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= diff --git a/go.mod b/go.mod index f8546b3a..f1cc86ec 100644 --- a/go.mod +++ b/go.mod @@ -212,6 +212,7 @@ require ( require ( github.com/bytedance/sonic v1.14.1 github.com/shirou/gopsutil/v4 v4.25.8 + github.com/yusing/gointernals v0.1.16 ) require ( diff --git a/go.sum b/go.sum index b5e06267..85292d6d 100644 --- a/go.sum +++ b/go.sum @@ -1648,6 +1648,8 @@ github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusing/ds v0.2.0 h1:lPhDU5eA2uvquVrBrzLCrQXRJJgSXlUYA53TbuK2sQY= github.com/yusing/ds v0.2.0/go.mod h1:XhKV4l7cZwBbbl7lRzNC9zX27zvCM0frIwiuD40ULRk= +github.com/yusing/gointernals v0.1.16 h1:GrhZZdxzA+jojLEqankctJrOuAYDb7kY1C93S1pVR34= +github.com/yusing/gointernals v0.1.16/go.mod h1:B/0FVXt4WPmgzVy3ynzkqKi+BSGaJVmwCJBRXYapo34= github.com/yusing/goutils v0.4.1 h1:80uFNxXfm4zXMYDku0rWMLyqEiXO0UOMFOaUC4b/6fI= github.com/yusing/goutils v0.4.1/go.mod h1:xsoLWLtIiu7k+9Bn6azERDs5o3Djb3b2/DW1htHrOjg= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= diff --git a/internal/dnsproviders/go.mod b/internal/dnsproviders/go.mod index d492e40b..41f154f7 100644 --- a/internal/dnsproviders/go.mod +++ b/internal/dnsproviders/go.mod @@ -154,6 +154,7 @@ require ( github.com/vultr/govultr/v3 v3.24.0 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect github.com/yusing/godoxy/internal/utils v0.1.0 // indirect + github.com/yusing/gointernals v0.1.16 // indirect github.com/yusing/goutils v0.4.2 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect diff --git a/internal/dnsproviders/go.sum b/internal/dnsproviders/go.sum index 60a80cb1..2b98584c 100644 --- a/internal/dnsproviders/go.sum +++ b/internal/dnsproviders/go.sum @@ -1525,6 +1525,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yusing/gointernals v0.1.16 h1:GrhZZdxzA+jojLEqankctJrOuAYDb7kY1C93S1pVR34= +github.com/yusing/gointernals v0.1.16/go.mod h1:B/0FVXt4WPmgzVy3ynzkqKi+BSGaJVmwCJBRXYapo34= github.com/yusing/goutils v0.4.2 h1:JRC14SUJ54nTIHxi8Z66cHJSMlifsyKO1z8RZCHBkUI= github.com/yusing/goutils v0.4.2/go.mod h1:67EfLJlq9WQfP/uRxm0dD+WFRsxTK0MzZG3jXOI3wX8= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= diff --git a/internal/logging/accesslog/fields.go b/internal/logging/accesslog/fields.go index 7b5e013c..1c15ddff 100644 --- a/internal/logging/accesslog/fields.go +++ b/internal/logging/accesslog/fields.go @@ -10,7 +10,7 @@ import ( type ( FieldConfig struct { - Default FieldMode `json:"default" validate:"oneof=keep drop redact"` + Default FieldMode `json:"default" validate:"omitempty,oneof=keep drop redact"` Config map[string]FieldMode `json:"config" validate:"dive,oneof=keep drop redact"` } FieldMode string diff --git a/internal/serialization/serialization.go b/internal/serialization/serialization.go index bd93b1df..4367bd1c 100644 --- a/internal/serialization/serialization.go +++ b/internal/serialization/serialization.go @@ -14,6 +14,7 @@ import ( "github.com/goccy/go-yaml" "github.com/puzpuzpuz/xsync/v4" "github.com/yusing/godoxy/internal/utils" + gi "github.com/yusing/gointernals" gperr "github.com/yusing/goutils/errs" strutils "github.com/yusing/goutils/strings" ) @@ -41,53 +42,30 @@ var ( var mapUnmarshalerType = reflect.TypeFor[MapUnmarshaller]() -var defaultValues = xsync.NewMapOf[reflect.Type, func() any]() +var defaultValues = make(map[reflect.Type]func() any) +// RegisterDefaultValueFactory registers a factory function for a type. +// This is not concurrent safe. Intended to be used in init functions. func RegisterDefaultValueFactory[T any](factory func() *T) { t := reflect.TypeFor[T]() - if t.Kind() == reflect.Ptr { + if t.Kind() == reflect.Pointer { panic("pointer of pointer") } - if _, ok := defaultValues.Load(t); ok { + if _, ok := defaultValues[t]; ok { panic("default value for " + t.String() + " already registered") } - defaultValues.Store(t, func() any { return factory() }) + defaultValues[t] = func() any { return factory() } } -func New(t reflect.Type) reflect.Value { - if dv, ok := defaultValues.Load(t); ok { - return reflect.ValueOf(dv()) +// initPtr initialize the ptr with default value if exists, +// otherwise, initialize the ptr with zero value. +func initPtr(dst reflect.Value) { + dstT := dst.Type() + if dv, ok := defaultValues[dstT.Elem()]; ok { + dst.Set(reflect.ValueOf(dv())) + } else { + gi.ReflectInitPtr(dst) } - return reflect.New(t) -} - -func extractFields(t reflect.Type) (all, anonymous []reflect.StructField) { - for t.Kind() == reflect.Ptr { - t = t.Elem() - } - if t.Kind() != reflect.Struct { - return nil, nil - } - n := t.NumField() - fields := make([]reflect.StructField, 0, n) - for i := range n { - field := t.Field(i) - if !field.IsExported() { - continue - } - if field.Tag.Get(tagDeserialize) == "-" { - continue - } - if field.Anonymous { - f1, f2 := extractFields(field.Type) - fields = append(fields, f1...) - anonymous = append(anonymous, field) - anonymous = append(anonymous, f2...) - } else { - fields = append(fields, field) - } - } - return fields, anonymous } func ValidateWithFieldTags(s any) gperr.Error { @@ -141,33 +119,150 @@ func ValidateWithCustomValidator(v reflect.Value) gperr.Error { func dive(dst reflect.Value) (v reflect.Value, t reflect.Type, err gperr.Error) { dstT := dst.Type() for { - switch dst.Kind() { - case reflect.Pointer, reflect.Interface: - if dst.IsNil() { - if !dst.CanSet() { - err = gperr.Errorf("dive: dst is %w and is not settable", ErrNilValue) - return v, t, err - } - dst.Set(New(dstT.Elem())) - } + switch dstT.Kind() { + case reflect.Pointer: dst = dst.Elem() - dstT = dst.Type() - case reflect.Map: - if dst.IsNil() { - dst.Set(reflect.MakeMap(dstT)) - } - return dst, dstT, nil - case reflect.Slice: - if dst.IsNil() { - dst.Set(reflect.MakeSlice(dstT, 0, 0)) - } - return dst, dstT, nil + dstT = dstT.Elem() default: return dst, dstT, nil } } } +func fnv1IgnoreCaseSnake(s string) uint32 { + const ( + offset32 uint32 = 2166136261 + prime32 uint32 = 16777619 + ) + hash := offset32 + for _, r := range s { + if r == '_' { + continue + } + if r >= 'A' && r <= 'Z' { + r += 'a' - 'A' + } + hash = hash*prime32 ^ uint32(r) + } + return hash +} + +type typeInfo struct { + keyFieldIndexes map[uint32][]int + fieldNames map[string]struct{} + hasValidateTag bool +} + +func (t typeInfo) getField(v reflect.Value, k string) (reflect.Value, bool) { + hash := fnv1IgnoreCaseSnake(k) + if field, ok := t.keyFieldIndexes[hash]; ok { + return fieldByIndexWithLazyPtrInitialization(v, field), true + } + return reflect.Value{}, false +} + +func fieldByIndexWithLazyPtrInitialization(v reflect.Value, index []int) reflect.Value { + if len(index) == 1 { + return v.Field(index[0]) + } + for i, x := range index { + if i > 0 { + if v.Kind() == reflect.Pointer && v.Type().Elem().Kind() == reflect.Struct { + if v.IsNil() { + initPtr(v) + } + v = v.Elem() + } + } + v = v.Field(x) + } + return v +} + +var getTypeInfo func(t reflect.Type) typeInfo + +func init() { + m := xsync.NewMap[reflect.Type, typeInfo](xsync.WithGrowOnly(), xsync.WithPresize(100)) + getTypeInfo = func(t reflect.Type) typeInfo { + if v, ok := m.Load(t); ok { + return v + } + v := initTypeKeyFieldIndexesMap(t) + m.Store(t, v) + return v + } +} + +func initTypeKeyFieldIndexesMap(t reflect.Type) typeInfo { + hasValidateTag := false + numFields := t.NumField() + + keyFieldIndexes := make(map[uint32][]int, numFields) + fieldNames := make(map[string]struct{}, numFields) + + for i := range numFields { + field := t.Field(i) + if field.Tag.Get(tagDeserialize) == "-" || field.Tag.Get(tagJSON) == "-" { + continue + } + + if !field.IsExported() { + continue + } + if field.Anonymous { + fieldT := field.Type + if fieldT.Kind() == reflect.Pointer { + fieldT = fieldT.Elem() + } + if fieldT.Kind() != reflect.Struct { + goto notAnonymousStruct + } + typeInfo := getTypeInfo(fieldT) + for k, v := range typeInfo.keyFieldIndexes { + keyFieldIndexes[k] = append(field.Index, v...) + } + for k := range typeInfo.fieldNames { + fieldNames[k] = struct{}{} + } + hasValidateTag = hasValidateTag || typeInfo.hasValidateTag + continue + } + notAnonymousStruct: + var key string + if jsonTag, ok := field.Tag.Lookup(tagJSON); ok { + if jsonTag == "-" { + continue + } + key = jsonTag + if idxComma := strings.Index(key, ","); idxComma != -1 { + key = key[:idxComma] + } + } else { + key = field.Name + } + keyFieldIndexes[fnv1IgnoreCaseSnake(key)] = field.Index + fieldNames[key] = struct{}{} + + if !hasValidateTag { + _, hasValidateTag = field.Tag.Lookup(tagValidate) + } + + aliases, ok := field.Tag.Lookup(tagAliases) + if ok { + for alias := range strings.SplitSeq(aliases, ",") { + keyFieldIndexes[fnv1IgnoreCaseSnake(alias)] = field.Index + fieldNames[alias] = struct{}{} + } + } + } + + return typeInfo{ + keyFieldIndexes: keyFieldIndexes, + fieldNames: fieldNames, + hasValidateTag: hasValidateTag, + } +} + // MapUnmarshalValidate takes a SerializedObject and a target value, and assigns the values in the SerializedObject to the target value. // MapUnmarshalValidate ignores case differences between the field names in the SerializedObject and the target. // @@ -182,11 +277,10 @@ func dive(dst reflect.Value) (v reflect.Value, t reflect.Type, err gperr.Error) // // The function returns an error if the target value is not a struct or a map[string]any, or if there is an error during deserialization. func MapUnmarshalValidate(src SerializedObject, dst any) (err gperr.Error) { - return mapUnmarshalValidate(src, dst, true) + return mapUnmarshalValidate(src, reflect.ValueOf(dst), true) } -func mapUnmarshalValidate(src SerializedObject, dst any, checkValidateTag bool) (err gperr.Error) { - dstV := reflect.ValueOf(dst) +func mapUnmarshalValidate(src SerializedObject, dstV reflect.Value, checkValidateTag bool) (err gperr.Error) { dstT := dstV.Type() if src != nil && dstT.Implements(mapUnmarshalerType) { @@ -204,7 +298,7 @@ func mapUnmarshalValidate(src SerializedObject, dst any, checkValidateTag bool) if src == nil { if dstV.CanSet() { - dstV.Set(reflect.Zero(dstT)) + dstV.SetZero() return nil } return gperr.Errorf("deserialize: src is %w and dst is not settable", ErrNilValue) @@ -218,49 +312,18 @@ func mapUnmarshalValidate(src SerializedObject, dst any, checkValidateTag bool) switch dstV.Kind() { case reflect.Struct, reflect.Interface: - hasValidateTag := false - mapping := make(map[string]reflect.Value) - fields, anonymous := extractFields(dstT) - for _, anon := range anonymous { - if field := dstV.FieldByName(anon.Name); field.Kind() == reflect.Ptr && field.IsNil() { - field.Set(New(anon.Type.Elem())) - } - } - for _, field := range fields { - var key string - if jsonTag, ok := field.Tag.Lookup(tagJSON); ok { - if jsonTag == "-" { - continue - } - key = strutils.CommaSeperatedList(jsonTag)[0] - } else { - key = field.Name - } - key = strutils.ToLowerNoSnake(key) - mapping[key] = dstV.FieldByName(field.Name) - - if !hasValidateTag { - _, hasValidateTag = field.Tag.Lookup(tagValidate) - } - - aliases, ok := field.Tag.Lookup(tagAliases) - if ok { - for _, alias := range strutils.CommaSeperatedList(aliases) { - mapping[alias] = dstV.FieldByName(field.Name) - } - } - } + info := getTypeInfo(dstT) for k, v := range src { - if field, ok := mapping[strutils.ToLowerNoSnake(k)]; ok { - err := Convert(reflect.ValueOf(v), field, !hasValidateTag) + if field, ok := info.getField(dstV, k); ok { + err := Convert(reflect.ValueOf(v), field, !info.hasValidateTag) if err != nil { errs.Add(err.Subject(k)) } } else { - errs.Add(ErrUnknownField.Subject(k).With(gperr.DoYouMean(utils.NearestField(k, mapping)))) + errs.Add(ErrUnknownField.Subject(k).With(gperr.DoYouMean(utils.NearestField(k, info.fieldNames)))) } } - if hasValidateTag && checkValidateTag { + if info.hasValidateTag && checkValidateTag { errs.Add(ValidateWithFieldTags(dstV.Interface())) } if err := ValidateWithCustomValidator(dstV); err != nil { @@ -268,18 +331,25 @@ func mapUnmarshalValidate(src SerializedObject, dst any, checkValidateTag bool) } return errs.Error() case reflect.Map: + if dstV.IsNil() { + if !dstV.CanSet() { + return gperr.Errorf("dive: dst is %w and is not settable", ErrNilValue) + } + gi.ReflectInitMap(dstV, len(src)) + } + if dstT.Key().Kind() != reflect.String { + return gperr.Errorf("deserialize: %w for map of non string keys (map of %s)", ErrUnsupportedConversion, dstT.Elem().String()) + } + // ?: should we clear the map? for k, v := range src { - mapVT := dstT.Elem() - tmp := New(mapVT).Elem() - err := Convert(reflect.ValueOf(v), tmp, true) + elem := gi.ReflectStrMapAssign(dstV, k) + err := Convert(reflect.ValueOf(v), elem, true) if err != nil { errs.Add(err.Subject(k)) continue } - if err := ValidateWithCustomValidator(tmp.Addr()); err != nil { + if err := ValidateWithCustomValidator(elem); err != nil { errs.Add(err.Subject(k)) - } else { - dstV.SetMapIndex(reflect.ValueOf(k), tmp) } } if err := ValidateWithCustomValidator(dstV); err != nil { @@ -291,10 +361,6 @@ func mapUnmarshalValidate(src SerializedObject, dst any, checkValidateTag bool) } } -func isIntFloat(t reflect.Kind) bool { - return t >= reflect.Bool && t <= reflect.Float64 -} - // Convert attempts to convert the src to dst. // // If src is a map, it is deserialized into dst. @@ -316,8 +382,7 @@ func Convert(src reflect.Value, dst reflect.Value, checkValidateTag bool) gperr. if !dst.CanSet() { return gperr.Errorf("convert: src is %w", ErrNilValue) } - // manually set nil - dst.Set(reflect.Zero(dst.Type())) + dst.SetZero() return nil } @@ -326,10 +391,10 @@ func Convert(src reflect.Value, dst reflect.Value, checkValidateTag bool) gperr. return gperr.Errorf("convert: src is %w", ErrNilValue) } switch dst.Kind() { - case reflect.Pointer, reflect.Interface: - dst.Set(reflect.New(dst.Type().Elem())) + case reflect.Pointer: + initPtr(dst) default: - dst.Set(reflect.Zero(dst.Type())) + dst.SetZero() } return nil } @@ -344,7 +409,10 @@ func Convert(src reflect.Value, dst reflect.Value, checkValidateTag bool) gperr. if dst.Kind() == reflect.Pointer { if dst.IsNil() { - dst.Set(New(dstT.Elem())) + if !dst.CanSet() { + return ErrUnsettable.Subject(dstT.String()) + } + initPtr(dst) } dst = dst.Elem() dstT = dst.Type() @@ -353,15 +421,12 @@ func Convert(src reflect.Value, dst reflect.Value, checkValidateTag bool) gperr. srcKind := srcT.Kind() switch { - case srcT.AssignableTo(dstT): + case srcT == dstT, srcT.AssignableTo(dstT): if !dst.CanSet() { return ErrUnsettable.Subject(dstT.String()) } dst.Set(src) return nil - // case srcT.ConvertibleTo(dstT): - // dst.Set(src.Convert(dstT)) - // return nil case srcKind == reflect.String: if !dst.CanSet() { return ErrUnsettable.Subject(dstT.String()) @@ -369,25 +434,18 @@ func Convert(src reflect.Value, dst reflect.Value, checkValidateTag bool) gperr. if convertible, err := ConvertString(src.String(), dst); convertible { return err } - case isIntFloat(srcKind): - if !dst.CanSet() { - return ErrUnsettable.Subject(dstT.String()) - } - var strV string - switch { - case src.CanInt(): - strV = strconv.FormatInt(src.Int(), 10) - case srcKind == reflect.Bool: - strV = strconv.FormatBool(src.Bool()) - case src.CanUint(): - strV = strconv.FormatUint(src.Uint(), 10) - case src.CanFloat(): - strV = strconv.FormatFloat(src.Float(), 'f', -1, 64) - } - if convertible, err := ConvertString(strV, dst); convertible { + case gi.ReflectIsNumeric(src) && gi.ReflectIsNumeric(dst): + dst.Set(src.Convert(dst.Type())) + return nil + case gi.ReflectIsNumeric(src): + // try ConvertString + if convertible, err := ConvertString(gi.ReflectToStr(src), dst); convertible { return err } - case srcKind == reflect.Map: + case dstT.Kind() == reflect.String: + dst.SetString(gi.ReflectToStr(src)) + return nil + case srcKind == reflect.Map: // map to map if src.Len() == 0 { return nil } @@ -395,85 +453,77 @@ func Convert(src reflect.Value, dst reflect.Value, checkValidateTag bool) gperr. if !ok { return ErrUnsupportedConversion.Subject(dstT.String() + " to " + srcT.String()) } - return mapUnmarshalValidate(obj, dst.Addr().Interface(), checkValidateTag) - case srcKind == reflect.Slice: - if src.Len() == 0 { + return mapUnmarshalValidate(obj, dst.Addr(), checkValidateTag) + case srcKind == reflect.Slice: // slice to slice + srcLen := src.Len() + if srcLen == 0 { + dst.SetZero() return nil } if dstT.Kind() != reflect.Slice { return ErrUnsupportedConversion.Subject(dstT.String() + " to " + srcT.String()) } sliceErrs := gperr.NewBuilder() - newSlice := reflect.MakeSlice(dstT, src.Len(), src.Len()) i := 0 + gi.ReflectInitSlice(dst, srcLen, srcLen) for j, v := range src.Seq2() { - tmp := New(dstT.Elem()).Elem() - err := Convert(v, tmp, checkValidateTag) + err := Convert(v, dst.Index(i), checkValidateTag) if err != nil { sliceErrs.Add(err.Subjectf("[%d]", j)) continue } - newSlice.Index(i).Set(tmp) i++ } if err := sliceErrs.Error(); err != nil { + dst.SetLen(i) // shrink to number of elements that were successfully converted return err } - dst.Set(newSlice) return nil } - return ErrUnsupportedConversion.Subjectf("%s to %s", srcT, dstT) + + return ErrUnsupportedConversion.Subjectf("%s to %s", srcT.String(), dstT.String()) } +var parserType = reflect.TypeFor[strutils.Parser]() + func ConvertString(src string, dst reflect.Value) (convertible bool, convErr gperr.Error) { convertible = true dstT := dst.Type() - if dst.Kind() == reflect.Ptr { + if dst.Kind() == reflect.Pointer { if dst.IsNil() { - dst.Set(New(dstT.Elem())) + initPtr(dst) } dst = dst.Elem() dstT = dst.Type() } if dst.Kind() == reflect.String { dst.SetString(src) - return convertible, convErr + return true, nil } switch dstT { case reflect.TypeFor[time.Duration](): if src == "" { - dst.Set(reflect.Zero(dstT)) - return convertible, convErr + dst.SetZero() + return true, nil } d, err := time.ParseDuration(src) if err != nil { return true, gperr.Wrap(err) } - dst.Set(reflect.ValueOf(d)) - return convertible, convErr + gi.ReflectValueSet(dst, d) + return true, nil default: } - if dstKind := dst.Kind(); isIntFloat(dstKind) { - var i any - var err error - switch { - case dstKind == reflect.Bool: - i, err = strconv.ParseBool(src) - case dst.CanInt(): - i, err = strconv.ParseInt(src, 10, dstT.Bits()) - case dst.CanUint(): - i, err = strconv.ParseUint(src, 10, dstT.Bits()) - case dst.CanFloat(): - i, err = strconv.ParseFloat(src, dstT.Bits()) - } + if gi.ReflectIsNumeric(dst) || dst.Kind() == reflect.Bool { + err := gi.ReflectStrToNumBool(dst, src) if err != nil { return true, gperr.Wrap(err) } - dst.Set(reflect.ValueOf(i).Convert(dstT)) - return convertible, convErr + return true, nil } // check if (*T).Convertor is implemented - if parser, ok := dst.Addr().Interface().(strutils.Parser); ok { + if dst.Addr().Type().Implements(parserType) { + parser := dst.Addr().Interface().(strutils.Parser) return true, gperr.Wrap(parser.Parse(src)) } // yaml like @@ -485,10 +535,10 @@ func ConvertString(src string, dst reflect.Value) (convertible bool, convErr gpe // one liner is comma separated list if !isMultiline && src[0] != '-' { values := strutils.CommaSeperatedList(src) - dst.Set(reflect.MakeSlice(dst.Type(), len(values), len(values))) + gi.ReflectInitSlice(dst, len(values), len(values)) errs := gperr.NewBuilder() for i, v := range values { - err := Convert(reflect.ValueOf(v), dst.Index(i), true) + _, err := ConvertString(v, dst.Index(i)) if err != nil { errs.Add(err.Subjectf("[%d]", i)) } @@ -496,16 +546,16 @@ func ConvertString(src string, dst reflect.Value) (convertible bool, convErr gpe if errs.HasError() { return true, errs.Error() } - return convertible, convErr + return true, nil } - sl := make([]any, 0) + sl := []any{} err := yaml.Unmarshal([]byte(src), &sl) if err != nil { return true, gperr.Wrap(err) } tmp = sl case reflect.Map, reflect.Struct: - rawMap := make(SerializedObject) + rawMap := SerializedObject{} err := yaml.Unmarshal([]byte(src), &rawMap) if err != nil { return true, gperr.Wrap(err) diff --git a/internal/serialization/serialization_benchmark_test.go b/internal/serialization/serialization_benchmark_test.go new file mode 100644 index 00000000..56531046 --- /dev/null +++ b/internal/serialization/serialization_benchmark_test.go @@ -0,0 +1,95 @@ +package serialization + +import ( + "reflect" + "testing" + + "github.com/goccy/go-yaml" +) + +func BenchmarkDeserialize(b *testing.B) { + type AnonymousStruct struct { + J float64 `json:"j"` + K int `json:"k"` + } + type complexStruct struct { + A string `json:"a"` + B int `json:"b"` + C []uint `json:"c"` + D map[string]string `json:"d"` + E []map[string]string `json:"e"` + F *complexStruct + G struct { + G1 float64 `json:"g1"` + G2 int `json:"g2"` + } + H []*complexStruct `json:"h"` + *AnonymousStruct + } + src := SerializedObject{ + "a": "a", + "b": "123", + "c": "1,2,3", + "d": "a: a\nb: b\nc: c", + "e": "- a: a\n b: b\n c: c", + "f": map[string]any{"a": "a", "b": "456", "c": []string{"1", "2", "3"}}, + "g": map[string]any{"g1": "1.23", "g2": 123}, + "h": []map[string]any{{"a": 123, "b": "456", "c": []string{"1", "2", "3"}}}, + "j": "1.23", + "k": 123, + } + for b.Loop() { + dst := complexStruct{} + err := MapUnmarshalValidate(src, &dst) + if err != nil { + b.Fatal(string(err.Plain())) + } + } +} + +func BenchmarkStringToSlice(b *testing.B) { + b.Run("ConvertString", func(b *testing.B) { + for b.Loop() { + dst := make([]int, 0) + _, _ = ConvertString("- 1\n- 2\n- 3", reflect.ValueOf(&dst)) + } + }) + b.Run("yaml.Unmarshal", func(b *testing.B) { + for b.Loop() { + dst := make([]int, 0) + _ = yaml.Unmarshal([]byte("- 1\n- 2\n- 3"), &dst) + } + }) +} + +func BenchmarkStringToMap(b *testing.B) { + b.Run("ConvertString", func(b *testing.B) { + for b.Loop() { + dst := make(map[string]string) + _, _ = ConvertString(" a: b\n c: d", reflect.ValueOf(&dst)) + } + }) + b.Run("yaml.Unmarshal", func(b *testing.B) { + for b.Loop() { + dst := make(map[string]string) + _ = yaml.Unmarshal([]byte(" a: b\n c: d"), &dst) + } + }) +} + +func BenchmarkStringToStruct(b *testing.B) { + dst := struct { + A string `json:"a"` + B int `json:"b"` + }{} + b.Run("ConvertString", func(b *testing.B) { + for b.Loop() { + _, _ = ConvertString(" a: a\n b: 123", reflect.ValueOf(&dst)) + } + }) + b.Run("yaml.Unmarshal", func(b *testing.B) { + for b.Loop() { + _ = yaml.Unmarshal([]byte(" a: a\n b: 123"), &dst) + } + }) +} diff --git a/internal/serialization/serialization_test.go b/internal/serialization/serialization_test.go index 61c27748..1b0a6c27 100644 --- a/internal/serialization/serialization_test.go +++ b/internal/serialization/serialization_test.go @@ -6,7 +6,6 @@ import ( "strconv" "testing" - "github.com/goccy/go-yaml" "github.com/stretchr/testify/require" expect "github.com/yusing/goutils/testing" ) @@ -262,20 +261,6 @@ func TestStringToSlice(t *testing.T) { }) } -func BenchmarkStringToSlice(b *testing.B) { - for range b.N { - dst := make([]int, 0) - _, _ = ConvertString("- 1\n- 2\n- 3", reflect.ValueOf(&dst)) - } -} - -func BenchmarkStringToSliceYAML(b *testing.B) { - for range b.N { - dst := make([]int, 0) - _ = yaml.Unmarshal([]byte("- 1\n- 2\n- 3"), &dst) - } -} - func TestStringToMap(t *testing.T) { t.Run("yaml-like", func(t *testing.T) { dst := make(map[string]string) @@ -286,20 +271,6 @@ func TestStringToMap(t *testing.T) { }) } -func BenchmarkStringToMap(b *testing.B) { - for range b.N { - dst := make(map[string]string) - _, _ = ConvertString(" a: b\n c: d", reflect.ValueOf(&dst)) - } -} - -func BenchmarkStringToMapYAML(b *testing.B) { - for range b.N { - dst := make(map[string]string) - _ = yaml.Unmarshal([]byte(" a: b\n c: d"), &dst) - } -} - func TestStringToStruct(t *testing.T) { t.Run("yaml-like", func(t *testing.T) { dst := struct { @@ -335,23 +306,3 @@ autocert: require.NoError(t, UnmarshalValidateYAML(data, &cfg)) require.Equal(t, "test", cfg.Autocert.Options.AuthToken) } - -func BenchmarkStringToStruct(b *testing.B) { - for range b.N { - dst := struct { - A string `json:"a"` - B int `json:"b"` - }{} - _, _ = ConvertString(" a: a\n b: 123", reflect.ValueOf(&dst)) - } -} - -func BenchmarkStringToStructYAML(b *testing.B) { - for range b.N { - dst := struct { - A string `yaml:"a"` - B int `yaml:"b"` - }{} - _ = yaml.Unmarshal([]byte(" a: a\n b: 123"), &dst) - } -}