From 2b4c39a79e49393a4352d1bf8ec7b146d1096018 Mon Sep 17 00:00:00 2001 From: yusing Date: Wed, 15 Oct 2025 23:51:47 +0800 Subject: [PATCH] perf(mem): reduced memory usage in metrics and task by string interning and deduplicating fields --- goutils | 2 +- internal/gopsutil | 2 +- internal/metrics/period/entries.go | 20 +++--- internal/metrics/period/poller.go | 31 ++++++--- internal/metrics/systeminfo/system_info.go | 12 ++-- internal/metrics/uptime/uptime.go | 6 +- internal/route/routes/query.go | 77 ++++++++++------------ 7 files changed, 79 insertions(+), 71 deletions(-) diff --git a/goutils b/goutils index fcbd5ce8..9d482a23 160000 --- a/goutils +++ b/goutils @@ -1 +1 @@ -Subproject commit fcbd5ce8c1e1eb918db37a66eb8d2defc85d7543 +Subproject commit 9d482a238dd75543c5582d1778c0dea484952f43 diff --git a/internal/gopsutil b/internal/gopsutil index 6e3478be..91e7deea 160000 --- a/internal/gopsutil +++ b/internal/gopsutil @@ -1 +1 @@ -Subproject commit 6e3478be651ce5597aae9c4122e602808685212a +Subproject commit 91e7deeaf0939cbd5f02d79be707e630d502405d diff --git a/internal/metrics/period/entries.go b/internal/metrics/period/entries.go index f65df585..cb6dd3ae 100644 --- a/internal/metrics/period/entries.go +++ b/internal/metrics/period/entries.go @@ -1,6 +1,7 @@ package period import ( + "encoding/json" "time" "github.com/bytedance/sonic" @@ -68,19 +69,22 @@ func (e *Entries[T]) Get() []T { return res[:] } +type entriesJSON[T any] struct { + Entries []T `json:"entries"` + Interval time.Duration `json:"interval"` +} + func (e *Entries[T]) MarshalJSON() ([]byte, error) { - return sonic.Marshal(map[string]any{ - "entries": e.Get(), - "interval": e.interval, + return sonic.Marshal(entriesJSON[T]{ + Entries: e.Get(), + Interval: e.interval, }) } func (e *Entries[T]) UnmarshalJSON(data []byte) error { - var v struct { - Entries []T `json:"entries"` - Interval time.Duration `json:"interval"` - } - if err := sonic.Unmarshal(data, &v); err != nil { + var v entriesJSON[T] + v.Entries = make([]T, 0, maxEntries) + if err := json.Unmarshal(data, &v); err != nil { return err } if len(v.Entries) == 0 { diff --git a/internal/metrics/period/poller.go b/internal/metrics/period/poller.go index e5e246c4..390b735f 100644 --- a/internal/metrics/period/poller.go +++ b/internal/metrics/period/poller.go @@ -2,6 +2,7 @@ package period import ( "context" + "encoding/json" "fmt" "net/url" "os" @@ -72,11 +73,16 @@ func (p *Poller[T, AggregateT]) savePath() string { } func (p *Poller[T, AggregateT]) load() error { - entries, err := os.ReadFile(p.savePath()) + content, err := os.ReadFile(p.savePath()) if err != nil { return err } - if err := sonic.Unmarshal(entries, &p.period); err != nil { + + if len(content) == 0 { + return nil + } + + if err := json.Unmarshal(content, p.period); err != nil { return err } // Validate and fix intervals after loading to ensure data integrity. @@ -86,11 +92,17 @@ func (p *Poller[T, AggregateT]) load() error { func (p *Poller[T, AggregateT]) save() error { initDataDirOnce.Do(initDataDir) - entries, err := sonic.Marshal(p.period) + f, err := os.OpenFile(p.savePath(), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) if err != nil { return err } - return os.WriteFile(p.savePath(), entries, 0o644) + defer f.Close() + + err = sonic.ConfigDefault.NewEncoder(f).Encode(p.period) + if err != nil { + return err + } + return nil } func (p *Poller[T, AggregateT]) WithResultFilter(filter FilterFunc[T]) *Poller[T, AggregateT] { @@ -114,15 +126,15 @@ func (p *Poller[T, AggregateT]) appendErr(err error) { p.errs = append(p.errs, pollErr{err: err, count: 1}) } -func (p *Poller[T, AggregateT]) gatherErrs() (string, bool) { +func (p *Poller[T, AggregateT]) gatherErrs() (error, bool) { if len(p.errs) == 0 { - return "", false + return nil, false } - errs := gperr.NewBuilder(fmt.Sprintf("poller %s has encountered %d errors in the last %s:", p.name, len(p.errs), gatherErrsInterval)) + var errs gperr.Builder for _, e := range p.errs { errs.Addf("%w: %d times", e.err, e.count) } - return errs.String(), true + return errs.Error(), true } func (p *Poller[T, AggregateT]) clearErrs() { @@ -164,6 +176,7 @@ func (p *Poller[T, AggregateT]) Start() { if err != nil { l.Err(err).Msg("failed to save metrics data") } + l.Debug().Int("entries", p.period.Total()).Msg("poller finished and saved") t.Finish(err) }() @@ -183,7 +196,7 @@ func (p *Poller[T, AggregateT]) Start() { if tickCount%gatherErrsTicks == 0 { errs, ok := p.gatherErrs() if ok { - log.Error().Msg(errs) + gperr.LogError(fmt.Sprintf("poller %s has encountered %d errors in the last %s:", p.name, len(p.errs), gatherErrsInterval), errs) } p.clearErrs() } diff --git a/internal/metrics/systeminfo/system_info.go b/internal/metrics/systeminfo/system_info.go index 14b280da..1773f148 100644 --- a/internal/metrics/systeminfo/system_info.go +++ b/internal/metrics/systeminfo/system_info.go @@ -167,7 +167,7 @@ func (s *SystemInfo) collectDisksInfo(ctx context.Context, lastResult *SystemInf if lastUsage, ok := lastResult.DisksIO[name]; ok { disk.ReadSpeed = float32(disk.ReadBytes-lastUsage.ReadBytes) / float32(interval) disk.WriteSpeed = float32(disk.WriteBytes-lastUsage.WriteBytes) / float32(interval) - disk.Iops = diff(disk.ReadCount+disk.WriteCount, lastUsage.ReadCount+lastUsage.WriteCount) / uint64(interval) //nolint:gosec + disk.Iops = diff(disk.IOCount, lastUsage.IOCount) / uint64(interval) //nolint:gosec } } } @@ -179,12 +179,12 @@ func (s *SystemInfo) collectDisksInfo(ctx context.Context, lastResult *SystemInf s.Disks = make(map[string]disk.UsageStat, len(partitions)) errs := gperr.NewBuilder("failed to get disks info") for _, partition := range partitions { - diskInfo, err := disk.UsageWithContext(ctx, partition.Mountpoint) + diskInfo, err := disk.UsageWithContext(ctx, partition.Mountpoint.Value()) if err != nil { errs.Add(err) continue } - s.Disks[partition.Device] = diskInfo + s.Disks[partition.Device.Value()] = diskInfo } if errs.HasError() { @@ -247,10 +247,10 @@ func aggregate(entries []*SystemInfo, query url.Values) (total int, result Aggre } case SystemInfoAggregateModeMemoryUsagePercent: for _, entry := range entries { - if entry.Memory.UsedPercent > 0 { + if percent := entry.Memory.UsedPercent(); percent > 0 { aggregated.Entries = append(aggregated.Entries, map[string]any{ "timestamp": entry.Timestamp, - "memory_usage_percent": entry.Memory.UsedPercent, + "memory_usage_percent": percent, }) } } @@ -331,7 +331,7 @@ func aggregate(entries []*SystemInfo, query url.Values) (total int, result Aggre } m := make(map[string]any, len(entry.Sensors)+1) for _, sensor := range entry.Sensors { - m[sensor.SensorKey] = sensor.Temperature + m[sensor.SensorKey.Value()] = sensor.Temperature } m["timestamp"] = entry.Timestamp aggregated.Entries = append(aggregated.Entries, m) diff --git a/internal/metrics/uptime/uptime.go b/internal/metrics/uptime/uptime.go index 58845df8..99b7f04a 100644 --- a/internal/metrics/uptime/uptime.go +++ b/internal/metrics/uptime/uptime.go @@ -17,8 +17,8 @@ import ( type ( StatusByAlias struct { - Map map[string]routes.HealthInfo `json:"statuses"` - Timestamp int64 `json:"timestamp"` + Map map[string]routes.HealthInfoWithoutDetail `json:"statuses"` + Timestamp int64 `json:"timestamp"` } // @name RouteStatusesByAlias Status struct { Status types.HealthStatus `json:"status" swaggertype:"string" enums:"healthy,unhealthy,unknown,napping,starting"` @@ -44,7 +44,7 @@ var Poller = period.NewPoller("uptime", getStatuses, aggregateStatuses) func getStatuses(ctx context.Context, _ StatusByAlias) (StatusByAlias, error) { return StatusByAlias{ - Map: routes.GetHealthInfo(), + Map: routes.GetHealthInfoWithoutDetail(), Timestamp: time.Now().Unix(), }, nil } diff --git a/internal/route/routes/query.go b/internal/route/routes/query.go index 7be9ee22..527774cb 100644 --- a/internal/route/routes/query.go +++ b/internal/route/routes/query.go @@ -1,56 +1,21 @@ package routes import ( - "math" "time" - "github.com/bytedance/sonic" "github.com/yusing/godoxy/internal/types" ) type HealthInfo struct { + HealthInfoWithoutDetail + Detail string `json:"detail"` +} // @name HealthInfo + +type HealthInfoWithoutDetail struct { Status types.HealthStatus `json:"status" swaggertype:"string" enums:"healthy,unhealthy,napping,starting,error,unknown"` Uptime time.Duration `json:"uptime" swaggertype:"number"` // uptime in milliseconds Latency time.Duration `json:"latency" swaggertype:"number"` // latency in microseconds - Detail string `json:"detail"` -} - -func (info *HealthInfo) MarshalJSON() ([]byte, error) { - return sonic.Marshal(map[string]any{ - "status": info.Status.String(), - "latency": info.Latency.Microseconds(), - "uptime": info.Uptime.Milliseconds(), - "detail": info.Detail, - }) -} - -func (info *HealthInfo) UnmarshalJSON(data []byte) error { - var v struct { - Status string `json:"status"` - Latency int64 `json:"latency"` - Uptime int64 `json:"uptime"` - Detail string `json:"detail"` - } - if err := sonic.Unmarshal(data, &v); err != nil { - return err - } - - // overflow check - // Check if latency (in microseconds) would overflow when converted to nanoseconds - if v.Latency > math.MaxInt64/int64(time.Microsecond) { - v.Latency = 0 - } - // Check if uptime (in milliseconds) would overflow when converted to nanoseconds - if v.Uptime > math.MaxInt64/int64(time.Millisecond) { - v.Uptime = 0 - } - - info.Status = types.NewHealthStatusFromString(v.Status) - info.Latency = time.Duration(v.Latency) * time.Microsecond - info.Uptime = time.Duration(v.Uptime) * time.Millisecond - info.Detail = v.Detail - return nil -} +} // @name HealthInfoWithoutDetail func GetHealthInfo() map[string]HealthInfo { healthMap := make(map[string]HealthInfo, NumRoutes()) @@ -60,19 +25,45 @@ func GetHealthInfo() map[string]HealthInfo { return healthMap } +func GetHealthInfoWithoutDetail() map[string]HealthInfoWithoutDetail { + healthMap := make(map[string]HealthInfoWithoutDetail, NumRoutes()) + for r := range Iter { + healthMap[r.Name()] = getHealthInfoWithoutDetail(r) + } + return healthMap +} + func getHealthInfo(r types.Route) HealthInfo { mon := r.HealthMonitor() if mon == nil { return HealthInfo{ - Status: types.StatusUnknown, + HealthInfoWithoutDetail: HealthInfoWithoutDetail{ + Status: types.StatusUnknown, + }, Detail: "n/a", } } return HealthInfo{ + HealthInfoWithoutDetail: HealthInfoWithoutDetail{ + Status: mon.Status(), + Uptime: mon.Uptime(), + Latency: mon.Latency(), + }, + Detail: mon.Detail(), + } +} + +func getHealthInfoWithoutDetail(r types.Route) HealthInfoWithoutDetail { + mon := r.HealthMonitor() + if mon == nil { + return HealthInfoWithoutDetail{ + Status: types.StatusUnknown, + } + } + return HealthInfoWithoutDetail{ Status: mon.Status(), Uptime: mon.Uptime(), Latency: mon.Latency(), - Detail: mon.Detail(), } }