From 79f40f3d22a7ae0f25a46ff1db89452b76081879 Mon Sep 17 00:00:00 2001 From: yusing Date: Thu, 23 Jan 2025 04:16:06 +0800 Subject: [PATCH] implement icon cache expiry, cleanup code and upgrade deps --- go.mod | 4 +- go.sum | 8 +- internal/api/v1/favicon/cache.go | 133 +++++++++++++++++++++++++++ internal/api/v1/favicon/content.go | 37 ++++++++ internal/api/v1/favicon/favicon.go | 89 ------------------ internal/homepage/override_config.go | 3 + internal/route/http.go | 2 +- internal/utils/serialization.go | 6 +- 8 files changed, 183 insertions(+), 99 deletions(-) create mode 100644 internal/api/v1/favicon/cache.go create mode 100644 internal/api/v1/favicon/content.go diff --git a/go.mod b/go.mod index a4cc8cf3..726acb9d 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ require ( github.com/PuerkitoBio/goquery v1.10.1 github.com/coder/websocket v1.8.12 github.com/coreos/go-oidc/v3 v3.12.0 - github.com/docker/cli v27.5.0+incompatible - github.com/docker/docker v27.5.0+incompatible + github.com/docker/cli v27.5.1+incompatible + github.com/docker/docker v27.5.1+incompatible github.com/fsnotify/fsnotify v1.8.0 github.com/go-acme/lego/v4 v4.21.0 github.com/go-playground/validator/v10 v10.24.0 diff --git a/go.sum b/go.sum index 25d85f7a..56b031e1 100644 --- a/go.sum +++ b/go.sum @@ -27,10 +27,10 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/cli v27.5.0+incompatible h1:aMphQkcGtpHixwwhAXJT1rrK/detk2JIvDaFkLctbGM= -github.com/docker/cli v27.5.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/docker v27.5.0+incompatible h1:um++2NcQtGRTz5eEgO6aJimo6/JxrTXC941hd05JO6U= -github.com/docker/docker v27.5.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/cli v27.5.1+incompatible h1:JB9cieUT9YNiMITtIsguaN55PLOHhBSz3LKVc6cqWaY= +github.com/docker/cli v27.5.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/docker v27.5.1+incompatible h1:4PYU5dnBYqRQi0294d1FBECqT9ECWeQAIfE8q4YnPY8= +github.com/docker/docker v27.5.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= diff --git a/internal/api/v1/favicon/cache.go b/internal/api/v1/favicon/cache.go new file mode 100644 index 00000000..53acf816 --- /dev/null +++ b/internal/api/v1/favicon/cache.go @@ -0,0 +1,133 @@ +package favicon + +import ( + "encoding/json" + "sync" + "time" + + "github.com/yusing/go-proxy/internal/common" + "github.com/yusing/go-proxy/internal/logging" + route "github.com/yusing/go-proxy/internal/route/types" + "github.com/yusing/go-proxy/internal/task" + "github.com/yusing/go-proxy/internal/utils" +) + +type cacheEntry struct { + Icon []byte `json:"icon"` + LastAccess time.Time `json:"lastAccess"` +} + +// cache key can be absolute url or route name. +var ( + iconCache = make(map[string]*cacheEntry) + iconCacheMu sync.RWMutex +) + +const ( + iconCacheTTL = 24 * time.Hour + cleanUpInterval = time.Hour +) + +func InitIconCache() { + err := utils.LoadJSONIfExist(common.IconCachePath, &iconCache) + if err != nil { + logging.Error().Err(err).Msg("failed to load icon cache") + } else { + logging.Info().Msgf("icon cache loaded (%d icons)", len(iconCache)) + } + + go func() { + cleanupTicker := time.NewTicker(cleanUpInterval) + defer cleanupTicker.Stop() + for { + select { + case <-task.RootContextCanceled(): + return + case <-cleanupTicker.C: + pruneExpiredIconCache() + } + } + }() + + task.OnProgramExit("save_favicon_cache", func() { + iconCacheMu.Lock() + defer iconCacheMu.Unlock() + + if len(iconCache) == 0 { + return + } + + if err := utils.SaveJSON(common.IconCachePath, &iconCache, 0o644); err != nil { + logging.Error().Err(err).Msg("failed to save icon cache") + } + }) +} + +func pruneExpiredIconCache() { + iconCacheMu.Lock() + defer iconCacheMu.Unlock() + + nPruned := 0 + for key, icon := range iconCache { + if icon.IsExpired() { + delete(iconCache, key) + nPruned++ + } + } + logging.Info().Int("pruned", nPruned).Msg("pruned expired icon cache") +} + +func routeKey(r route.HTTPRoute) string { + return r.RawEntry().Provider + ":" + r.TargetName() +} + +func PruneRouteIconCache(route route.HTTPRoute) { + iconCacheMu.Lock() + defer iconCacheMu.Unlock() + delete(iconCache, routeKey(route)) +} + +func loadIconCache(key string) *fetchResult { + iconCacheMu.RLock() + defer iconCacheMu.RUnlock() + + icon, ok := iconCache[key] + if ok && icon != nil { + logging.Debug(). + Str("key", key). + Msg("icon found in cache") + icon.LastAccess = time.Now() + return &fetchResult{icon: icon.Icon} + } + return nil +} + +func storeIconCache(key string, icon []byte) { + iconCacheMu.Lock() + defer iconCacheMu.Unlock() + iconCache[key] = &cacheEntry{Icon: icon, LastAccess: time.Now()} +} + +func (e *cacheEntry) IsExpired() bool { + return time.Since(e.LastAccess) > iconCacheTTL +} + +func (e *cacheEntry) UnmarshalJSON(data []byte) error { + attempt := struct { + Icon []byte `json:"icon"` + LastAccess time.Time `json:"lastAccess"` + }{} + err := json.Unmarshal(data, &attempt) + if err == nil { + e.Icon = attempt.Icon + e.LastAccess = attempt.LastAccess + return nil + } + // fallback to bytes + err = json.Unmarshal(data, &e.Icon) + if err == nil { + e.LastAccess = time.Now() + return nil + } + return err +} diff --git a/internal/api/v1/favicon/content.go b/internal/api/v1/favicon/content.go new file mode 100644 index 00000000..5d4ba3e3 --- /dev/null +++ b/internal/api/v1/favicon/content.go @@ -0,0 +1,37 @@ +package favicon + +import ( + "bufio" + "errors" + "net" + "net/http" +) + +type content struct { + header http.Header + data []byte + status int +} + +func newContent() *content { + return &content{ + header: make(http.Header), + } +} + +func (c *content) Header() http.Header { + return c.header +} + +func (c *content) Write(data []byte) (int, error) { + c.data = append(c.data, data...) + return len(data), nil +} + +func (c *content) WriteHeader(statusCode int) { + c.status = statusCode +} + +func (c *content) Hijack() (net.Conn, *bufio.ReadWriter, error) { + return nil, nil, errors.New("not supported") +} diff --git a/internal/api/v1/favicon/favicon.go b/internal/api/v1/favicon/favicon.go index 798d1dbb..299ff8c3 100644 --- a/internal/api/v1/favicon/favicon.go +++ b/internal/api/v1/favicon/favicon.go @@ -1,38 +1,26 @@ package favicon import ( - "bufio" "bytes" "context" "errors" "io" - "net" "net/http" "net/url" "path" "strings" - "sync" "time" "github.com/PuerkitoBio/goquery" "github.com/vincent-petithory/dataurl" U "github.com/yusing/go-proxy/internal/api/v1/utils" - "github.com/yusing/go-proxy/internal/common" "github.com/yusing/go-proxy/internal/homepage" "github.com/yusing/go-proxy/internal/logging" gphttp "github.com/yusing/go-proxy/internal/net/http" "github.com/yusing/go-proxy/internal/route/routes" route "github.com/yusing/go-proxy/internal/route/types" - "github.com/yusing/go-proxy/internal/task" - "github.com/yusing/go-proxy/internal/utils" ) -type content struct { - header http.Header - data []byte - status int -} - type fetchResult struct { icon []byte contentType string @@ -40,29 +28,6 @@ type fetchResult struct { errMsg string } -func newContent() *content { - return &content{ - header: make(http.Header), - } -} - -func (c *content) Header() http.Header { - return c.header -} - -func (c *content) Write(data []byte) (int, error) { - c.data = append(c.data, data...) - return len(data), nil -} - -func (c *content) WriteHeader(statusCode int) { - c.status = statusCode -} - -func (c *content) Hijack() (net.Conn, *bufio.ReadWriter, error) { - return nil, nil, errors.New("not supported") -} - func (res *fetchResult) OK() bool { return res.icon != nil } @@ -156,60 +121,6 @@ func getFavIconFromURL(iconURL *homepage.IconURL) *fetchResult { return &fetchResult{statusCode: http.StatusBadRequest, errMsg: "invalid icon source"} } -// cache key can be absolute url or route name. -var ( - iconCache = make(map[string][]byte) - iconCacheMu sync.RWMutex -) - -func InitIconCache() { - err := utils.LoadJSONIfExist(common.IconCachePath, &iconCache) - if err != nil { - logging.Error().Err(err).Msg("failed to load icon cache") - } else { - logging.Info().Msgf("icon cache loaded (%d icons)", len(iconCache)) - } - - task.OnProgramExit("save_favicon_cache", func() { - iconCacheMu.Lock() - defer iconCacheMu.Unlock() - - if err := utils.SaveJSON(common.IconCachePath, &iconCache, 0o644); err != nil { - logging.Error().Err(err).Msg("failed to save icon cache") - } - }) -} - -func routeKey(r route.HTTPRoute) string { - return r.RawEntry().Provider + ":" + r.TargetName() -} - -func ResetIconCache(route route.HTTPRoute) { - iconCacheMu.Lock() - defer iconCacheMu.Unlock() - delete(iconCache, routeKey(route)) -} - -func loadIconCache(key string) *fetchResult { - iconCacheMu.RLock() - defer iconCacheMu.RUnlock() - icon, ok := iconCache[key] - if ok && icon != nil { - logging.Debug(). - Str("key", key). - Msg("icon found in cache") - - return &fetchResult{icon: icon} - } - return nil -} - -func storeIconCache(key string, icon []byte) { - iconCacheMu.Lock() - defer iconCacheMu.Unlock() - iconCache[key] = icon -} - func fetchIconAbsolute(url string) *fetchResult { if result := loadIconCache(url); result != nil { return result diff --git a/internal/homepage/override_config.go b/internal/homepage/override_config.go index 11f84595..8da06a84 100644 --- a/internal/homepage/override_config.go +++ b/internal/homepage/override_config.go @@ -40,6 +40,9 @@ func InitOverridesConfig() { logging.Info().Msgf("homepage overrides config loaded, %d items", len(overrideConfigInstance.ItemOverrides)) } task.OnProgramExit("save_homepage_json_config", func() { + if len(overrideConfigInstance.ItemOverrides) == 0 { + return + } if err := utils.SaveJSON(common.HomepageJSONConfigPath, overrideConfigInstance, 0o644); err != nil { logging.Error().Err(err).Msg("failed to save homepage overrides config") } diff --git a/internal/route/http.go b/internal/route/http.go index e6e2542b..dd355dd7 100755 --- a/internal/route/http.go +++ b/internal/route/http.go @@ -163,7 +163,7 @@ func (r *HTTPRoute) Start(parent task.Parent) E.Error { r.task.OnCancel("metrics_cleanup", r.rp.UnregisterMetrics) } - r.task.OnCancel("reset_favicon", func() { favicon.ResetIconCache(r) }) + r.task.OnCancel("reset_favicon", func() { favicon.PruneRouteIconCache(r) }) return nil } diff --git a/internal/utils/serialization.go b/internal/utils/serialization.go index 33728830..13da6216 100644 --- a/internal/utils/serialization.go +++ b/internal/utils/serialization.go @@ -429,12 +429,12 @@ func DeserializeJSON[T any](data []byte, target T) E.Error { return Deserialize(m, target) } -func LoadJSON[T any](path string, dst *T) error { +func loadSerialized[T any](path string, dst *T, deserialize func(data []byte, dst any) error) error { data, err := os.ReadFile(path) if err != nil { return err } - return json.Unmarshal(data, dst) + return deserialize(data, dst) } func SaveJSON[T any](path string, src *T, perm os.FileMode) error { @@ -453,5 +453,5 @@ func LoadJSONIfExist[T any](path string, dst *T) error { } return err } - return LoadJSON(path, dst) + return loadSerialized(path, dst, json.Unmarshal) }