refactor: move favicon into homepage module

internal/homepage/content.go

@@ -0,0 +1,37 @@
+package homepage
+
+import (
+	"bufio"
+	"errors"
+	"net"
+	"net/http"
+)
+
+type content struct {
+	header http.Header
+	data   []byte
+	status int
+}
+
+func newContent() *content {
+	return &content{
+		header: make(http.Header),
+	}
+}
+
+func (c *content) Header() http.Header {
+	return c.header
+}
+
+func (c *content) Write(data []byte) (int, error) {
+	c.data = append(c.data, data...)
+	return len(data), nil
+}
+
+func (c *content) WriteHeader(statusCode int) {
+	c.status = statusCode
+}
+
+func (c *content) Hijack() (net.Conn, *bufio.ReadWriter, error) {
+	return nil, nil, errors.New("not supported")
+}

internal/homepage/favicon.go

@@ -0,0 +1,195 @@
+package homepage
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/vincent-petithory/dataurl"
+	gphttp "github.com/yusing/go-proxy/internal/net/gphttp"
+	"github.com/yusing/go-proxy/internal/utils/strutils"
+)
+
+type FetchResult struct {
+	Icon       []byte
+	StatusCode int
+	ErrMsg     string
+
+	contentType string
+}
+
+func (res *FetchResult) OK() bool {
+	return res.Icon != nil
+}
+
+func (res *FetchResult) ContentType() string {
+	if res.contentType == "" {
+		if bytes.HasPrefix(res.Icon, []byte("<svg")) || bytes.HasPrefix(res.Icon, []byte("<?xml")) {
+			return "image/svg+xml"
+		}
+		return "image/x-icon"
+	}
+	return res.contentType
+}
+
+const maxRedirectDepth = 5
+
+func FetchFavIconFromURL(iconURL *IconURL) *FetchResult {
+	switch iconURL.IconSource {
+	case IconSourceAbsolute:
+		return fetchIconAbsolute(iconURL.URL())
+	case IconSourceRelative:
+		return &FetchResult{StatusCode: http.StatusBadRequest, ErrMsg: "unexpected relative icon"}
+	case IconSourceWalkXCode, IconSourceSelfhSt:
+		return fetchKnownIcon(iconURL)
+	}
+	return &FetchResult{StatusCode: http.StatusBadRequest, ErrMsg: "invalid icon source"}
+}
+
+func fetchIconAbsolute(url string) *FetchResult {
+	if result := loadIconCache(url); result != nil {
+		return result
+	}
+
+	resp, err := gphttp.Get(url)
+	if err != nil || resp.StatusCode != http.StatusOK {
+		if err == nil {
+			err = errors.New(resp.Status)
+		}
+		return &FetchResult{StatusCode: http.StatusBadGateway, ErrMsg: "connection error"}
+	}
+
+	defer resp.Body.Close()
+	icon, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return &FetchResult{StatusCode: http.StatusInternalServerError, ErrMsg: "internal error"}
+	}
+
+	storeIconCache(url, icon)
+	return &FetchResult{Icon: icon}
+}
+
+var nameSanitizer = strings.NewReplacer(
+	"_", "-",
+	" ", "-",
+	"(", "",
+	")", "",
+)
+
+func sanitizeName(name string) string {
+	return strings.ToLower(nameSanitizer.Replace(name))
+}
+
+func fetchKnownIcon(url *IconURL) *FetchResult {
+	// if icon isn't in the list, no need to fetch
+	if !url.HasIcon() {
+		return &FetchResult{StatusCode: http.StatusNotFound, ErrMsg: "no such icon"}
+	}
+
+	return fetchIconAbsolute(url.URL())
+}
+
+func fetchIcon(filetype, filename string) *FetchResult {
+	result := fetchKnownIcon(NewSelfhStIconURL(filename, filetype))
+	if result.Icon == nil {
+		return result
+	}
+	return fetchKnownIcon(NewWalkXCodeIconURL(filename, filetype))
+}
+
+func FindIcon(ctx context.Context, r route, uri string) *FetchResult {
+	key := routeKey(r)
+	if result := loadIconCache(key); result != nil {
+		return result
+	}
+
+	result := fetchIcon("png", sanitizeName(r.Reference()))
+	if !result.OK() {
+		if r, ok := r.(httpRoute); ok {
+			// fallback to parse html
+			result = findIconSlow(ctx, r, uri, 0)
+		}
+	}
+	if result.OK() {
+		storeIconCache(key, result.Icon)
+	}
+	return result
+}
+
+func findIconSlow(ctx context.Context, r httpRoute, uri string, depth int) *FetchResult {
+	ctx, cancel := context.WithTimeoutCause(ctx, 3*time.Second, errors.New("favicon request timeout"))
+	defer cancel()
+
+	newReq, err := http.NewRequestWithContext(ctx, "GET", r.TargetURL().String(), nil)
+	if err != nil {
+		return &FetchResult{StatusCode: http.StatusInternalServerError, ErrMsg: "cannot create request"}
+	}
+	newReq.Header.Set("Accept-Encoding", "identity") // disable compression
+
+	u, err := url.ParseRequestURI(strutils.SanitizeURI(uri))
+	if err != nil {
+		return &FetchResult{StatusCode: http.StatusInternalServerError, ErrMsg: "cannot parse uri"}
+	}
+	newReq.URL.Path = u.Path
+	newReq.URL.RawPath = u.RawPath
+	newReq.URL.RawQuery = u.RawQuery
+	newReq.RequestURI = u.String()
+
+	c := newContent()
+	r.ServeHTTP(c, newReq)
+	if c.status != http.StatusOK {
+		switch c.status {
+		case 0:
+			return &FetchResult{StatusCode: http.StatusBadGateway, ErrMsg: "connection error"}
+		default:
+			if loc := c.Header().Get("Location"); loc != "" {
+				if depth > maxRedirectDepth {
+					return &FetchResult{StatusCode: http.StatusBadGateway, ErrMsg: "too many redirects"}
+				}
+				loc = strutils.SanitizeURI(loc)
+				if loc == "/" || loc == newReq.URL.Path {
+					return &FetchResult{StatusCode: http.StatusBadGateway, ErrMsg: "circular redirect"}
+				}
+				return findIconSlow(ctx, r, loc, depth+1)
+			}
+		}
+		return &FetchResult{StatusCode: c.status, ErrMsg: "upstream error: " + string(c.data)}
+	}
+	// return icon data
+	if !gphttp.GetContentType(c.header).IsHTML() {
+		return &FetchResult{Icon: c.data, contentType: c.header.Get("Content-Type")}
+	}
+	// try extract from "link[rel=icon]" from path "/"
+	doc, err := goquery.NewDocumentFromReader(bytes.NewBuffer(c.data))
+	if err != nil {
+		return &FetchResult{StatusCode: http.StatusInternalServerError, ErrMsg: "failed to parse html"}
+	}
+	ele := doc.Find("head > link[rel=icon]").First()
+	if ele.Length() == 0 {
+		return &FetchResult{StatusCode: http.StatusNotFound, ErrMsg: "icon element not found"}
+	}
+	href := ele.AttrOr("href", "")
+	if href == "" {
+		return &FetchResult{StatusCode: http.StatusNotFound, ErrMsg: "icon href not found"}
+	}
+	// https://en.wikipedia.org/wiki/Data_URI_scheme
+	if strings.HasPrefix(href, "data:image/") {
+		dataURI, err := dataurl.DecodeString(href)
+		if err != nil {
+			return &FetchResult{StatusCode: http.StatusInternalServerError, ErrMsg: "failed to decode favicon"}
+		}
+		return &FetchResult{Icon: dataURI.Data, contentType: dataURI.ContentType()}
+	}
+	switch {
+	case strings.HasPrefix(href, "http://"), strings.HasPrefix(href, "https://"):
+		return fetchIconAbsolute(href)
+	default:
+		return findIconSlow(ctx, r, href, 0)
+	}
+}

internal/homepage/icon_cache.go

@@ -0,0 +1,137 @@
+package homepage
+
+import (
+	"encoding/json"
+	"sync"
+	"time"
+
+	"github.com/yusing/go-proxy/internal/common"
+	"github.com/yusing/go-proxy/internal/logging"
+	"github.com/yusing/go-proxy/internal/task"
+	"github.com/yusing/go-proxy/internal/utils"
+)
+
+type cacheEntry struct {
+	Icon       []byte    `json:"icon"`
+	LastAccess time.Time `json:"lastAccess"`
+}
+
+// cache key can be absolute url or route name.
+var (
+	iconCache   = make(map[string]*cacheEntry)
+	iconCacheMu sync.RWMutex
+)
+
+const (
+	iconCacheTTL    = 3 * 24 * time.Hour
+	cleanUpInterval = time.Hour
+)
+
+func InitIconCache() {
+	iconCacheMu.Lock()
+	defer iconCacheMu.Unlock()
+
+	err := utils.LoadJSONIfExist(common.IconCachePath, &iconCache)
+	if err != nil {
+		logging.Error().Err(err).Msg("failed to load icon cache")
+	} else if len(iconCache) > 0 {
+		logging.Info().Int("count", len(iconCache)).Msg("icon cache loaded")
+	}
+
+	go func() {
+		cleanupTicker := time.NewTicker(cleanUpInterval)
+		defer cleanupTicker.Stop()
+		for {
+			select {
+			case <-task.RootContextCanceled():
+				return
+			case <-cleanupTicker.C:
+				pruneExpiredIconCache()
+			}
+		}
+	}()
+
+	task.OnProgramExit("save_favicon_cache", func() {
+		iconCacheMu.Lock()
+		defer iconCacheMu.Unlock()
+
+		if len(iconCache) == 0 {
+			return
+		}
+
+		if err := utils.SaveJSON(common.IconCachePath, &iconCache, 0o644); err != nil {
+			logging.Error().Err(err).Msg("failed to save icon cache")
+		}
+	})
+}
+
+func pruneExpiredIconCache() {
+	iconCacheMu.Lock()
+	defer iconCacheMu.Unlock()
+
+	nPruned := 0
+	for key, icon := range iconCache {
+		if icon.IsExpired() {
+			delete(iconCache, key)
+			nPruned++
+		}
+	}
+	if nPruned > 0 {
+		logging.Info().Int("pruned", nPruned).Msg("pruned expired icon cache")
+	}
+}
+
+func routeKey(r route) string {
+	return r.ProviderName() + ":" + r.TargetName()
+}
+
+func PruneRouteIconCache(route route) {
+	iconCacheMu.Lock()
+	defer iconCacheMu.Unlock()
+	delete(iconCache, routeKey(route))
+}
+
+func loadIconCache(key string) *FetchResult {
+	iconCacheMu.RLock()
+	defer iconCacheMu.RUnlock()
+
+	icon, ok := iconCache[key]
+	if ok && icon != nil {
+		logging.Debug().
+			Str("key", key).
+			Msg("icon found in cache")
+		icon.LastAccess = time.Now()
+		return &FetchResult{Icon: icon.Icon}
+	}
+	return nil
+}
+
+func storeIconCache(key string, icon []byte) {
+	iconCacheMu.Lock()
+	defer iconCacheMu.Unlock()
+	iconCache[key] = &cacheEntry{Icon: icon, LastAccess: time.Now()}
+}
+
+func (e *cacheEntry) IsExpired() bool {
+	return time.Since(e.LastAccess) > iconCacheTTL
+}
+
+func (e *cacheEntry) UnmarshalJSON(data []byte) error {
+	attempt := struct {
+		Icon       []byte    `json:"icon"`
+		LastAccess time.Time `json:"lastAccess"`
+	}{}
+	err := json.Unmarshal(data, &attempt)
+	if err == nil {
+		e.Icon = attempt.Icon
+		e.LastAccess = attempt.LastAccess
+		return nil
+	}
+	// fallback to bytes
+	err = json.Unmarshal(data, &e.Icon)
+	if err == nil {
+		e.LastAccess = time.Now()
+		return nil
+	}
+	return err
+}

internal/homepage/icon_url.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/yusing/go-proxy/internal"
 	"github.com/yusing/go-proxy/internal/gperr"
 )
 
@@ -62,10 +61,10 @@ func NewWalkXCodeIconURL(name, format string) *IconURL {
 // otherwise returns true.
 func (u *IconURL) HasIcon() bool {
 	if u.IconSource == IconSourceSelfhSt {
-		return internal.HasSelfhstIcon(u.Extra.Name, u.Extra.FileType)
+		return HasSelfhstIcon(u.Extra.Name, u.Extra.FileType)
 	}
 	if u.IconSource == IconSourceWalkXCode {
-		return internal.HasWalkxCodeIcon(u.Extra.Name, u.Extra.FileType)
+		return HasWalkxCodeIcon(u.Extra.Name, u.Extra.FileType)
 	}
 	return true
 }

internal/homepage/list-icons.go

@@ -0,0 +1,297 @@
+package homepage
+
+import (
+	"encoding/json"
+	"io"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/lithammer/fuzzysearch/fuzzy"
+	"github.com/yusing/go-proxy/internal/common"
+	"github.com/yusing/go-proxy/internal/logging"
+	"github.com/yusing/go-proxy/internal/utils"
+)
+
+type GitHubContents struct { //! keep this, may reuse in future
+	Type string `json:"type"`
+	Path string `json:"path"`
+	Name string `json:"name"`
+	Sha  string `json:"sha"`
+	Size int    `json:"size"`
+}
+
+type (
+	IconsMap map[string]map[string]struct{}
+	IconList []string
+	Cache    struct {
+		WalkxCode, Selfhst IconsMap
+		DisplayNames       ReferenceDisplayNameMap
+		IconList           IconList // combined into a single list
+	}
+	ReferenceDisplayNameMap map[string]string
+)
+
+func (icons *Cache) needUpdate() bool {
+	return len(icons.WalkxCode) == 0 || len(icons.Selfhst) == 0 || len(icons.IconList) == 0 || len(icons.DisplayNames) == 0
+}
+
+const updateInterval = 2 * time.Hour
+
+var (
+	iconsCache   *Cache
+	iconsCahceMu sync.RWMutex
+	lastUpdate   time.Time
+)
+
+const (
+	walkxcodeIcons = "https://cdn.jsdelivr.net/gh/walkxcode/dashboard-icons@master/tree.json"
+	selfhstIcons   = "https://cdn.selfh.st/directory/icons.json"
+)
+
+func InitIconListCache() {
+	iconsCahceMu.Lock()
+	defer iconsCahceMu.Unlock()
+
+	iconsCache = &Cache{
+		WalkxCode:    make(IconsMap),
+		Selfhst:      make(IconsMap),
+		DisplayNames: make(ReferenceDisplayNameMap),
+		IconList:     []string{},
+	}
+	// err := utils.LoadJSONIfExist(common.IconListCachePath, iconsCache)
+	// if err != nil {
+	// 	logging.Error().Err(err).Msg("failed to load icon list cache config")
+	// } else if len(iconsCache.IconList) > 0 {
+	// 	logging.Info().
+	// 		Int("icons", len(iconsCache.IconList)).
+	// 		Int("display_names", len(iconsCache.DisplayNames)).
+	// 		Msg("icon list cache loaded")
+	// }
+}
+
+func ListAvailableIcons() (*Cache, error) {
+	iconsCahceMu.RLock()
+	if time.Since(lastUpdate) < updateInterval {
+		if !iconsCache.needUpdate() {
+			iconsCahceMu.RUnlock()
+			return iconsCache, nil
+		}
+	}
+	iconsCahceMu.RUnlock()
+
+	iconsCahceMu.Lock()
+	defer iconsCahceMu.Unlock()
+
+	logging.Info().Msg("updating icon data")
+	icons, err := fetchIconData()
+	if err != nil {
+		return nil, err
+	}
+	logging.Info().
+		Int("icons", len(icons.IconList)).
+		Int("display_names", len(icons.DisplayNames)).
+		Msg("icons list updated")
+
+	iconsCache = icons
+	lastUpdate = time.Now()
+
+	err = utils.SaveJSON(common.IconListCachePath, iconsCache, 0o644)
+	if err != nil {
+		logging.Warn().Err(err).Msg("failed to save icon list cache")
+	}
+	return icons, nil
+}
+
+func SearchIcons(keyword string, limit int) ([]string, error) {
+	icons, err := ListAvailableIcons()
+	if err != nil {
+		return nil, err
+	}
+	if keyword == "" {
+		return utils.Slice(icons.IconList, limit), nil
+	}
+	return utils.Slice(fuzzy.Find(keyword, icons.IconList), limit), nil
+}
+
+func HasWalkxCodeIcon(name string, filetype string) bool {
+	icons, err := ListAvailableIcons()
+	if err != nil {
+		logging.Error().Err(err).Msg("failed to list icons")
+		return false
+	}
+	if _, ok := icons.WalkxCode[filetype]; !ok {
+		return false
+	}
+	_, ok := icons.WalkxCode[filetype][name+"."+filetype]
+	return ok
+}
+
+func HasSelfhstIcon(name string, filetype string) bool {
+	icons, err := ListAvailableIcons()
+	if err != nil {
+		logging.Error().Err(err).Msg("failed to list icons")
+		return false
+	}
+	if _, ok := icons.Selfhst[filetype]; !ok {
+		return false
+	}
+	_, ok := icons.Selfhst[filetype][name+"."+filetype]
+	return ok
+}
+
+func GetDisplayName(reference string) (string, bool) {
+	icons, err := ListAvailableIcons()
+	if err != nil {
+		logging.Error().Err(err).Msg("failed to list icons")
+		return "", false
+	}
+	displayName, ok := icons.DisplayNames[reference]
+	return displayName, ok
+}
+
+func fetchIconData() (*Cache, error) {
+	walkxCodeIconMap, walkxCodeIconList, err := fetchWalkxCodeIcons()
+	if err != nil {
+		return nil, err
+	}
+
+	n := 0
+	for _, items := range walkxCodeIconMap {
+		n += len(items)
+	}
+
+	selfhstIconMap, selfhstIconList, referenceToNames, err := fetchSelfhstIcons()
+	if err != nil {
+		return nil, err
+	}
+
+	return &Cache{
+		WalkxCode:    walkxCodeIconMap,
+		Selfhst:      selfhstIconMap,
+		DisplayNames: referenceToNames,
+		IconList:     append(walkxCodeIconList, selfhstIconList...),
+	}, nil
+}
+
+/*
+format:
+
+	{
+		"png": [
+			"*.png",
+		],
+		"svg": [
+			"*.svg",
+		],
+		"webp": [
+			"*.webp",
+		]
+	}
+*/
+func fetchWalkxCodeIcons() (IconsMap, IconList, error) {
+	req, err := http.NewRequest(http.MethodGet, walkxcodeIcons, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	data := make(map[string][]string)
+	err = json.Unmarshal(body, &data)
+	if err != nil {
+		return nil, nil, err
+	}
+	icons := make(IconsMap, len(data))
+	iconList := make(IconList, 0, 2000)
+	for fileType, files := range data {
+		icons[fileType] = make(map[string]struct{}, len(files))
+		for _, icon := range files {
+			icons[fileType][icon] = struct{}{}
+			iconList = append(iconList, "@walkxcode/"+icon)
+		}
+	}
+	return icons, iconList, nil
+}
+
+/*
+format:
+
+	{
+			"Name": "2FAuth",
+			"Reference": "2fauth",
+			"SVG": "Yes",
+			"PNG": "Yes",
+			"WebP": "Yes",
+			"Light": "Yes",
+			"Category": "Self-Hosted",
+			"CreatedAt": "2024-08-16 00:27:23+00:00"
+	}
+*/
+func fetchSelfhstIcons() (IconsMap, IconList, ReferenceDisplayNameMap, error) {
+	type SelfhStIcon struct {
+		Name      string `json:"Name"`
+		Reference string `json:"Reference"`
+		SVG       string `json:"SVG"`
+		PNG       string `json:"PNG"`
+		WebP      string `json:"WebP"`
+		// Light          string
+		// Category       string
+		// CreatedAt      string
+	}
+
+	req, err := http.NewRequest(http.MethodGet, selfhstIcons, nil)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	data := make([]SelfhStIcon, 0, 2000)
+	err = json.Unmarshal(body, &data)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	iconList := make(IconList, 0, len(data)*3)
+	icons := make(IconsMap)
+	icons["svg"] = make(map[string]struct{}, len(data))
+	icons["png"] = make(map[string]struct{}, len(data))
+	icons["webp"] = make(map[string]struct{}, len(data))
+
+	referenceToNames := make(ReferenceDisplayNameMap, len(data))
+
+	for _, item := range data {
+		if item.SVG == "Yes" {
+			icons["svg"][item.Reference+".svg"] = struct{}{}
+			iconList = append(iconList, "@selfhst/"+item.Reference+".svg")
+		}
+		if item.PNG == "Yes" {
+			icons["png"][item.Reference+".png"] = struct{}{}
+			iconList = append(iconList, "@selfhst/"+item.Reference+".png")
+		}
+		if item.WebP == "Yes" {
+			icons["webp"][item.Reference+".webp"] = struct{}{}
+			iconList = append(iconList, "@selfhst/"+item.Reference+".webp")
+		}
+		referenceToNames[item.Reference] = item.Name
+	}
+
+	return icons, iconList, referenceToNames, nil
+}

internal/homepage/route.go

@@ -0,0 +1,19 @@
+package homepage
+
+import (
+	"net/http"
+
+	net "github.com/yusing/go-proxy/internal/net/types"
+)
+
+type route interface {
+	TargetName() string
+	ProviderName() string
+	Reference() string
+	TargetURL() *net.URL
+}
+
+type httpRoute interface {
+	route
+	http.Handler
+}