diff --git a/CHANGELOG.md b/CHANGELOG.md index 1666a28..75b1830 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # 更新日志 +25w41a - 2025-05-28 +--- +- PRE-RELEASE: 此版本是v3.4.1预发布版本,请勿在生产环境中使用; +- ADD: 为`errorpage`部分增加lru缓存, 避免重复渲染 +- CHANGE: 替换到实验性的`encode/json/v2` + 3.4.0 - 2025-05-21 --- - ADD: 初步实现多`target` Docker代理 diff --git a/DEV-VERSION b/DEV-VERSION index 9c87980..cd3e78b 100644 --- a/DEV-VERSION +++ b/DEV-VERSION @@ -1 +1 @@ -25w40b \ No newline at end of file +25w41a \ No newline at end of file diff --git a/auth/blacklist.go b/auth/blacklist.go index ba2091b..014a41d 100644 --- a/auth/blacklist.go +++ b/auth/blacklist.go @@ -1,12 +1,13 @@ package auth import ( - "encoding/json" "fmt" "ghproxy/config" "os" "strings" "sync" + + "github.com/go-json-experiment/json" ) type Blacklist struct { diff --git a/auth/whitelist.go b/auth/whitelist.go index ee93c20..1218307 100644 --- a/auth/whitelist.go +++ b/auth/whitelist.go @@ -1,12 +1,13 @@ package auth import ( - "encoding/json" "fmt" "ghproxy/config" "os" "strings" "sync" + + "github.com/go-json-experiment/json" ) // Whitelist 用于存储白名单信息 diff --git a/go.mod b/go.mod index d4a930a..0c5cb9b 100644 --- a/go.mod +++ b/go.mod @@ -5,14 +5,18 @@ go 1.24.3 require ( github.com/BurntSushi/toml v1.5.0 github.com/WJQSERVER-STUDIO/httpc v0.5.1 - github.com/WJQSERVER-STUDIO/logger v1.7.1 + github.com/WJQSERVER-STUDIO/logger v1.7.2 github.com/cloudwego/hertz v0.10.0 github.com/hertz-contrib/http2 v0.1.8 golang.org/x/net v0.40.0 golang.org/x/time v0.11.0 ) -require github.com/WJQSERVER-STUDIO/go-utils/limitreader v0.0.2 +require ( + github.com/WJQSERVER-STUDIO/go-utils/limitreader v0.0.2 + github.com/go-json-experiment/json v0.0.0-20250517221953-25912455fbc8 + github.com/hashicorp/golang-lru/v2 v2.0.7 +) require ( github.com/WJQSERVER-STUDIO/go-utils/copyb v0.0.4 // indirect @@ -26,7 +30,7 @@ require ( github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/klauspost/cpuid/v2 v2.2.10 // indirect - github.com/nyaruka/phonenumbers v1.6.1 // indirect + github.com/nyaruka/phonenumbers v1.6.3 // indirect github.com/tidwall/gjson v1.18.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect diff --git a/go.sum b/go.sum index a614fc3..1d2dadf 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,8 @@ github.com/WJQSERVER-STUDIO/go-utils/log v0.0.3 h1:t6nyLhmo9pSfVHm1Wu1WyLsTpXFSj github.com/WJQSERVER-STUDIO/go-utils/log v0.0.3/go.mod h1:j9Q+xnwpOfve7/uJnZ2izRQw6NNoXjvJHz7vUQAaLZE= github.com/WJQSERVER-STUDIO/httpc v0.5.1 h1:+TKCPYBuj7PAHuiduGCGAqsHAa4QtsUfoVwRN777q64= github.com/WJQSERVER-STUDIO/httpc v0.5.1/go.mod h1:M7KNUZjjhCkzzcg9lBPs9YfkImI+7vqjAyjdA19+joE= -github.com/WJQSERVER-STUDIO/logger v1.7.1 h1:sAFsF3umimY0Vmue5WnGf1Qxvm/vlhK2srZakWVtlFU= -github.com/WJQSERVER-STUDIO/logger v1.7.1/go.mod h1:cvP0XdFIMLtDWOZeKhklshzipkVU1zufsU4rKNfoM24= +github.com/WJQSERVER-STUDIO/logger v1.7.2 h1:Tu9WICwlrY+BMQmY7k4llDB1ziFtZ9VmK7/85VIPN+M= +github.com/WJQSERVER-STUDIO/logger v1.7.2/go.mod h1:yzXPtot0OvR1gzx4+rlFrv/sccUpz0gIXVBwUx3H7fM= github.com/bytedance/gopkg v0.1.1/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= github.com/bytedance/gopkg v0.1.2 h1:8o2feYuxknDpN+O7kPwvSXfMEKfYvJYiA2K7aonoMEQ= github.com/bytedance/gopkg v0.1.2/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= @@ -35,10 +35,14 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/go-json-experiment/json v0.0.0-20250517221953-25912455fbc8 h1:o8UqXPI6SVwQt04RGsqKp3qqmbOfTNMqDrWsc4O47kk= +github.com/go-json-experiment/json v0.0.0-20250517221953-25912455fbc8/go.mod h1:TiCD2a1pcmjd7YnhGH0f/zKNcCD06B029pHhzV23c2M= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hertz-contrib/http2 v0.1.8 h1:kjfCGkUxJZHgfPsnRjx1FLJBG55KvtvSQD214guBQLw= github.com/hertz-contrib/http2 v0.1.8/go.mod h1:m42hrl8fiTwE4p8c7JdRUZpkePEthvV89q3elL2GeD0= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= @@ -50,8 +54,8 @@ github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgSh github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/nyaruka/phonenumbers v1.6.1 h1:XAJcTdYow16VrVKfglznMpJZz8KMJoMjx/91sX+K940= -github.com/nyaruka/phonenumbers v1.6.1/go.mod h1:7gjs+Lchqm49adhAKB5cdcng5ZXgt6x7Jgvi0ZorUtU= +github.com/nyaruka/phonenumbers v1.6.3 h1:JU7Q30+UM/03/vto6Q4EiZfEuRpTVyXMqImIbI942Qw= +github.com/nyaruka/phonenumbers v1.6.3/go.mod h1:7gjs+Lchqm49adhAKB5cdcng5ZXgt6x7Jgvi0ZorUtU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= diff --git a/proxy/docker.go b/proxy/docker.go index 3f4c43a..08b23c3 100644 --- a/proxy/docker.go +++ b/proxy/docker.go @@ -2,9 +2,10 @@ package proxy import ( "context" - "encoding/json" "fmt" + "github.com/go-json-experiment/json" + "ghproxy/config" "ghproxy/weakcache" "io" diff --git a/proxy/error.go b/proxy/error.go index 756c881..b684193 100644 --- a/proxy/error.go +++ b/proxy/error.go @@ -2,12 +2,18 @@ package proxy import ( "bytes" + "crypto/sha256" + "encoding/gob" + "encoding/hex" + "sync" + "fmt" "html/template" "io/fs" "github.com/WJQSERVER-STUDIO/logger" "github.com/cloudwego/hertz/pkg/app" + lru "github.com/hashicorp/golang-lru/v2" ) // 日志模块 @@ -22,7 +28,7 @@ var ( func HandleError(c *app.RequestContext, message string) { ErrorPage(c, NewErrorWithStatusLookup(500, message)) - logError(message) + logError("Error handled: %s", message) } type GHProxyErrors struct { @@ -123,6 +129,22 @@ type ErrorPageData struct { ErrorMessage string } +// ToCacheKey 为 ErrorPageData 生成一个唯一的 SHA256 字符串键。 +// 使用 gob 序列化来确保结构体内容到字节序列的顺序一致性，然后计算哈希。 +func (d ErrorPageData) ToCacheKey() string { + var buf bytes.Buffer + enc := gob.NewEncoder(&buf) + err := enc.Encode(d) + if err != nil { + logError("Failed to gob encode ErrorPageData for cache key: %v", err) + panic(fmt.Errorf("failed to gob encode ErrorPageData: %w", err)) + } + + hasher := sha256.New() + hasher.Write(buf.Bytes()) + return hex.EncodeToString(hasher.Sum(nil)) +} + func ErrPageUnwarper(errInfo *GHProxyErrors) ErrorPageData { return ErrorPageData{ StatusCode: errInfo.StatusCode, @@ -133,25 +155,130 @@ func ErrPageUnwarper(errInfo *GHProxyErrors) ErrorPageData { } } -func ErrorPage(c *app.RequestContext, errInfo *GHProxyErrors) { - pageData, err := htmlTemplateRender(errPagesFs, ErrPageUnwarper(errInfo)) - if err != nil { - c.JSON(errInfo.StatusCode, map[string]string{"error": errInfo.ErrorMessage}) - logDebug("Error reading page.tmpl: %v", err) - return - } - c.Data(errInfo.StatusCode, "text/html; charset=utf-8", pageData) - return +// SizedLRUCache 实现了基于字节大小限制的 LRU 缓存。 +// 它包装了 hashicorp/golang-lru/v2.Cache，并额外管理缓存的总字节大小。 +type SizedLRUCache struct { + cache *lru.Cache[string, []byte] + mu sync.Mutex // 保护 currentBytes 字段 + maxBytes int64 // 缓存的最大字节容量 + currentBytes int64 // 缓存当前占用的字节数 } -func htmlTemplateRender(fsys fs.FS, data interface{}) ([]byte, error) { - tmplPath := "page.tmpl" - tmpl, err := template.ParseFS(fsys, tmplPath) +// NewSizedLRUCache 创建一个新的 SizedLRUCache 实例。 +// 内部的 lru.Cache 的条目容量被设置为一个较大的值 (例如 10000)， +// 因为主要的逐出逻辑将由字节大小限制来控制。 +func NewSizedLRUCache(maxBytes int64) (*SizedLRUCache, error) { + if maxBytes <= 0 { + return nil, fmt.Errorf("maxBytes must be positive") + } + + c := &SizedLRUCache{ + maxBytes: maxBytes, + } + + // 创建内部 LRU 缓存，并提供一个 OnEvictedFunc 回调函数。 + // 当内部 LRU 缓存因其自身的条目容量限制或 RemoveOldest 方法被调用而逐出条目时， + // 此回调函数会被执行，从而更新 currentBytes。 + var err error + c.cache, err = lru.NewWithEvict[string, []byte](10000, func(key string, value []byte) { + c.mu.Lock() + defer c.mu.Unlock() + c.currentBytes -= int64(len(value)) + logDebug("LRU evicted key: %s, size: %d, current total: %d", key, len(value), c.currentBytes) + }) if err != nil { - return nil, fmt.Errorf("error parsing template: %w", err) + return nil, err + } + return c, nil +} + +// Get 从缓存中检索值。 +func (c *SizedLRUCache) Get(key string) ([]byte, bool) { + return c.cache.Get(key) +} + +// Add 向缓存中添加或更新一个键值对，并在必要时执行逐出以满足字节限制。 +func (c *SizedLRUCache) Add(key string, value []byte) { + c.mu.Lock() // 保护 currentBytes 和逐出逻辑 + defer c.mu.Unlock() + + itemSize := int64(len(value)) + + // 如果待添加的条目本身就大于缓存的最大容量，则不进行缓存。 + if itemSize > c.maxBytes { + logWarning("Item key %s (size %d) larger than cache max capacity %d. Not caching.", key, itemSize, c.maxBytes) + return + } + + // 如果键已存在，则首先从 currentBytes 中减去旧值的大小，并从内部 LRU 中移除旧条目。 + if oldVal, ok := c.cache.Get(key); ok { + c.currentBytes -= int64(len(oldVal)) + c.cache.Remove(key) + logDebug("Key %s exists, removed old size %d. Current total: %d", key, len(oldVal), c.currentBytes) + } + + // 主动逐出最旧的条目，直到有足够的空间容纳新条目。 + for c.currentBytes+itemSize > c.maxBytes && c.cache.Len() > 0 { + _, oldVal, existed := c.cache.RemoveOldest() + if !existed { + logWarning("Attempted to remove oldest, but item not found.") + break + } + logDebug("Proactively evicted item (size %d) to free space. Current total: %d", len(oldVal), c.currentBytes) + } + + // 添加新条目到内部 LRU 缓存。 + c.cache.Add(key, value) + c.currentBytes += itemSize // 手动增加新条目的大小到 currentBytes。 + logDebug("Item added: key %s, size: %d, current total: %d", key, itemSize, c.currentBytes) +} + +const maxErrorPageCacheBytes = 512 * 1024 // 错误页面缓存的最大容量：512KB + +var errorPageCache *SizedLRUCache + +func init() { + // 初始化 SizedLRUCache。 + var err error + errorPageCache, err = NewSizedLRUCache(maxErrorPageCacheBytes) + if err != nil { + logError("Failed to initialize error page LRU cache: %v", err) + panic(err) + } +} + +// parsedTemplateOnce 用于确保 HTML 模板只被解析一次。 +var ( + parsedTemplateOnce sync.Once + parsedTemplate *template.Template + parsedTemplateErr error +) + +// getParsedTemplate 用于获取缓存的解析后的 HTML 模板。 +func getParsedTemplate() (*template.Template, error) { + parsedTemplateOnce.Do(func() { + tmplPath := "page.tmpl" + // 确保 errPagesFs 已初始化。这要求在任何 ErrorPage 调用之前调用 InitErrPagesFS。 + if errPagesFs == nil { + parsedTemplateErr = fmt.Errorf("errPagesFs not initialized. Call InitErrPagesFS first") + return + } + parsedTemplate, parsedTemplateErr = template.ParseFS(errPagesFs, tmplPath) + if parsedTemplateErr != nil { + parsedTemplate = nil + } + }) + return parsedTemplate, parsedTemplateErr +} + +// htmlTemplateRender 修改为使用缓存的模板。 +func htmlTemplateRender(data interface{}) ([]byte, error) { + tmpl, err := getParsedTemplate() + if err != nil { + return nil, fmt.Errorf("failed to get parsed template: %w", err) } if tmpl == nil { - return nil, fmt.Errorf("template is nil") + return nil, fmt.Errorf("template is nil after parsing") } // 创建一个 bytes.Buffer 用于存储渲染结果 @@ -159,9 +286,39 @@ func htmlTemplateRender(fsys fs.FS, data interface{}) ([]byte, error) { err = tmpl.Execute(&buf, data) if err != nil { - return nil, fmt.Errorf("error executing template: %w", err) + return nil, fmt.Errorf("failed to execute template: %w", err) } // 返回 buffer 的内容作为 []byte return buf.Bytes(), nil } + +func ErrorPage(c *app.RequestContext, errInfo *GHProxyErrors) { + // 将 errInfo 转换为 ErrorPageData 结构体 + pageDataStruct := ErrPageUnwarper(errInfo) + // 使用 ErrorPageData 生成一个唯一的 SHA256 缓存键 + cacheKey := pageDataStruct.ToCacheKey() + + var pageData []byte + var err error + + // 尝试从缓存中获取页面数据 + if cachedPage, found := errorPageCache.Get(cacheKey); found { + pageData = cachedPage + logDebug("Serving error page from cache (Key: %s)", cacheKey) + } else { + // 如果不在缓存中，则渲染页面 + pageData, err = htmlTemplateRender(pageDataStruct) + if err != nil { + c.JSON(errInfo.StatusCode, map[string]string{"error": errInfo.ErrorMessage}) + logDebug("Failed to render error page for status %d (Key: %s): %v", errInfo.StatusCode, cacheKey, err) + return + } + + // 将渲染结果存入缓存 + errorPageCache.Add(cacheKey, pageData) + logDebug("Cached error page (Key: %s, Size: %d bytes)", cacheKey, len(pageData)) + } + + c.Data(errInfo.StatusCode, "text/html; charset=utf-8", pageData) +}