mirror of
https://github.com/WJQSERVER-STUDIO/ghproxy.git
synced 2026-02-03 00:01:10 +08:00
Merge 3.4.1 #110
This commit is contained in:
WJQSERVER
GitHub
commit
e40e1aadee
9 changed files with 215 additions and 29 deletions
16
CHANGELOG.md
16
CHANGELOG.md
|
|
@ -1,5 +1,21 @@
|
|||
# 更新日志
|
||||
|
||||
3.4.1 - 2025-05-29
|
||||
---
|
||||
- ADD: 为`errorpage`部分增加lru缓存, 避免重复渲染
|
||||
- CHANGE: 把json库替换到[sonic](github.com/bytedance/sonic)
|
||||
|
||||
25w41b - 2025-05-28
|
||||
---
|
||||
- PRE-RELEASE: 此版本是v3.4.1预发布版本,请勿在生产环境中使用;
|
||||
- CHANGE: 把json库替换到[sonic](github.com/bytedance/sonic)
|
||||
|
||||
25w41a - 2025-05-28
|
||||
---
|
||||
- PRE-RELEASE: 此版本是v3.4.1预发布版本,请勿在生产环境中使用;
|
||||
- ADD: 为`errorpage`部分增加lru缓存, 避免重复渲染
|
||||
- CHANGE: 替换到实验性的`encoding/json/v2`
|
||||
|
||||
3.4.0 - 2025-05-21
|
||||
---
|
||||
- ADD: 初步实现多`target` Docker代理
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
25w40b
|
||||
25w41b
|
||||
2
VERSION
2
VERSION
|
|
@ -1 +1 @@
|
|||
3.4.0
|
||||
3.4.1
|
||||
|
|
@ -1,12 +1,13 @@
|
|||
package auth
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"ghproxy/config"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
json "github.com/bytedance/sonic"
|
||||
)
|
||||
|
||||
type Blacklist struct {
|
||||
|
|
|
|||
|
|
@ -1,12 +1,13 @@
|
|||
package auth
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"ghproxy/config"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
json "github.com/bytedance/sonic"
|
||||
)
|
||||
|
||||
// Whitelist 用于存储白名单信息
|
||||
|
|
|
|||
11
go.mod
11
go.mod
|
|
@ -5,20 +5,23 @@ go 1.24.3
|
|||
require (
|
||||
github.com/BurntSushi/toml v1.5.0
|
||||
github.com/WJQSERVER-STUDIO/httpc v0.5.1
|
||||
github.com/WJQSERVER-STUDIO/logger v1.7.1
|
||||
github.com/WJQSERVER-STUDIO/logger v1.7.2
|
||||
github.com/cloudwego/hertz v0.10.0
|
||||
github.com/hertz-contrib/http2 v0.1.8
|
||||
golang.org/x/net v0.40.0
|
||||
golang.org/x/time v0.11.0
|
||||
)
|
||||
|
||||
require github.com/WJQSERVER-STUDIO/go-utils/limitreader v0.0.2
|
||||
require (
|
||||
github.com/WJQSERVER-STUDIO/go-utils/limitreader v0.0.2
|
||||
github.com/bytedance/sonic v1.13.2
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/WJQSERVER-STUDIO/go-utils/copyb v0.0.4 // indirect
|
||||
github.com/WJQSERVER-STUDIO/go-utils/log v0.0.3 // indirect
|
||||
github.com/bytedance/gopkg v0.1.2 // indirect
|
||||
github.com/bytedance/sonic v1.13.2 // indirect
|
||||
github.com/bytedance/sonic/loader v0.2.4 // indirect
|
||||
github.com/cloudwego/base64x v0.1.5 // indirect
|
||||
github.com/cloudwego/gopkg v0.1.4 // indirect
|
||||
|
|
@ -26,7 +29,7 @@ require (
|
|||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/google/go-cmp v0.7.0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
|
||||
github.com/nyaruka/phonenumbers v1.6.1 // indirect
|
||||
github.com/nyaruka/phonenumbers v1.6.3 // indirect
|
||||
github.com/tidwall/gjson v1.18.0 // indirect
|
||||
github.com/tidwall/match v1.1.1 // indirect
|
||||
github.com/tidwall/pretty v1.2.1 // indirect
|
||||
|
|
|
|||
10
go.sum
10
go.sum
|
|
@ -8,8 +8,8 @@ github.com/WJQSERVER-STUDIO/go-utils/log v0.0.3 h1:t6nyLhmo9pSfVHm1Wu1WyLsTpXFSj
|
|||
github.com/WJQSERVER-STUDIO/go-utils/log v0.0.3/go.mod h1:j9Q+xnwpOfve7/uJnZ2izRQw6NNoXjvJHz7vUQAaLZE=
|
||||
github.com/WJQSERVER-STUDIO/httpc v0.5.1 h1:+TKCPYBuj7PAHuiduGCGAqsHAa4QtsUfoVwRN777q64=
|
||||
github.com/WJQSERVER-STUDIO/httpc v0.5.1/go.mod h1:M7KNUZjjhCkzzcg9lBPs9YfkImI+7vqjAyjdA19+joE=
|
||||
github.com/WJQSERVER-STUDIO/logger v1.7.1 h1:sAFsF3umimY0Vmue5WnGf1Qxvm/vlhK2srZakWVtlFU=
|
||||
github.com/WJQSERVER-STUDIO/logger v1.7.1/go.mod h1:cvP0XdFIMLtDWOZeKhklshzipkVU1zufsU4rKNfoM24=
|
||||
github.com/WJQSERVER-STUDIO/logger v1.7.2 h1:Tu9WICwlrY+BMQmY7k4llDB1ziFtZ9VmK7/85VIPN+M=
|
||||
github.com/WJQSERVER-STUDIO/logger v1.7.2/go.mod h1:yzXPtot0OvR1gzx4+rlFrv/sccUpz0gIXVBwUx3H7fM=
|
||||
github.com/bytedance/gopkg v0.1.1/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM=
|
||||
github.com/bytedance/gopkg v0.1.2 h1:8o2feYuxknDpN+O7kPwvSXfMEKfYvJYiA2K7aonoMEQ=
|
||||
github.com/bytedance/gopkg v0.1.2/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM=
|
||||
|
|
@ -39,6 +39,8 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
|||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/hertz-contrib/http2 v0.1.8 h1:kjfCGkUxJZHgfPsnRjx1FLJBG55KvtvSQD214guBQLw=
|
||||
github.com/hertz-contrib/http2 v0.1.8/go.mod h1:m42hrl8fiTwE4p8c7JdRUZpkePEthvV89q3elL2GeD0=
|
||||
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
|
||||
|
|
@ -50,8 +52,8 @@ github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgSh
|
|||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/nyaruka/phonenumbers v1.6.1 h1:XAJcTdYow16VrVKfglznMpJZz8KMJoMjx/91sX+K940=
|
||||
github.com/nyaruka/phonenumbers v1.6.1/go.mod h1:7gjs+Lchqm49adhAKB5cdcng5ZXgt6x7Jgvi0ZorUtU=
|
||||
github.com/nyaruka/phonenumbers v1.6.3 h1:JU7Q30+UM/03/vto6Q4EiZfEuRpTVyXMqImIbI942Qw=
|
||||
github.com/nyaruka/phonenumbers v1.6.3/go.mod h1:7gjs+Lchqm49adhAKB5cdcng5ZXgt6x7Jgvi0ZorUtU=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
|
||||
|
|
|
|||
|
|
@ -2,9 +2,10 @@ package proxy
|
|||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
json "github.com/bytedance/sonic"
|
||||
|
||||
"ghproxy/config"
|
||||
"ghproxy/weakcache"
|
||||
"io"
|
||||
|
|
|
|||
194
proxy/error.go
194
proxy/error.go
|
|
@ -2,12 +2,18 @@ package proxy
|
|||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/gob"
|
||||
"encoding/hex"
|
||||
"sync"
|
||||
|
||||
"fmt"
|
||||
"html/template"
|
||||
"io/fs"
|
||||
|
||||
"github.com/WJQSERVER-STUDIO/logger"
|
||||
"github.com/cloudwego/hertz/pkg/app"
|
||||
lru "github.com/hashicorp/golang-lru/v2"
|
||||
)
|
||||
|
||||
// 日志模块
|
||||
|
|
@ -22,7 +28,7 @@ var (
|
|||
|
||||
func HandleError(c *app.RequestContext, message string) {
|
||||
ErrorPage(c, NewErrorWithStatusLookup(500, message))
|
||||
logError(message)
|
||||
logError("Error handled: %s", message)
|
||||
}
|
||||
|
||||
type GHProxyErrors struct {
|
||||
|
|
@ -123,6 +129,22 @@ type ErrorPageData struct {
|
|||
ErrorMessage string
|
||||
}
|
||||
|
||||
// ToCacheKey 为 ErrorPageData 生成一个唯一的 SHA256 字符串键。
|
||||
// 使用 gob 序列化来确保结构体内容到字节序列的顺序一致性，然后计算哈希。
|
||||
func (d ErrorPageData) ToCacheKey() string {
|
||||
var buf bytes.Buffer
|
||||
enc := gob.NewEncoder(&buf)
|
||||
err := enc.Encode(d)
|
||||
if err != nil {
|
||||
logError("Failed to gob encode ErrorPageData for cache key: %v", err)
|
||||
return ""
|
||||
}
|
||||
|
||||
hasher := sha256.New()
|
||||
hasher.Write(buf.Bytes())
|
||||
return hex.EncodeToString(hasher.Sum(nil))
|
||||
}
|
||||
|
||||
func ErrPageUnwarper(errInfo *GHProxyErrors) ErrorPageData {
|
||||
return ErrorPageData{
|
||||
StatusCode: errInfo.StatusCode,
|
||||
|
|
@ -133,25 +155,130 @@ func ErrPageUnwarper(errInfo *GHProxyErrors) ErrorPageData {
|
|||
}
|
||||
}
|
||||
|
||||
func ErrorPage(c *app.RequestContext, errInfo *GHProxyErrors) {
|
||||
pageData, err := htmlTemplateRender(errPagesFs, ErrPageUnwarper(errInfo))
|
||||
if err != nil {
|
||||
c.JSON(errInfo.StatusCode, map[string]string{"error": errInfo.ErrorMessage})
|
||||
logDebug("Error reading page.tmpl: %v", err)
|
||||
return
|
||||
}
|
||||
c.Data(errInfo.StatusCode, "text/html; charset=utf-8", pageData)
|
||||
return
|
||||
// SizedLRUCache 实现了基于字节大小限制的 LRU 缓存。
|
||||
// 它包装了 hashicorp/golang-lru/v2.Cache，并额外管理缓存的总字节大小。
|
||||
type SizedLRUCache struct {
|
||||
cache *lru.Cache[string, []byte]
|
||||
mu sync.Mutex // 保护 currentBytes 字段
|
||||
maxBytes int64 // 缓存的最大字节容量
|
||||
currentBytes int64 // 缓存当前占用的字节数
|
||||
}
|
||||
|
||||
func htmlTemplateRender(fsys fs.FS, data interface{}) ([]byte, error) {
|
||||
tmplPath := "page.tmpl"
|
||||
tmpl, err := template.ParseFS(fsys, tmplPath)
|
||||
// NewSizedLRUCache 创建一个新的 SizedLRUCache 实例。
|
||||
// 内部的 lru.Cache 的条目容量被设置为一个较大的值 (例如 10000)，
|
||||
// 因为主要的逐出逻辑将由字节大小限制来控制。
|
||||
func NewSizedLRUCache(maxBytes int64) (*SizedLRUCache, error) {
|
||||
if maxBytes <= 0 {
|
||||
return nil, fmt.Errorf("maxBytes must be positive")
|
||||
}
|
||||
|
||||
c := &SizedLRUCache{
|
||||
maxBytes: maxBytes,
|
||||
}
|
||||
|
||||
// 创建内部 LRU 缓存，并提供一个 OnEvictedFunc 回调函数。
|
||||
// 当内部 LRU 缓存因其自身的条目容量限制或 RemoveOldest 方法被调用而逐出条目时，
|
||||
// 此回调函数会被执行，从而更新 currentBytes。
|
||||
var err error
|
||||
c.cache, err = lru.NewWithEvict[string, []byte](10000, func(key string, value []byte) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.currentBytes -= int64(len(value))
|
||||
logDebug("LRU evicted key: %s, size: %d, current total: %d", key, len(value), c.currentBytes)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error parsing template: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// Get 从缓存中检索值。
|
||||
func (c *SizedLRUCache) Get(key string) ([]byte, bool) {
|
||||
return c.cache.Get(key)
|
||||
}
|
||||
|
||||
// Add 向缓存中添加或更新一个键值对，并在必要时执行逐出以满足字节限制。
|
||||
func (c *SizedLRUCache) Add(key string, value []byte) {
|
||||
c.mu.Lock() // 保护 currentBytes 和逐出逻辑
|
||||
defer c.mu.Unlock()
|
||||
|
||||
itemSize := int64(len(value))
|
||||
|
||||
// 如果待添加的条目本身就大于缓存的最大容量，则不进行缓存。
|
||||
if itemSize > c.maxBytes {
|
||||
logWarning("Item key %s (size %d) larger than cache max capacity %d. Not caching.", key, itemSize, c.maxBytes)
|
||||
return
|
||||
}
|
||||
|
||||
// 如果键已存在，则首先从 currentBytes 中减去旧值的大小，并从内部 LRU 中移除旧条目。
|
||||
if oldVal, ok := c.cache.Get(key); ok {
|
||||
c.currentBytes -= int64(len(oldVal))
|
||||
c.cache.Remove(key)
|
||||
logDebug("Key %s exists, removed old size %d. Current total: %d", key, len(oldVal), c.currentBytes)
|
||||
}
|
||||
|
||||
// 主动逐出最旧的条目，直到有足够的空间容纳新条目。
|
||||
for c.currentBytes+itemSize > c.maxBytes && c.cache.Len() > 0 {
|
||||
_, oldVal, existed := c.cache.RemoveOldest()
|
||||
if !existed {
|
||||
logWarning("Attempted to remove oldest, but item not found.")
|
||||
break
|
||||
}
|
||||
logDebug("Proactively evicted item (size %d) to free space. Current total: %d", len(oldVal), c.currentBytes)
|
||||
}
|
||||
|
||||
// 添加新条目到内部 LRU 缓存。
|
||||
c.cache.Add(key, value)
|
||||
c.currentBytes += itemSize // 手动增加新条目的大小到 currentBytes。
|
||||
logDebug("Item added: key %s, size: %d, current total: %d", key, itemSize, c.currentBytes)
|
||||
}
|
||||
|
||||
const maxErrorPageCacheBytes = 512 * 1024 // 错误页面缓存的最大容量：512KB
|
||||
|
||||
var errorPageCache *SizedLRUCache
|
||||
|
||||
func init() {
|
||||
// 初始化 SizedLRUCache。
|
||||
var err error
|
||||
errorPageCache, err = NewSizedLRUCache(maxErrorPageCacheBytes)
|
||||
if err != nil {
|
||||
logError("Failed to initialize error page LRU cache: %v", err)
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
// parsedTemplateOnce 用于确保 HTML 模板只被解析一次。
|
||||
var (
|
||||
parsedTemplateOnce sync.Once
|
||||
parsedTemplate *template.Template
|
||||
parsedTemplateErr error
|
||||
)
|
||||
|
||||
// getParsedTemplate 用于获取缓存的解析后的 HTML 模板。
|
||||
func getParsedTemplate() (*template.Template, error) {
|
||||
parsedTemplateOnce.Do(func() {
|
||||
tmplPath := "page.tmpl"
|
||||
// 确保 errPagesFs 已初始化。这要求在任何 ErrorPage 调用之前调用 InitErrPagesFS。
|
||||
if errPagesFs == nil {
|
||||
parsedTemplateErr = fmt.Errorf("errPagesFs not initialized. Call InitErrPagesFS first")
|
||||
return
|
||||
}
|
||||
parsedTemplate, parsedTemplateErr = template.ParseFS(errPagesFs, tmplPath)
|
||||
if parsedTemplateErr != nil {
|
||||
parsedTemplate = nil
|
||||
}
|
||||
})
|
||||
return parsedTemplate, parsedTemplateErr
|
||||
}
|
||||
|
||||
// htmlTemplateRender 修改为使用缓存的模板。
|
||||
func htmlTemplateRender(data interface{}) ([]byte, error) {
|
||||
tmpl, err := getParsedTemplate()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get parsed template: %w", err)
|
||||
}
|
||||
if tmpl == nil {
|
||||
return nil, fmt.Errorf("template is nil")
|
||||
return nil, fmt.Errorf("template is nil after parsing")
|
||||
}
|
||||
|
||||
// 创建一个 bytes.Buffer 用于存储渲染结果
|
||||
|
|
@ -159,9 +286,44 @@ func htmlTemplateRender(fsys fs.FS, data interface{}) ([]byte, error) {
|
|||
|
||||
err = tmpl.Execute(&buf, data)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error executing template: %w", err)
|
||||
return nil, fmt.Errorf("failed to execute template: %w", err)
|
||||
}
|
||||
|
||||
// 返回 buffer 的内容作为 []byte
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
func ErrorPage(c *app.RequestContext, errInfo *GHProxyErrors) {
|
||||
// 将 errInfo 转换为 ErrorPageData 结构体
|
||||
pageDataStruct := ErrPageUnwarper(errInfo)
|
||||
// 使用 ErrorPageData 生成一个唯一的 SHA256 缓存键
|
||||
cacheKey := pageDataStruct.ToCacheKey()
|
||||
if cacheKey == "" {
|
||||
c.JSON(errInfo.StatusCode, map[string]string{"error": errInfo.ErrorMessage})
|
||||
logWarning("Failed to generate cache key for error page: %v", errInfo)
|
||||
return
|
||||
}
|
||||
|
||||
var pageData []byte
|
||||
var err error
|
||||
|
||||
// 尝试从缓存中获取页面数据
|
||||
if cachedPage, found := errorPageCache.Get(cacheKey); found {
|
||||
pageData = cachedPage
|
||||
logDebug("Serving error page from cache (Key: %s)", cacheKey)
|
||||
} else {
|
||||
// 如果不在缓存中，则渲染页面
|
||||
pageData, err = htmlTemplateRender(pageDataStruct)
|
||||
if err != nil {
|
||||
c.JSON(errInfo.StatusCode, map[string]string{"error": errInfo.ErrorMessage})
|
||||
logWarning("Failed to render error page for status %d (Key: %s): %v", errInfo.StatusCode, cacheKey, err)
|
||||
return
|
||||
}
|
||||
|
||||
// 将渲染结果存入缓存
|
||||
errorPageCache.Add(cacheKey, pageData)
|
||||
logDebug("Cached error page (Key: %s, Size: %d bytes)", cacheKey, len(pageData))
|
||||
}
|
||||
|
||||
c.Data(errInfo.StatusCode, "text/html; charset=utf-8", pageData)
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue