diff --git a/CHANGELOG.md b/CHANGELOG.md index 97f9876..6f57a7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # 更新日志 +25w14t-2 +--- +- PRE-RELEASE: 此版本是测试验证版本,请勿在生产环境中使用; +- CHANGE: 使用`touka-httpc`封装`HTTP Client`，更新到`v0.1.0`版本, 参看`touka-httpc` +- CHANGE: 重构`whitelist`实现 +- CHANGE: 对`proxy`进行结构性调整 +- CHANGE: `chunckedreq`与`gitreq`共用`BufferPool`和`HTTP Client` +- CHANGE: 新增`HTTP Client`配置块 + 25w14t-1 --- - PRE-RELEASE: 此版本是测试验证版本,请勿在生产环境中使用; diff --git a/DEV-VERSION b/DEV-VERSION index cca0ffd..b0f9cf0 100644 --- a/DEV-VERSION +++ b/DEV-VERSION @@ -1 +1 @@ -25w14t-1 \ No newline at end of file +25w14t-2 \ No newline at end of file diff --git a/README.md b/README.md index bf604ad..97b1c8a 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,12 @@ port = 8080 # 监听端口 sizeLimit = 125 # 125MB enableH2C = "on" # 是否开启H2C传输(latest和dev版本请开启) on/off +[httpc] +mode = "auto" # "auto" or "advanced" HTTP客户端模式 自动/高级模式 +maxIdleConns = 100 # only for advanced mode 仅用于高级模式 +maxIdleConnsPerHost = 60 # only for advanced mode 仅用于高级模式 +maxConnsPerHost = 0 # only for advanced mode 仅用于高级模式 + [pages] enabled = false # 是否开启内置静态页面(Docker版本请关闭此项) staticPath = "/data/www" # 静态页面文件路径 diff --git a/auth/auth.go b/auth/auth.go index fdc5833..99cb6de 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -18,10 +18,18 @@ var ( func Init(cfg *config.Config) { if cfg.Blacklist.Enabled { - InitBlacklist(cfg) + err := InitBlacklist(cfg) + if err != nil { + logError(err.Error()) + return + } } if cfg.Whitelist.Enabled { - LoadWhitelist(cfg) + err := InitWhitelist(cfg) + if err != nil { + logError(err.Error()) + return + } } logDebug("Auth Init") } diff --git a/auth/whitelist.go b/auth/whitelist.go index 000b016..ee93c20 100644 --- a/auth/whitelist.go +++ b/auth/whitelist.go @@ -2,58 +2,90 @@ package auth import ( "encoding/json" + "fmt" "ghproxy/config" "os" "strings" + "sync" ) -type WhitelistConfig struct { - Whitelist []string `json:"whitelist"` +// Whitelist 用于存储白名单信息 +type Whitelist struct { + userSet map[string]struct{} // 用户级白名单 + repoSet map[string]map[string]struct{} // 仓库级白名单 + initOnce sync.Once // 确保初始化只执行一次 + initialized bool // 初始化状态标识 } var ( - whitelistfile = "/data/ghproxy/config/whitelist.json" - whitelist *WhitelistConfig + whitelistInstance *Whitelist + whitelistInitErr error ) -func LoadWhitelist(cfg *config.Config) { - whitelistfile = cfg.Whitelist.WhitelistFile - whitelist = &WhitelistConfig{} +// InitWhitelist 初始化白名单（线程安全，仅执行一次） +func InitWhitelist(cfg *config.Config) error { + whitelistInstance = &Whitelist{ + userSet: make(map[string]struct{}), + repoSet: make(map[string]map[string]struct{}), + } - data, err := os.ReadFile(whitelistfile) + data, err := os.ReadFile(cfg.Whitelist.WhitelistFile) if err != nil { - logError("Failed to read whitelist file: %v", err) + return fmt.Errorf("failed to read whitelist: %w", err) } - err = json.Unmarshal(data, whitelist) - if err != nil { - logError("Failed to unmarshal whitelist JSON: %v", err) + var list struct { + Entries []string `json:"whitelist"` } -} - -func CheckWhitelist(fullrepo string, user string, repo string) bool { - return forRangeCheckWhitelist(whitelist.Whitelist, fullrepo, user) -} - -func sliceRepoName_Whitelist(fullrepo string) (string, string) { - s := strings.Split(fullrepo, "/") - if len(s) != 2 { - return "", "" + if err := json.Unmarshal(data, &list); err != nil { + return fmt.Errorf("invalid whitelist format: %w", err) } - return s[0], s[1] -} -func forRangeCheckWhitelist(wlist []string, fullrepo string, user string) bool { - for _, passd := range wlist { - users, _ := sliceRepoName_Whitelist(passd) - if users == user { - if strings.HasSuffix(passd, "/*") { - return true - } - if fullrepo == passd { - return true + for _, entry := range list.Entries { + user, repo := splitUserRepoWhitelist(entry) + switch { + case repo == "" || repo == "*": + whitelistInstance.userSet[user] = struct{}{} + default: + if _, exists := whitelistInstance.repoSet[user]; !exists { + whitelistInstance.repoSet[user] = make(map[string]struct{}) } + whitelistInstance.repoSet[user][repo] = struct{}{} } } + + whitelistInstance.initialized = true + return nil +} + +// CheckWhitelist 检查用户和仓库是否在白名单中（无锁设计） +func CheckWhitelist(username, repo string) bool { + if whitelistInstance == nil || !whitelistInstance.initialized { + return false + } + + // 先检查用户级白名单 + if _, exists := whitelistInstance.userSet[username]; exists { + return true + } + + // 再检查仓库级白名单 + if repos, userExists := whitelistInstance.repoSet[username]; userExists { + // 允许仓库名为空时的全用户仓库匹配 + if repo == "" { + return true + } + _, repoExists := repos[repo] + return repoExists + } + return false } + +// splitUserRepoWhitelist 分割用户和仓库信息（仅初始化时使用） +func splitUserRepoWhitelist(fullRepo string) (user, repo string) { + if idx := strings.Index(fullRepo, "/"); idx > 0 { + return fullRepo[:idx], fullRepo[idx+1:] + } + return fullRepo, "" +} diff --git a/config/config.go b/config/config.go index 9335b14..bad1557 100644 --- a/config/config.go +++ b/config/config.go @@ -6,6 +6,7 @@ import ( type Config struct { Server ServerConfig + Httpc HttpcConfig Pages PagesConfig Log LogConfig CORS CORSConfig @@ -24,6 +25,20 @@ type ServerConfig struct { Debug bool `toml:"debug"` } +/* +[httpc] +mode = "auto" # "auto" or "advanced" +maxIdleConns = 100 # only for advanced mode +maxIdleConnsPerHost = 60 # only for advanced mode +maxConnsPerHost = 0 # only for advanced mode +*/ +type HttpcConfig struct { + Mode string `toml:"mode"` + MaxIdleConns int `toml:"maxIdleConns"` + MaxIdleConnsPerHost int `toml:"maxIdleConnsPerHost"` + MaxConnsPerHost int `toml:"maxConnsPerHost"` +} + type PagesConfig struct { Enabled bool `toml:"enabled"` StaticDir string `toml:"staticDir"` diff --git a/config/config.toml b/config/config.toml index 5613f2e..12f1422 100644 --- a/config/config.toml +++ b/config/config.toml @@ -5,6 +5,12 @@ sizeLimit = 125 # MB enableH2C = "on" # "on" or "off" debug = false +[httpc] +mode = "auto" # "auto" or "advanced" +maxIdleConns = 100 # only for advanced mode +maxIdleConnsPerHost = 60 # only for advanced mode +maxConnsPerHost = 0 # only for advanced mode + [pages] enabled = false staticDir = "/data/www" diff --git a/deploy/config.toml b/deploy/config.toml index c217b5d..bdbcfcf 100644 --- a/deploy/config.toml +++ b/deploy/config.toml @@ -5,6 +5,12 @@ sizeLimit = 125 # MB enableH2C = "on" debug = false +[httpc] +mode = "auto" # "auto" or "advanced" +maxIdleConns = 100 # only for advanced mode +maxIdleConnsPerHost = 60 # only for advanced mode +maxConnsPerHost = 0 # only for advanced mode + [pages] enabled = false staticDir = "/usr/local/ghproxy/pages" diff --git a/go.mod b/go.mod index 73e5d65..0447312 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/BurntSushi/toml v1.4.0 github.com/WJQSERVER-STUDIO/go-utils/logger v1.3.0 github.com/gin-gonic/gin v1.10.0 - github.com/satomitouka/touka-httpc v0.0.3 + github.com/satomitouka/touka-httpc v0.1.0 golang.org/x/net v0.35.0 golang.org/x/time v0.10.0 ) @@ -19,7 +19,7 @@ require ( github.com/gin-contrib/sse v1.0.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.24.0 // indirect + github.com/go-playground/validator/v10 v10.25.0 // indirect github.com/goccy/go-json v0.10.5 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/json-iterator/go v1.1.12 // indirect diff --git a/go.sum b/go.sum index 897730f..f54b7f8 100644 --- a/go.sum +++ b/go.sum @@ -27,6 +27,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.24.0 h1:KHQckvo8G6hlWnrPX4NJJ+aBfWNAE/HH+qdL2cBpCmg= github.com/go-playground/validator/v10 v10.24.0/go.mod h1:GGzBIJMuE98Ic/kJsBXbz1x/7cByt++cQ+YOuDM5wus= +github.com/go-playground/validator/v10 v10.25.0 h1:5Dh7cjvzR7BRZadnsVOzPhWsrwUr0nmsZJxEAnFLNO8= +github.com/go-playground/validator/v10 v10.25.0/go.mod h1:GGzBIJMuE98Ic/kJsBXbz1x/7cByt++cQ+YOuDM5wus= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= @@ -53,6 +55,18 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/satomitouka/touka-httpc v0.0.3 h1:SLb14DWBIDeIaNQ0wMwRwJMjUDakHVR1Jbdct3Qi8fA= github.com/satomitouka/touka-httpc v0.0.3/go.mod h1:ULB/0Ze0Apm46YKl35Jmj1hW5YLVVeOGqCqn+ijqGPM= +github.com/satomitouka/touka-httpc v0.0.4 h1:sZs/2kqTSyLQ/pDHs/71l7MSG46j4rZNKfqn3CFAboU= +github.com/satomitouka/touka-httpc v0.0.4/go.mod h1:ULB/0Ze0Apm46YKl35Jmj1hW5YLVVeOGqCqn+ijqGPM= +github.com/satomitouka/touka-httpc v0.0.5 h1:ov1v29vrjvwRNbGqFJHmrCp+3/qXLoyWubO4kTDvb28= +github.com/satomitouka/touka-httpc v0.0.5/go.mod h1:ULB/0Ze0Apm46YKl35Jmj1hW5YLVVeOGqCqn+ijqGPM= +github.com/satomitouka/touka-httpc v0.0.6 h1:1iSaTB9KpviXy2NHvMXuRzy5mkcvle+fktWPhpS907c= +github.com/satomitouka/touka-httpc v0.0.6/go.mod h1:ULB/0Ze0Apm46YKl35Jmj1hW5YLVVeOGqCqn+ijqGPM= +github.com/satomitouka/touka-httpc v0.0.7 h1:igoLqXs6R1yNIdKMcfpwRB1l6KLLus6DvWT3xL1T5FY= +github.com/satomitouka/touka-httpc v0.0.7/go.mod h1:ULB/0Ze0Apm46YKl35Jmj1hW5YLVVeOGqCqn+ijqGPM= +github.com/satomitouka/touka-httpc v0.0.8 h1:KW521Z2z9BarnTgCajug/W/tIbnoIH+CzA7CON19iAg= +github.com/satomitouka/touka-httpc v0.0.8/go.mod h1:ULB/0Ze0Apm46YKl35Jmj1hW5YLVVeOGqCqn+ijqGPM= +github.com/satomitouka/touka-httpc v0.1.0 h1:CXCsr6NhdskK/W/ezvhwK2CP8QGCxewkBhsEjrM7K8s= +github.com/satomitouka/touka-httpc v0.1.0/go.mod h1:ULB/0Ze0Apm46YKl35Jmj1hW5YLVVeOGqCqn+ijqGPM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= diff --git a/proxy/chunkreq.go b/proxy/chunkreq.go index a315e5c..313dd41 100644 --- a/proxy/chunkreq.go +++ b/proxy/chunkreq.go @@ -7,66 +7,15 @@ import ( "io" "net/http" "strconv" - "sync" - "time" "github.com/gin-gonic/gin" - httpc "github.com/satomitouka/touka-httpc" ) -var BufferSize int = 32 * 1024 // 32KB - -var ( - ctr *http.Transport - BufferPool *sync.Pool - cclient *httpc.Client -) - -func InitReq(cfg *config.Config) { - initChunkedHTTPClient(cfg) - initGitHTTPClient(cfg) - - // 初始化固定大小的缓存池 - BufferPool = &sync.Pool{ - New: func() interface{} { - return make([]byte, BufferSize) - }, - } -} - -func initChunkedHTTPClient(cfg *config.Config) { - /* - ctr = &http.Transport{ - MaxIdleConns: 100, - MaxConnsPerHost: 60, - IdleConnTimeout: 20 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - ResponseHeaderTimeout: 10 * time.Second, - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - }).DialContext, - } - */ - ctr = &http.Transport{ - MaxIdleConns: 100, - MaxConnsPerHost: 60, - IdleConnTimeout: 20 * time.Second, - } - if cfg.Outbound.Enabled { - initTransport(cfg, ctr) - } - cclient = httpc.New( - httpc.WithTransport(ctr), - ) -} - func ChunkedProxyRequest(c *gin.Context, u string, cfg *config.Config, mode string, runMode string) { method := c.Request.Method // 发送HEAD请求, 预获取Content-Length - headReq, err := cclient.NewRequest("HEAD", u, nil) + headReq, err := client.NewRequest("HEAD", u, nil) if err != nil { HandleError(c, fmt.Sprintf("Failed to create request: %v", err)) return @@ -75,7 +24,7 @@ func ChunkedProxyRequest(c *gin.Context, u string, cfg *config.Config, mode stri removeWSHeader(headReq) // 删除Conection Upgrade头, 避免与HTTP/2冲突(检查是否存在Upgrade头) AuthPassThrough(c, cfg, headReq) - headResp, err := cclient.Do(headReq) + headResp, err := client.Do(headReq) if err != nil { HandleError(c, fmt.Sprintf("Failed to send request: %v", err)) return @@ -107,7 +56,7 @@ func ChunkedProxyRequest(c *gin.Context, u string, cfg *config.Config, mode stri bodyReader := bytes.NewBuffer(body) - req, err := cclient.NewRequest(method, u, bodyReader) + req, err := client.NewRequest(method, u, bodyReader) if err != nil { HandleError(c, fmt.Sprintf("Failed to create request: %v", err)) return @@ -116,7 +65,7 @@ func ChunkedProxyRequest(c *gin.Context, u string, cfg *config.Config, mode stri removeWSHeader(req) // 删除Conection Upgrade头, 避免与HTTP/2冲突(检查是否存在Upgrade头) AuthPassThrough(c, cfg, req) - resp, err := cclient.Do(req) + resp, err := client.Do(req) if err != nil { HandleError(c, fmt.Sprintf("Failed to send request: %v", err)) return diff --git a/proxy/gitreq.go b/proxy/gitreq.go index 4ceeac9..3b2c13f 100644 --- a/proxy/gitreq.go +++ b/proxy/gitreq.go @@ -7,42 +7,16 @@ import ( "io" "net/http" "strconv" - "time" "github.com/gin-gonic/gin" - httpc "github.com/satomitouka/touka-httpc" ) -var ( - gclient *httpc.Client - gtr *http.Transport -) - -func initGitHTTPClient(cfg *config.Config) { - gtr = &http.Transport{ - MaxIdleConns: 30, - MaxConnsPerHost: 30, - IdleConnTimeout: 30 * time.Second, - } - if cfg.Outbound.Enabled { - initTransport(cfg, gtr) - } - /* - gclient = &http.Client{ - Transport: gtr, - } - */ - gclient = httpc.New( - httpc.WithTransport(gtr), - ) -} - func GitReq(c *gin.Context, u string, cfg *config.Config, mode string, runMode string) { method := c.Request.Method logInfo("%s %s %s %s %s", c.ClientIP(), method, u, c.Request.Header.Get("User-Agent"), c.Request.Proto) // 发送HEAD请求, 预获取Content-Length - headReq, err := gclient.NewRequest("HEAD", u, nil) + headReq, err := client.NewRequest("HEAD", u, nil) if err != nil { HandleError(c, fmt.Sprintf("Failed to create request: %v", err)) return @@ -50,7 +24,7 @@ func GitReq(c *gin.Context, u string, cfg *config.Config, mode string, runMode s setRequestHeaders(c, headReq) AuthPassThrough(c, cfg, headReq) - headResp, err := gclient.Do(headReq) + headResp, err := client.Do(headReq) if err != nil { HandleError(c, fmt.Sprintf("Failed to send request: %v", err)) return @@ -84,7 +58,7 @@ func GitReq(c *gin.Context, u string, cfg *config.Config, mode string, runMode s bodyReader := bytes.NewBuffer(body) // 创建请求 - req, err := gclient.NewRequest(method, u, bodyReader) + req, err := client.NewRequest(method, u, bodyReader) if err != nil { HandleError(c, fmt.Sprintf("Failed to create request: %v", err)) return @@ -92,7 +66,7 @@ func GitReq(c *gin.Context, u string, cfg *config.Config, mode string, runMode s setRequestHeaders(c, req) AuthPassThrough(c, cfg, req) - resp, err := gclient.Do(req) + resp, err := client.Do(req) if err != nil { HandleError(c, fmt.Sprintf("Failed to send request: %v", err)) return @@ -139,7 +113,15 @@ func GitReq(c *gin.Context, u string, cfg *config.Config, mode string, runMode s c.Status(resp.StatusCode) - if _, err := io.Copy(c.Writer, resp.Body); err != nil { - logError("%s %s %s %s %s Response-Copy-Error: %v", c.ClientIP(), method, u, c.Request.Header.Get("User-Agent"), c.Request.Proto, err) + // 使用固定32KB缓冲池 + buffer := BufferPool.Get().([]byte) + defer BufferPool.Put(buffer) + + _, err = io.CopyBuffer(c.Writer, resp.Body, buffer) + if err != nil { + logError("%s %s %s %s %s Failed to copy response body: %v", c.ClientIP(), method, u, c.Request.Header.Get("User-Agent"), c.Request.Proto, err) + return + } else { + c.Writer.Flush() // 确保刷入 } } diff --git a/proxy/handler.go b/proxy/handler.go index 480aa67..f999b47 100644 --- a/proxy/handler.go +++ b/proxy/handler.go @@ -61,7 +61,7 @@ func NoRouteHandler(cfg *config.Config, limiter *rate.RateLimiter, iplimiter *ra // 白名单检查 if cfg.Whitelist.Enabled { - whitelist := auth.CheckWhitelist(repouser, username, repo) + whitelist := auth.CheckWhitelist(username, repo) if !whitelist { logErrMsg := fmt.Sprintf("%s %s %s %s %s Whitelist Blocked repo: %s", c.ClientIP(), c.Request.Method, rawPath, c.Request.Header.Get("User-Agent"), c.Request.Proto, repouser) errMsg := fmt.Sprintf("Whitelist Blocked repo: %s", repouser) diff --git a/proxy/httpc.go b/proxy/httpc.go new file mode 100644 index 0000000..8048821 --- /dev/null +++ b/proxy/httpc.go @@ -0,0 +1,88 @@ +package proxy + +import ( + "fmt" + "ghproxy/config" + "net/http" + "sync" + "time" + + httpc "github.com/satomitouka/touka-httpc" +) + +var BufferSize int = 32 * 1024 // 32KB + +var ( + tr *http.Transport + BufferPool *sync.Pool + client *httpc.Client +) + +func InitReq(cfg *config.Config) { + initHTTPClient(cfg) + + // 初始化固定大小的缓存池 + BufferPool = &sync.Pool{ + New: func() interface{} { + return make([]byte, BufferSize) + }, + } +} + +func initHTTPClient(cfg *config.Config) { + /* + ctr = &http.Transport{ + MaxIdleConns: 100, + MaxConnsPerHost: 60, + IdleConnTimeout: 20 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + ResponseHeaderTimeout: 10 * time.Second, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + } + */ + if cfg.Httpc.Mode == "auto" { + tr = &http.Transport{ + //MaxIdleConns: 160, + IdleConnTimeout: 30 * time.Second, + WriteBufferSize: 32 * 1024, // 32KB + ReadBufferSize: 32 * 1024, // 32KB + } + } else if cfg.Httpc.Mode == "advanced" { + tr = &http.Transport{ + MaxIdleConns: cfg.Httpc.MaxIdleConns, + MaxConnsPerHost: cfg.Httpc.MaxConnsPerHost, + MaxIdleConnsPerHost: cfg.Httpc.MaxIdleConnsPerHost, + WriteBufferSize: 32 * 1024, // 32KB + ReadBufferSize: 32 * 1024, // 32KB + } + } else { + // 错误的模式 + logError("unknown httpc mode: %s", cfg.Httpc.Mode) + fmt.Println("unknown httpc mode: ", cfg.Httpc.Mode) + logWarning("use Auto to Run HTTP Client") + fmt.Println("use Auto to Run HTTP Client") + tr = &http.Transport{ + //MaxIdleConns: 160, + IdleConnTimeout: 30 * time.Second, + WriteBufferSize: 32 * 1024, // 32KB + ReadBufferSize: 32 * 1024, // 32KB + } + } + if cfg.Outbound.Enabled { + initTransport(cfg, tr) + } + if cfg.Server.Debug { + client = httpc.New( + httpc.WithTransport(tr), + httpc.WithDumpLog(), + ) + } else { + client = httpc.New( + httpc.WithTransport(tr), + ) + } +}