wjqserver/ghproxy
1
0
Fork 0
mirror of https://github.com/WJQSERVER-STUDIO/ghproxy.git synced 2026-02-03 00:01:10 +08:00
Code Issues Projects Releases Packages Wiki Activity Actions

25w14t-2

Browse source
This commit is contained in:
WJQSERVER 2025-02-16 19:48:53 +08:00
parent bd666e08d1
commit 785a74dfeb
14 changed files with 241 additions and 126 deletions
Download patch file Download diff file Expand all files Collapse all files

9
CHANGELOG.md
View file

@ -1,5 +1,14 @@
# 更新日志
25w14t-2
---
- PRE-RELEASE: 此版本是测试验证版本,请勿在生产环境中使用;
- CHANGE: 使用`touka-httpc`封装`HTTP Client`，更新到`v0.1.0`版本, 参看`touka-httpc`
- CHANGE: 重构`whitelist`实现
- CHANGE: 对`proxy`进行结构性调整
- CHANGE: `chunckedreq`与`gitreq`共用`BufferPool`和`HTTP Client`
- CHANGE: 新增`HTTP Client`配置块
25w14t-1
---
- PRE-RELEASE: 此版本是测试验证版本,请勿在生产环境中使用;

2
DEV-VERSION
View file

@ -1 +1 @@
25w14t-1
25w14t-2

6
README.md
View file

@ -91,6 +91,12 @@ port = 8080 # 监听端口
sizeLimit = 125 # 125MB
enableH2C = "on" # 是否开启H2C传输(latest和dev版本请开启) on/off
[httpc]
mode = "auto" # "auto" or "advanced" HTTP客户端模式 自动/高级模式
maxIdleConns = 100 # only for advanced mode 仅用于高级模式
maxIdleConnsPerHost = 60 # only for advanced mode 仅用于高级模式
maxConnsPerHost = 0 # only for advanced mode 仅用于高级模式
[pages]
enabled = false # 是否开启内置静态页面(Docker版本请关闭此项)
staticPath = "/data/www" # 静态页面文件路径

12
auth/auth.go
View file

@ -18,10 +18,18 @@ var (
func Init(cfg *config.Config) {
if cfg.Blacklist.Enabled {
InitBlacklist(cfg)
err := InitBlacklist(cfg)
if err != nil {
logError(err.Error())
return
}
}
if cfg.Whitelist.Enabled {
LoadWhitelist(cfg)
err := InitWhitelist(cfg)
if err != nil {
logError(err.Error())
return
}
}
logDebug("Auth Init")
}

98
auth/whitelist.go
View file

@ -2,58 +2,90 @@ package auth
import (
"encoding/json"
"fmt"
"ghproxy/config"
"os"
"strings"
"sync"
)
type WhitelistConfig struct {
Whitelist []string `json:"whitelist"`
// Whitelist 用于存储白名单信息
type Whitelist struct {
userSet map[string]struct{} // 用户级白名单
repoSet map[string]map[string]struct{} // 仓库级白名单
initOnce sync.Once // 确保初始化只执行一次
initialized bool // 初始化状态标识
}
var (
whitelistfile = "/data/ghproxy/config/whitelist.json"
whitelist *WhitelistConfig
whitelistInstance *Whitelist
whitelistInitErr error
)
func LoadWhitelist(cfg *config.Config) {
whitelistfile = cfg.Whitelist.WhitelistFile
whitelist = &WhitelistConfig{}
// InitWhitelist 初始化白名单（线程安全，仅执行一次）
func InitWhitelist(cfg *config.Config) error {
whitelistInstance = &Whitelist{
userSet: make(map[string]struct{}),
repoSet: make(map[string]map[string]struct{}),
}
data, err := os.ReadFile(whitelistfile)
data, err := os.ReadFile(cfg.Whitelist.WhitelistFile)
if err != nil {
logError("Failed to read whitelist file: %v", err)
return fmt.Errorf("failed to read whitelist: %w", err)
}
err = json.Unmarshal(data, whitelist)
if err != nil {
logError("Failed to unmarshal whitelist JSON: %v", err)
var list struct {
Entries []string `json:"whitelist"`
}
}
func CheckWhitelist(fullrepo string, user string, repo string) bool {
return forRangeCheckWhitelist(whitelist.Whitelist, fullrepo, user)
}
func sliceRepoName_Whitelist(fullrepo string) (string, string) {
s := strings.Split(fullrepo, "/")
if len(s) != 2 {
return "", ""
if err := json.Unmarshal(data, &list); err != nil {
return fmt.Errorf("invalid whitelist format: %w", err)
}
return s[0], s[1]
}
func forRangeCheckWhitelist(wlist []string, fullrepo string, user string) bool {
for _, passd := range wlist {
users, _ := sliceRepoName_Whitelist(passd)
if users == user {
if strings.HasSuffix(passd, "/*") {
return true
}
if fullrepo == passd {
return true
for _, entry := range list.Entries {
user, repo := splitUserRepoWhitelist(entry)
switch {
case repo == "" || repo == "*":
whitelistInstance.userSet[user] = struct{}{}
default:
if _, exists := whitelistInstance.repoSet[user]; !exists {
whitelistInstance.repoSet[user] = make(map[string]struct{})
}
whitelistInstance.repoSet[user][repo] = struct{}{}
}
}
whitelistInstance.initialized = true
return nil
}
// CheckWhitelist 检查用户和仓库是否在白名单中（无锁设计）
func CheckWhitelist(username, repo string) bool {
if whitelistInstance == nil || !whitelistInstance.initialized {
return false
}
// 先检查用户级白名单
if _, exists := whitelistInstance.userSet[username]; exists {
return true
}
// 再检查仓库级白名单
if repos, userExists := whitelistInstance.repoSet[username]; userExists {
// 允许仓库名为空时的全用户仓库匹配
if repo == "" {
return true
}
_, repoExists := repos[repo]
return repoExists
}
return false
}
// splitUserRepoWhitelist 分割用户和仓库信息（仅初始化时使用）
func splitUserRepoWhitelist(fullRepo string) (user, repo string) {
if idx := strings.Index(fullRepo, "/"); idx > 0 {
return fullRepo[:idx], fullRepo[idx+1:]
}
return fullRepo, ""
}

15
config/config.go
View file

@ -6,6 +6,7 @@ import (
type Config struct {
Server ServerConfig
Httpc HttpcConfig
Pages PagesConfig
Log LogConfig
CORS CORSConfig
@ -24,6 +25,20 @@ type ServerConfig struct {
Debug bool `toml:"debug"`
}
/*
[httpc]
mode = "auto" # "auto" or "advanced"
maxIdleConns = 100 # only for advanced mode
maxIdleConnsPerHost = 60 # only for advanced mode
maxConnsPerHost = 0 # only for advanced mode
*/
type HttpcConfig struct {
Mode string `toml:"mode"`
MaxIdleConns int `toml:"maxIdleConns"`
MaxIdleConnsPerHost int `toml:"maxIdleConnsPerHost"`
MaxConnsPerHost int `toml:"maxConnsPerHost"`
}
type PagesConfig struct {
Enabled bool `toml:"enabled"`
StaticDir string `toml:"staticDir"`

6
config/config.toml
View file

@ -5,6 +5,12 @@ sizeLimit = 125 # MB
enableH2C = "on" # "on" or "off"
debug = false
[httpc]
mode = "auto" # "auto" or "advanced"
maxIdleConns = 100 # only for advanced mode
maxIdleConnsPerHost = 60 # only for advanced mode
maxConnsPerHost = 0 # only for advanced mode
[pages]
enabled = false
staticDir = "/data/www"

6
deploy/config.toml
View file

@ -5,6 +5,12 @@ sizeLimit = 125 # MB
enableH2C = "on"
debug = false
[httpc]
mode = "auto" # "auto" or "advanced"
maxIdleConns = 100 # only for advanced mode
maxIdleConnsPerHost = 60 # only for advanced mode
maxConnsPerHost = 0 # only for advanced mode
[pages]
enabled = false
staticDir = "/usr/local/ghproxy/pages"

4
go.mod
View file

@ -6,7 +6,7 @@ require (
github.com/BurntSushi/toml v1.4.0
github.com/WJQSERVER-STUDIO/go-utils/logger v1.3.0
github.com/gin-gonic/gin v1.10.0
github.com/satomitouka/touka-httpc v0.0.3
github.com/satomitouka/touka-httpc v0.1.0
golang.org/x/net v0.35.0
golang.org/x/time v0.10.0
)
@ -19,7 +19,7 @@ require (
github.com/gin-contrib/sse v1.0.0 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.24.0 // indirect
github.com/go-playground/validator/v10 v10.25.0 // indirect
github.com/goccy/go-json v0.10.5 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect

14
go.sum
View file

@ -27,6 +27,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
github.com/go-playground/validator/v10 v10.24.0 h1:KHQckvo8G6hlWnrPX4NJJ+aBfWNAE/HH+qdL2cBpCmg=
github.com/go-playground/validator/v10 v10.24.0/go.mod h1:GGzBIJMuE98Ic/kJsBXbz1x/7cByt++cQ+YOuDM5wus=
github.com/go-playground/validator/v10 v10.25.0 h1:5Dh7cjvzR7BRZadnsVOzPhWsrwUr0nmsZJxEAnFLNO8=
github.com/go-playground/validator/v10 v10.25.0/go.mod h1:GGzBIJMuE98Ic/kJsBXbz1x/7cByt++cQ+YOuDM5wus=
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
@ -53,6 +55,18 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/satomitouka/touka-httpc v0.0.3 h1:SLb14DWBIDeIaNQ0wMwRwJMjUDakHVR1Jbdct3Qi8fA=
github.com/satomitouka/touka-httpc v0.0.3/go.mod h1:ULB/0Ze0Apm46YKl35Jmj1hW5YLVVeOGqCqn+ijqGPM=
github.com/satomitouka/touka-httpc v0.0.4 h1:sZs/2kqTSyLQ/pDHs/71l7MSG46j4rZNKfqn3CFAboU=
github.com/satomitouka/touka-httpc v0.0.4/go.mod h1:ULB/0Ze0Apm46YKl35Jmj1hW5YLVVeOGqCqn+ijqGPM=
github.com/satomitouka/touka-httpc v0.0.5 h1:ov1v29vrjvwRNbGqFJHmrCp+3/qXLoyWubO4kTDvb28=
github.com/satomitouka/touka-httpc v0.0.5/go.mod h1:ULB/0Ze0Apm46YKl35Jmj1hW5YLVVeOGqCqn+ijqGPM=
github.com/satomitouka/touka-httpc v0.0.6 h1:1iSaTB9KpviXy2NHvMXuRzy5mkcvle+fktWPhpS907c=
github.com/satomitouka/touka-httpc v0.0.6/go.mod h1:ULB/0Ze0Apm46YKl35Jmj1hW5YLVVeOGqCqn+ijqGPM=
github.com/satomitouka/touka-httpc v0.0.7 h1:igoLqXs6R1yNIdKMcfpwRB1l6KLLus6DvWT3xL1T5FY=
github.com/satomitouka/touka-httpc v0.0.7/go.mod h1:ULB/0Ze0Apm46YKl35Jmj1hW5YLVVeOGqCqn+ijqGPM=
github.com/satomitouka/touka-httpc v0.0.8 h1:KW521Z2z9BarnTgCajug/W/tIbnoIH+CzA7CON19iAg=
github.com/satomitouka/touka-httpc v0.0.8/go.mod h1:ULB/0Ze0Apm46YKl35Jmj1hW5YLVVeOGqCqn+ijqGPM=
github.com/satomitouka/touka-httpc v0.1.0 h1:CXCsr6NhdskK/W/ezvhwK2CP8QGCxewkBhsEjrM7K8s=
github.com/satomitouka/touka-httpc v0.1.0/go.mod h1:ULB/0Ze0Apm46YKl35Jmj1hW5YLVVeOGqCqn+ijqGPM=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=

59
proxy/chunkreq.go
View file

@ -7,66 +7,15 @@ import (
"io"
"net/http"
"strconv"
"sync"
"time"
"github.com/gin-gonic/gin"
httpc "github.com/satomitouka/touka-httpc"
)
var BufferSize int = 32 * 1024 // 32KB
var (
ctr *http.Transport
BufferPool *sync.Pool
cclient *httpc.Client
)
func InitReq(cfg *config.Config) {
initChunkedHTTPClient(cfg)
initGitHTTPClient(cfg)
// 初始化固定大小的缓存池
BufferPool = &sync.Pool{
New: func() interface{} {
return make([]byte, BufferSize)
},
}
}
func initChunkedHTTPClient(cfg *config.Config) {
/*
ctr = &http.Transport{
MaxIdleConns: 100,
MaxConnsPerHost: 60,
IdleConnTimeout: 20 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
ResponseHeaderTimeout: 10 * time.Second,
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).DialContext,
}
*/
ctr = &http.Transport{
MaxIdleConns: 100,
MaxConnsPerHost: 60,
IdleConnTimeout: 20 * time.Second,
}
if cfg.Outbound.Enabled {
initTransport(cfg, ctr)
}
cclient = httpc.New(
httpc.WithTransport(ctr),
)
}
func ChunkedProxyRequest(c *gin.Context, u string, cfg *config.Config, mode string, runMode string) {
method := c.Request.Method
// 发送HEAD请求, 预获取Content-Length
headReq, err := cclient.NewRequest("HEAD", u, nil)
headReq, err := client.NewRequest("HEAD", u, nil)
if err != nil {
HandleError(c, fmt.Sprintf("Failed to create request: %v", err))
return
@ -75,7 +24,7 @@ func ChunkedProxyRequest(c *gin.Context, u string, cfg *config.Config, mode stri
removeWSHeader(headReq) // 删除Conection Upgrade头, 避免与HTTP/2冲突(检查是否存在Upgrade头)
AuthPassThrough(c, cfg, headReq)
headResp, err := cclient.Do(headReq)
headResp, err := client.Do(headReq)
if err != nil {
HandleError(c, fmt.Sprintf("Failed to send request: %v", err))
return
@ -107,7 +56,7 @@ func ChunkedProxyRequest(c *gin.Context, u string, cfg *config.Config, mode stri
bodyReader := bytes.NewBuffer(body)
req, err := cclient.NewRequest(method, u, bodyReader)
req, err := client.NewRequest(method, u, bodyReader)
if err != nil {
HandleError(c, fmt.Sprintf("Failed to create request: %v", err))
return
@ -116,7 +65,7 @@ func ChunkedProxyRequest(c *gin.Context, u string, cfg *config.Config, mode stri
removeWSHeader(req) // 删除Conection Upgrade头, 避免与HTTP/2冲突(检查是否存在Upgrade头)
AuthPassThrough(c, cfg, req)
resp, err := cclient.Do(req)
resp, err := client.Do(req)
if err != nil {
HandleError(c, fmt.Sprintf("Failed to send request: %v", err))
return

46
proxy/gitreq.go
View file

@ -7,42 +7,16 @@ import (
"io"
"net/http"
"strconv"
"time"
"github.com/gin-gonic/gin"
httpc "github.com/satomitouka/touka-httpc"
)
var (
gclient *httpc.Client
gtr *http.Transport
)
func initGitHTTPClient(cfg *config.Config) {
gtr = &http.Transport{
MaxIdleConns: 30,
MaxConnsPerHost: 30,
IdleConnTimeout: 30 * time.Second,
}
if cfg.Outbound.Enabled {
initTransport(cfg, gtr)
}
/*
gclient = &http.Client{
Transport: gtr,
}
*/
gclient = httpc.New(
httpc.WithTransport(gtr),
)
}
func GitReq(c *gin.Context, u string, cfg *config.Config, mode string, runMode string) {
method := c.Request.Method
logInfo("%s %s %s %s %s", c.ClientIP(), method, u, c.Request.Header.Get("User-Agent"), c.Request.Proto)
// 发送HEAD请求, 预获取Content-Length
headReq, err := gclient.NewRequest("HEAD", u, nil)
headReq, err := client.NewRequest("HEAD", u, nil)
if err != nil {
HandleError(c, fmt.Sprintf("Failed to create request: %v", err))
return
@ -50,7 +24,7 @@ func GitReq(c *gin.Context, u string, cfg *config.Config, mode string, runMode s
setRequestHeaders(c, headReq)
AuthPassThrough(c, cfg, headReq)
headResp, err := gclient.Do(headReq)
headResp, err := client.Do(headReq)
if err != nil {
HandleError(c, fmt.Sprintf("Failed to send request: %v", err))
return
@ -84,7 +58,7 @@ func GitReq(c *gin.Context, u string, cfg *config.Config, mode string, runMode s
bodyReader := bytes.NewBuffer(body)
// 创建请求
req, err := gclient.NewRequest(method, u, bodyReader)
req, err := client.NewRequest(method, u, bodyReader)
if err != nil {
HandleError(c, fmt.Sprintf("Failed to create request: %v", err))
return
@ -92,7 +66,7 @@ func GitReq(c *gin.Context, u string, cfg *config.Config, mode string, runMode s
setRequestHeaders(c, req)
AuthPassThrough(c, cfg, req)
resp, err := gclient.Do(req)
resp, err := client.Do(req)
if err != nil {
HandleError(c, fmt.Sprintf("Failed to send request: %v", err))
return
@ -139,7 +113,15 @@ func GitReq(c *gin.Context, u string, cfg *config.Config, mode string, runMode s
c.Status(resp.StatusCode)
if _, err := io.Copy(c.Writer, resp.Body); err != nil {
logError("%s %s %s %s %s Response-Copy-Error: %v", c.ClientIP(), method, u, c.Request.Header.Get("User-Agent"), c.Request.Proto, err)
// 使用固定32KB缓冲池
buffer := BufferPool.Get().([]byte)
defer BufferPool.Put(buffer)
_, err = io.CopyBuffer(c.Writer, resp.Body, buffer)
if err != nil {
logError("%s %s %s %s %s Failed to copy response body: %v", c.ClientIP(), method, u, c.Request.Header.Get("User-Agent"), c.Request.Proto, err)
return
} else {
c.Writer.Flush() // 确保刷入
}
}

2
proxy/handler.go
View file

@ -61,7 +61,7 @@ func NoRouteHandler(cfg *config.Config, limiter *rate.RateLimiter, iplimiter *ra
// 白名单检查
if cfg.Whitelist.Enabled {
whitelist := auth.CheckWhitelist(repouser, username, repo)
whitelist := auth.CheckWhitelist(username, repo)
if !whitelist {
logErrMsg := fmt.Sprintf("%s %s %s %s %s Whitelist Blocked repo: %s", c.ClientIP(), c.Request.Method, rawPath, c.Request.Header.Get("User-Agent"), c.Request.Proto, repouser)
errMsg := fmt.Sprintf("Whitelist Blocked repo: %s", repouser)

88
proxy/httpc.go Normal file
View file

@ -0,0 +1,88 @@
package proxy
import (
"fmt"
"ghproxy/config"
"net/http"
"sync"
"time"
httpc "github.com/satomitouka/touka-httpc"
)
var BufferSize int = 32 * 1024 // 32KB
var (
tr *http.Transport
BufferPool *sync.Pool
client *httpc.Client
)
func InitReq(cfg *config.Config) {
initHTTPClient(cfg)
// 初始化固定大小的缓存池
BufferPool = &sync.Pool{
New: func() interface{} {
return make([]byte, BufferSize)
},
}
}
func initHTTPClient(cfg *config.Config) {
/*
ctr = &http.Transport{
MaxIdleConns: 100,
MaxConnsPerHost: 60,
IdleConnTimeout: 20 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
ResponseHeaderTimeout: 10 * time.Second,
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).DialContext,
}
*/
if cfg.Httpc.Mode == "auto" {
tr = &http.Transport{
//MaxIdleConns: 160,
IdleConnTimeout: 30 * time.Second,
WriteBufferSize: 32 * 1024, // 32KB
ReadBufferSize: 32 * 1024, // 32KB
}
} else if cfg.Httpc.Mode == "advanced" {
tr = &http.Transport{
MaxIdleConns: cfg.Httpc.MaxIdleConns,
MaxConnsPerHost: cfg.Httpc.MaxConnsPerHost,
MaxIdleConnsPerHost: cfg.Httpc.MaxIdleConnsPerHost,
WriteBufferSize: 32 * 1024, // 32KB
ReadBufferSize: 32 * 1024, // 32KB
}
} else {
// 错误的模式
logError("unknown httpc mode: %s", cfg.Httpc.Mode)
fmt.Println("unknown httpc mode: ", cfg.Httpc.Mode)
logWarning("use Auto to Run HTTP Client")
fmt.Println("use Auto to Run HTTP Client")
tr = &http.Transport{
//MaxIdleConns: 160,
IdleConnTimeout: 30 * time.Second,
WriteBufferSize: 32 * 1024, // 32KB
ReadBufferSize: 32 * 1024, // 32KB
}
}
if cfg.Outbound.Enabled {
initTransport(cfg, tr)
}
if cfg.Server.Debug {
client = httpc.New(
httpc.WithTransport(tr),
httpc.WithDumpLog(),
)
} else {
client = httpc.New(
httpc.WithTransport(tr),
)
}
}