diff --git a/.github/ISSUE_TEMPLATE/features_request.md b/.github/ISSUE_TEMPLATE/features_request.md index 603c02f..38732d9 100644 --- a/.github/ISSUE_TEMPLATE/features_request.md +++ b/.github/ISSUE_TEMPLATE/features_request.md @@ -2,7 +2,7 @@ name: Features request about: 提出新功能建议 title: "[Features]" -labels: enhancement +labels: 改进 assignees: '' --- diff --git a/CHANGELOG.md b/CHANGELOG.md index 639dc60..8361a09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,26 @@ # 更新日志 +3.0.3 - 2025-04-19 +--- +- CHANGE: 增加移除部分header的处置, 避免向服务端/客户端透露过多信息 +- FIX: 修正非预期的header操作行为 +- CHANGE: 合并header相关逻辑, 避免多次操作 +- CHANGE: 对editor模式下的input进行处置, 增加隐式关闭处理 +- CHANGE: 增加`netlib`配置项 + +25w29b - 2025-04-19 +--- +- PRE-RELEASE: 此版本是v3.0.3预发布版本,请勿在生产环境中使用; +- CHANGE: 增加`netlib`配置项 + +25w29a - 2025-04-17 +--- +- PRE-RELEASE: 此版本是v3.0.3预发布版本,请勿在生产环境中使用; +- CHANGE: 增加移除部分header的处置, 避免向服务端/客户端透露过多信息 +- FIX: 修正非预期的header操作行为 +- CHANGE: 合并header相关逻辑, 避免多次操作 +- CHANGE: 对editor模式下的input进行处置, 增加隐式关闭处理 + 3.0.2 - 2025-04-15 --- - CHANGE: 避免重复的re编译操作 diff --git a/DEV-VERSION b/DEV-VERSION index 0d83d4d..c307029 100644 --- a/DEV-VERSION +++ b/DEV-VERSION @@ -1 +1 @@ -25w28b \ No newline at end of file +25w29b \ No newline at end of file diff --git a/VERSION b/VERSION index d9c62ed..282895a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.0.2 \ No newline at end of file +3.0.3 \ No newline at end of file diff --git a/config/config.go b/config/config.go index 1bf1bdc..e737a63 100644 --- a/config/config.go +++ b/config/config.go @@ -22,15 +22,20 @@ type Config struct { /* [server] -host = "0.0.0.0" # 监听地址 -port = 8080 # 监听端口 -sizeLimit = 125 # 125MB -H2C = true # 是否开启H2C传输 +host = "0.0.0.0" +port = 8080 +netlib = "netpoll" # "netpoll" / "std" "standard" "net/http" "net" +sizeLimit = 125 # MB +memLimit = 0 # MB +H2C = true +cors = "*" # "*"/"" -> "*" ; "nil" -> "" ; +debug = false */ type ServerConfig struct { Port int `toml:"port"` Host string `toml:"host"` + NetLib string `toml:"netlib"` SizeLimit int `toml:"sizeLimit"` MemLimit int64 `toml:"memLimit"` H2C bool `toml:"H2C"` @@ -180,6 +185,7 @@ func DefaultConfig() *Config { Server: ServerConfig{ Port: 8080, Host: "0.0.0.0", + NetLib: "netpoll", SizeLimit: 125, MemLimit: 0, H2C: true, diff --git a/config/config.toml b/config/config.toml index 92b81f8..ac1ee59 100644 --- a/config/config.toml +++ b/config/config.toml @@ -1,6 +1,7 @@ [server] host = "0.0.0.0" port = 8080 +netlib = "netpoll" # "netpoll" / "std" "standard" "net/http" "net" sizeLimit = 125 # MB memLimit = 0 # MB H2C = true diff --git a/deploy/config.toml b/deploy/config.toml index a25acf1..b88e30b 100644 --- a/deploy/config.toml +++ b/deploy/config.toml @@ -1,6 +1,7 @@ [server] host = "127.0.0.1" port = 8080 +netlib = "netpoll" # "netpoll" / "std" "standard" "net/http" "net" sizeLimit = 125 # MB memLimit = 0 # MB H2C = true diff --git a/docs/config.md b/docs/config.md index ccd1ce2..bad8c65 100644 --- a/docs/config.md +++ b/docs/config.md @@ -12,6 +12,7 @@ [server] host = "0.0.0.0" port = 8080 +netlib = "netpoll" # "netpoll" / "std" "standard" "net/http" "net" sizeLimit = 125 # MB memLimit = 0 # MB H2C = true @@ -83,6 +84,10 @@ url = "socks5://127.0.0.1:1080" # "http://127.0.0.1:7890" * 类型: 整数 (`int`) * 默认值: `8080` * 说明: 设置 `ghproxy` 监听的端口号。 + * `netlib`: 底层网络库。 + * 类型: 字符串 (`string`) + * 默认值: `""` (HertZ默认处置) + * 说明: `"std"` `"standard"` `"net/http"` `"net"` 均会被设置为go标准库`net/http`, 设置为`"netpoll"`或`""`会由`HertZ`默认逻辑处理 * `sizeLimit`: 请求体大小限制。 * 类型: 整数 (`int`) * 默认值: `125` (MB) diff --git a/main.go b/main.go index 3cbd127..50df92e 100644 --- a/main.go +++ b/main.go @@ -19,15 +19,16 @@ import ( "ghproxy/rate" "github.com/WJQSERVER-STUDIO/go-utils/logger" + "github.com/hertz-contrib/http2/factory" "github.com/cloudwego/hertz/pkg/app" "github.com/cloudwego/hertz/pkg/app/middlewares/server/recovery" "github.com/cloudwego/hertz/pkg/app/server" "github.com/cloudwego/hertz/pkg/common/adaptor" "github.com/cloudwego/hertz/pkg/common/hlog" + "github.com/cloudwego/hertz/pkg/network/standard" - //"github.com/cloudwego/hertz/pkg/network/standard" - "github.com/hertz-contrib/http2/factory" + _ "net/http/pprof" ) var ( @@ -360,32 +361,47 @@ func init() { } func main() { - // 如果 showVersion 为 true，则在 init 阶段已退出，这里直接返回 if showVersion || showHelp { return } - logDebug("Run Mode: %s", runMode) + logDebug("Run Mode: %s Netlib: %s", runMode, cfg.Server.NetLib) - // 确保在程序配置加载且非版本显示模式下执行 if cfg == nil { fmt.Println("Config not loaded, exiting.") - return // 如果配置未加载，则不继续执行 + return } addr := fmt.Sprintf("%s:%d", cfg.Server.Host, cfg.Server.Port) - - if cfg.Server.H2C { - r = server.New( - server.WithHostPorts(addr), - server.WithH2C(true), - // server.WithALPN(true), - // server.WithTransport(standard.NewTransporter), - ) - r.AddProtocol("h2", factory.NewServerFactory()) + if cfg.Server.NetLib == "std" || cfg.Server.NetLib == "standard" || cfg.Server.NetLib == "net" || cfg.Server.NetLib == "net/http" { + if cfg.Server.H2C { + r = server.New( + server.WithH2C(true), + server.WithHostPorts(addr), + server.WithTransport(standard.NewTransporter), + ) + r.AddProtocol("h2", factory.NewServerFactory()) + } else { + r = server.New( + server.WithHostPorts(addr), + server.WithTransport(standard.NewTransporter), + ) + } + } else if cfg.Server.NetLib == "netpoll" || cfg.Server.NetLib == "" { + if cfg.Server.H2C { + r = server.New( + server.WithH2C(true), + server.WithHostPorts(addr), + ) + r.AddProtocol("h2", factory.NewServerFactory()) + } else { + r = server.New( + server.WithHostPorts(addr), + ) + } } else { - r = server.New( - server.WithHostPorts(addr), - ) + logError("Invalid NetLib: %s", cfg.Server.NetLib) + fmt.Printf("Invalid NetLib: %s\n", cfg.Server.NetLib) + os.Exit(1) } // 添加Recovery中间件 @@ -447,11 +463,18 @@ func main() { fmt.Printf("A Go Based High-Performance Github Proxy \n") fmt.Printf("Made by WJQSERVER-STUDIO\n") + if cfg.Server.Debug { + go func() { + http.ListenAndServe("localhost:6060", nil) + }() + } + r.Spin() defer logger.Close() defer func() { if hertZfile != nil { - err := hertZfile.Close() + var err error + err = hertZfile.Close() if err != nil { logError("Failed to close hertz log file: %v", err) } diff --git a/proxy/chunkreq.go b/proxy/chunkreq.go index 47f0e59..b02ec4c 100644 --- a/proxy/chunkreq.go +++ b/proxy/chunkreq.go @@ -5,29 +5,60 @@ import ( "context" "fmt" "ghproxy/config" + "io" "net/http" "strconv" "github.com/cloudwego/hertz/pkg/app" ) +var ( + respHeadersToRemove = map[string]struct{}{ + "Content-Security-Policy": {}, + "Referrer-Policy": {}, + "Strict-Transport-Security": {}, + "X-Github-Request-Id": {}, + "X-Timer": {}, + "X-Served-By": {}, + "X-Fastly-Request-Id": {}, + } + + reqHeadersToRemove = map[string]struct{}{ + "CF-IPCountry": {}, + "CF-RAY": {}, + "CF-Visitor": {}, + "CF-Connecting-IP": {}, + "CF-EW-Via": {}, + "CDN-Loop": {}, + "Upgrade": {}, + "Connection": {}, + } +) + func ChunkedProxyRequest(ctx context.Context, c *app.RequestContext, u string, cfg *config.Config, matcher string) { - method := c.Request.Method - body := c.Request.Body() + var ( + method []byte + bodyReader *bytes.Buffer + req *http.Request + resp *http.Response + err error + ) - bodyReader := bytes.NewBuffer(body) + method = c.Request.Method() + bodyReader = bytes.NewBuffer(c.Request.Body()) - req, err := client.NewRequest(string(method()), u, bodyReader) + req, err = client.NewRequest(string(method), u, bodyReader) if err != nil { HandleError(c, fmt.Sprintf("Failed to create request: %v", err)) return } + setRequestHeaders(c, req) - removeWSHeader(req) // 删除Conection Upgrade头, 避免与HTTP/2冲突(检查是否存在Upgrade头) + //removeWSHeader(req) // 删除Conection Upgrade头, 避免与HTTP/2冲突(检查是否存在Upgrade头) AuthPassThrough(c, cfg, req) - resp, err := client.Do(req) + resp, err = client.Do(req) if err != nil { HandleError(c, fmt.Sprintf("Failed to send request: %v", err)) return @@ -55,8 +86,9 @@ func ChunkedProxyRequest(ctx context.Context, c *app.RequestContext, u string, c bodySize = -1 } if err == nil && bodySize > sizelimit { - finalURL := resp.Request.URL.String() - err := resp.Body.Close() + var finalURL string + finalURL = resp.Request.URL.String() + err = resp.Body.Close() if err != nil { logError("Failed to close response body: %v", err) } @@ -66,20 +98,26 @@ func ChunkedProxyRequest(ctx context.Context, c *app.RequestContext, u string, c } } - for key, values := range resp.Header { - for _, value := range values { - c.Header(key, value) + /* + for header := range headersToRemove { + resp.Header.Del(header) } - } - headersToRemove := map[string]struct{}{ - "Content-Security-Policy": {}, - "Referrer-Policy": {}, - "Strict-Transport-Security": {}, - } + for key := range resp.Header { + var values []string = resp.Header.Values(key) + for _, value := range values { + c.Header(key, value) + } + } + */ - for header := range headersToRemove { - resp.Header.Del(header) + // 复制响应头，排除需要移除的 header + for key, values := range resp.Header { + if _, shouldRemove := respHeadersToRemove[key]; !shouldRemove { + for _, value := range values { + c.Header(key, value) + } + } } switch cfg.Server.Cors { @@ -105,7 +143,9 @@ func ChunkedProxyRequest(ctx context.Context, c *app.RequestContext, u string, c logInfo("Is Shell: %s %s %s %s %s", c.ClientIP(), method, u, c.Request.Header.Get("User-Agent"), c.Request.Header.GetProtocol()) c.Header("Content-Length", "") - reader, _, err := processLinks(resp.Body, compress, string(c.Request.Host()), cfg) + var reader io.Reader + + reader, _, err = processLinks(resp.Body, compress, string(c.Request.Host()), cfg) c.SetBodyStream(reader, -1) if err != nil { diff --git a/proxy/gitreq.go b/proxy/gitreq.go index c4f0cfd..55ede9a 100644 --- a/proxy/gitreq.go +++ b/proxy/gitreq.go @@ -43,7 +43,7 @@ func GitReq(ctx context.Context, c *app.RequestContext, u string, cfg *config.Co return } setRequestHeaders(c, req) - removeWSHeader(req) + //removeWSHeader(req) AuthPassThrough(c, cfg, req) resp, err = gitclient.Do(req) @@ -58,7 +58,7 @@ func GitReq(ctx context.Context, c *app.RequestContext, u string, cfg *config.Co return } setRequestHeaders(c, req) - removeWSHeader(req) + //removeWSHeader(req) AuthPassThrough(c, cfg, req) resp, err = client.Do(req) diff --git a/proxy/handler.go b/proxy/handler.go index f8b998c..9d214d3 100644 --- a/proxy/handler.go +++ b/proxy/handler.go @@ -41,13 +41,19 @@ func NoRouteHandler(cfg *config.Config, limiter *rate.RateLimiter, iplimiter *ra } } - rawPath := strings.TrimPrefix(string(c.Request.RequestURI()), "/") // 去掉前缀/ - matches := re.FindStringSubmatch(rawPath) // 匹配路径 + var ( + rawPath string + matches []string + errMsg string + ) + + rawPath = strings.TrimPrefix(string(c.Request.RequestURI()), "/") // 去掉前缀/ + matches = re.FindStringSubmatch(rawPath) // 匹配路径 logInfo("URL: %v", matches) // 匹配路径错误处理 if len(matches) < 3 { - errMsg := fmt.Sprintf("%s %s %s %s %s Invalid URL", c.ClientIP(), c.Method(), rawPath, c.Request.Header.UserAgent(), c.Request.Header.GetProtocol()) + errMsg = fmt.Sprintf("%s %s %s %s %s Invalid URL", c.ClientIP(), c.Method(), rawPath, c.Request.Header.UserAgent(), c.Request.Header.GetProtocol()) logWarning(errMsg) c.String(http.StatusForbidden, "Invalid URL Format. Path: %s", rawPath) return @@ -56,7 +62,14 @@ func NoRouteHandler(cfg *config.Config, limiter *rate.RateLimiter, iplimiter *ra // 制作url rawPath = "https://" + matches[2] - user, repo, matcher, err := Matcher(rawPath, cfg) + var ( + user string + repo string + matcher string + err error + ) + + user, repo, matcher, err = Matcher(rawPath, cfg) if err != nil { if errors.Is(err, ErrInvalidURL) { c.String(http.StatusForbidden, "Invalid URL Format. Path: %s", rawPath) @@ -69,18 +82,19 @@ func NoRouteHandler(cfg *config.Config, limiter *rate.RateLimiter, iplimiter *ra return } } - username := user - logInfo("%s %s %s %s %s Matched-Username: %s, Matched-Repo: %s", c.ClientIP(), c.Method(), rawPath, c.Request.Header.UserAgent(), c.Request.Header.GetProtocol(), username, repo) + logInfo("%s %s %s %s %s Matched-Username: %s, Matched-Repo: %s", c.ClientIP(), c.Method(), rawPath, c.Request.Header.UserAgent(), c.Request.Header.GetProtocol(), user, repo) // dump log 记录详细信息 c.ClientIP(), c.Method(), rawPath,c.Request.Header.UserAgent(), c.Request.Header.GetProtocol(), full Header logDump("%s %s %s %s %s %s", c.ClientIP(), c.Method(), rawPath, c.Request.Header.UserAgent(), c.Request.Header.GetProtocol(), c.Request.Header.Header()) - repouser := fmt.Sprintf("%s/%s", username, repo) + var repouser string + repouser = fmt.Sprintf("%s/%s", user, repo) // 白名单检查 if cfg.Whitelist.Enabled { - whitelist := auth.CheckWhitelist(username, repo) + var whitelist bool + whitelist = auth.CheckWhitelist(user, repo) if !whitelist { - errMsg := fmt.Sprintf("Whitelist Blocked repo: %s", repouser) + errMsg = fmt.Sprintf("Whitelist Blocked repo: %s", repouser) c.JSON(http.StatusForbidden, map[string]string{"error": errMsg}) logWarning("%s %s %s %s %s Whitelist Blocked repo: %s", c.ClientIP(), c.Method(), rawPath, c.Request.Header.UserAgent(), c.Request.Header.GetProtocol(), repouser) return @@ -89,9 +103,10 @@ func NoRouteHandler(cfg *config.Config, limiter *rate.RateLimiter, iplimiter *ra // 黑名单检查 if cfg.Blacklist.Enabled { - blacklist := auth.CheckBlacklist(username, repo) + var blacklist bool + blacklist = auth.CheckBlacklist(user, repo) if blacklist { - errMsg := fmt.Sprintf("Blacklist Blocked repo: %s", repouser) + errMsg = fmt.Sprintf("Blacklist Blocked repo: %s", repouser) c.JSON(http.StatusForbidden, map[string]string{"error": errMsg}) logWarning("%s %s %s %s %s Blacklist Blocked repo: %s", c.ClientIP(), c.Method(), rawPath, c.Request.Header.UserAgent(), c.Request.Header.GetProtocol(), repouser) return diff --git a/proxy/httpc.go b/proxy/httpc.go index 1f0c6e9..dae57b6 100644 --- a/proxy/httpc.go +++ b/proxy/httpc.go @@ -4,7 +4,6 @@ import ( "fmt" "ghproxy/config" "net/http" - "sync" "time" httpc "github.com/satomitouka/touka-httpc" @@ -13,11 +12,10 @@ import ( var BufferSize int = 32 * 1024 // 32KB var ( - tr *http.Transport - gittr *http.Transport - BufferPool *sync.Pool - client *httpc.Client - gitclient *httpc.Client + tr *http.Transport + gittr *http.Transport + client *httpc.Client + gitclient *httpc.Client ) func InitReq(cfg *config.Config) { @@ -25,13 +23,6 @@ func InitReq(cfg *config.Config) { if cfg.GitClone.Mode == "cache" { initGitHTTPClient(cfg) } - - // 初始化固定大小的缓存池 - BufferPool = &sync.Pool{ - New: func() interface{} { - return make([]byte, BufferSize) - }, - } } func initHTTPClient(cfg *config.Config) { diff --git a/proxy/match.go b/proxy/match.go index 8e6c0d9..1cae46c 100644 --- a/proxy/match.go +++ b/proxy/match.go @@ -246,7 +246,7 @@ func extractParts(rawURL string) (string, string, string, url.Values, error) { var urlPattern = regexp.MustCompile(`https?://[^\s'"]+`) // processLinks 处理链接，返回包含处理后数据的 io.Reader -func processLinks(input io.Reader, compress string, host string, cfg *config.Config) (readerOut io.Reader, written int64, err error) { +func processLinks(input io.ReadCloser, compress string, host string, cfg *config.Config) (readerOut io.Reader, written int64, err error) { pipeReader, pipeWriter := io.Pipe() // 创建 io.Pipe readerOut = pipeReader @@ -268,6 +268,13 @@ func processLinks(input io.Reader, compress string, host string, cfg *config.Con } }() + defer func() { + if err := input.Close(); err != nil { + logError("input close failed: %v", err) + } + + }() + var bufReader *bufio.Reader if compress == "gzip" { diff --git a/proxy/reqheader.go b/proxy/reqheader.go index 01eab46..c338706 100644 --- a/proxy/reqheader.go +++ b/proxy/reqheader.go @@ -6,13 +6,27 @@ import ( "github.com/cloudwego/hertz/pkg/app" ) +/* // 设置请求头 func setRequestHeaders(c *app.RequestContext, req *http.Request) { c.Request.Header.VisitAll(func(key, value []byte) { req.Header.Set(string(key), string(value)) }) } +*/ +func setRequestHeaders(c *app.RequestContext, req *http.Request) { + c.Request.Header.VisitAll(func(key, value []byte) { + headerKey := string(key) + headerValue := string(value) + if _, shouldRemove := reqHeadersToRemove[headerKey]; !shouldRemove { + req.Header.Set(headerKey, headerValue) + } + + }) +} + +/* // removeWSHeader removes the "Upgrade" and "Connection" headers from the given // Request, which are added by the client when it wants to upgrade the // connection to a WebSocket connection. @@ -20,3 +34,4 @@ func removeWSHeader(req *http.Request) { req.Header.Del("Upgrade") req.Header.Del("Connection") } +*/