Merge pull request #99 from WJQSERVER-STUDIO/dev

3.2.2

CHANGELOG.md

@@ -1,5 +1,20 @@
 # 更新日志
 
+3.2.2 - 2025-04-29
+---
+- ADD: 实验性的raw Header处置, 用于应对Github对zh-CN的限制
+- FIX: 修正Header部分的一些处理问题
+- REVERT: 为`git clone`部分回滚 3.1.0中的 "使用`bodystream`进行req方向的body复制, 而不是使用额外的`buffer reader`" 修改
+
+25w33b - 2025-04-29
+---
+- REVERT: 为`git clone`部分回滚 3.1.0中的 "使用`bodystream`进行req方向的body复制, 而不是使用额外的`buffer reader`" 修改
+
+25w33a - 2025-04-29
+---
+- ADD: 实验性的raw Header处置, 用于应对Github对zh-CN的限制
+- FIX: 修正Header部分的一些处理问题
+
 3.2.1 - 2025-04-29
 ---
 - FIX: 修复在`HertZ`路由匹配器下`matcher`键值不一致的问题

DEV-VERSION

@@ -1 +1 @@
-25w32a
+25w33b

VERSION

@@ -1 +1 @@
-3.2.1
+3.2.2

config/config.go

@@ -50,12 +50,14 @@ mode = "auto" # "auto" or "advanced"
 maxIdleConns = 100 # only for advanced mode
 maxIdleConnsPerHost = 60 # only for advanced mode
 maxConnsPerHost = 0 # only for advanced mode
+useCustomRawHeaders = false
 */
 type HttpcConfig struct {
 	Mode                string `toml:"mode"`
 	MaxIdleConns        int    `toml:"maxIdleConns"`
 	MaxIdleConnsPerHost int    `toml:"maxIdleConnsPerHost"`
 	MaxConnsPerHost     int    `toml:"maxConnsPerHost"`
+	UseCustomRawHeaders bool   `toml:"useCustomRawHeaders"`
 }
 
 /*

config/config.toml

@@ -13,6 +13,7 @@ mode = "auto" # "auto" or "advanced"
 maxIdleConns = 100 # only for advanced mode
 maxIdleConnsPerHost = 60 # only for advanced mode
 maxConnsPerHost = 0 # only for advanced mode
+useCustomRawHeaders = false
 
 [gitclone]
 mode = "bypass" # bypass / cache

docs/config.md

@@ -24,6 +24,7 @@ mode = "auto" # "auto" or "advanced"
 maxIdleConns = 100 # only for advanced mode
 maxIdleConnsPerHost = 60 # only for advanced mode
 maxConnsPerHost = 0 # only for advanced mode
+useCustomRawHeaders = false
 
 [gitclone]
 mode = "bypass" # bypass / cache
@@ -138,6 +139,10 @@ target = "ghcr" # ghcr/dockerhub
         *   类型: 整数 (`int`)
         *   默认值: `0` (不限制)
         *   说明:  设置 HTTP 客户端连接池中，每个主机允许建立的最大连接数。设置为 `0` 表示不限制。
+    *   `useCustomRawHeaders`: 使用预定义header避免github waf对应zh-CN的封锁
+        *   类型: 布尔值(`bool`)
+        *   默认值: `false`(停用)
+        *   说明: 启用后, 拉取raw文件会使用程序预定义的固定headers, 而不是原先的复制行为
 
 *   **`[gitclone]` - Git 克隆配置**
 

proxy/chunkreq.go

@@ -11,29 +11,6 @@ import (
 	"github.com/cloudwego/hertz/pkg/app"
 )
 
-var (
-	respHeadersToRemove = map[string]struct{}{
-		"Content-Security-Policy":   {},
-		"Referrer-Policy":           {},
-		"Strict-Transport-Security": {},
-		"X-Github-Request-Id":       {},
-		"X-Timer":                   {},
-		"X-Served-By":               {},
-		"X-Fastly-Request-Id":       {},
-	}
-
-	reqHeadersToRemove = map[string]struct{}{
-		"CF-IPCountry":     {},
-		"CF-RAY":           {},
-		"CF-Visitor":       {},
-		"CF-Connecting-IP": {},
-		"CF-EW-Via":        {},
-		"CDN-Loop":         {},
-		"Upgrade":          {},
-		"Connection":       {},
-	}
-)
-
 func ChunkedProxyRequest(ctx context.Context, c *app.RequestContext, u string, cfg *config.Config, matcher string) {
 
 	var (
@@ -51,7 +28,7 @@ func ChunkedProxyRequest(ctx context.Context, c *app.RequestContext, u string, c
 		return
 	}
 
-	setRequestHeaders(c, req)
+	setRequestHeaders(c, req, cfg, matcher)
 	AuthPassThrough(c, cfg, req)
 
 	resp, err = client.Do(req)

proxy/gitreq.go

@@ -1,6 +1,7 @@
 package proxy
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"ghproxy/config"
@@ -13,7 +14,10 @@ import (
 func GitReq(ctx context.Context, c *app.RequestContext, u string, cfg *config.Config, mode string) {
 	method := string(c.Request.Method())
 
-	logDump("Url Before FMT:%s", u)
+	bodyReader := bytes.NewBuffer(c.Request.Body())
+
+	//bodyReader := c.Request.BodyStream()
+
 	if cfg.GitClone.Mode == "cache" {
 		userPath, repoPath, remainingPath, queryParams, err := extractParts(u)
 		if err != nil {
@@ -22,7 +26,6 @@ func GitReq(ctx context.Context, c *app.RequestContext, u string, cfg *config.Co
 		}
 		// 构建新url
 		u = cfg.GitClone.SmartGitAddr + userPath + repoPath + remainingPath + "?" + queryParams.Encode()
-		logDump("New Url After FMT:%s", u)
 	}
 
 	var (
@@ -30,13 +33,17 @@ func GitReq(ctx context.Context, c *app.RequestContext, u string, cfg *config.Co
 	)
 
 	if cfg.GitClone.Mode == "cache" {
-		req, err := gitclient.NewRequest(method, u, c.Request.BodyStream())
+		rb := gitclient.NewRequestBuilder(method, u)
+		rb.NoDefaultHeaders()
+		rb.SetBody(bodyReader)
+
+		req, err := rb.Build()
 		if err != nil {
 			HandleError(c, fmt.Sprintf("Failed to create request: %v", err))
 			return
 		}
-		setRequestHeaders(c, req)
-		//removeWSHeader(req)
+
+		setRequestHeaders(c, req, cfg, "clone")
 		AuthPassThrough(c, cfg, req)
 
 		resp, err = gitclient.Do(req)
@@ -45,13 +52,17 @@ func GitReq(ctx context.Context, c *app.RequestContext, u string, cfg *config.Co
 			return
 		}
 	} else {
-		req, err := client.NewRequest(method, u, c.Request.BodyStream())
+		rb := client.NewRequestBuilder(string(c.Request.Method()), u)
+		rb.NoDefaultHeaders()
+		rb.SetBody(bodyReader)
+
+		req, err := rb.Build()
 		if err != nil {
 			HandleError(c, fmt.Sprintf("Failed to create request: %v", err))
 			return
 		}
-		setRequestHeaders(c, req)
-		//removeWSHeader(req)
+
+		setRequestHeaders(c, req, cfg, "clone")
 		AuthPassThrough(c, cfg, req)
 
 		resp, err = client.Do(req)
@@ -78,7 +89,8 @@ func GitReq(ctx context.Context, c *app.RequestContext, u string, cfg *config.Co
 
 	for key, values := range resp.Header {
 		for _, value := range values {
-			c.Header(key, value)
+			//c.Header(key, value)
+			c.Response.Header.Add(key, value)
 		}
 	}
 

proxy/reqheader.go

@@ -1,17 +1,75 @@
 package proxy
 
 import (
+	"ghproxy/config"
 	"net/http"
 
 	"github.com/cloudwego/hertz/pkg/app"
 )
 
-func setRequestHeaders(c *app.RequestContext, req *http.Request) {
-	c.Request.Header.VisitAll(func(key, value []byte) {
-		headerKey := string(key)
-		headerValue := string(value)
-		if _, shouldRemove := reqHeadersToRemove[headerKey]; !shouldRemove {
-			req.Header.Set(headerKey, headerValue)
+var (
+	respHeadersToRemove = map[string]struct{}{
+		"Content-Security-Policy":   {},
+		"Referrer-Policy":           {},
+		"Strict-Transport-Security": {},
+		"X-Github-Request-Id":       {},
+		"X-Timer":                   {},
+		"X-Served-By":               {},
+		"X-Fastly-Request-Id":       {},
+	}
+
+	reqHeadersToRemove = map[string]struct{}{
+		"CF-IPCountry":     {},
+		"CF-RAY":           {},
+		"CF-Visitor":       {},
+		"CF-Connecting-IP": {},
+		"CF-EW-Via":        {},
+		"CDN-Loop":         {},
+		"Upgrade":          {},
+		"Connection":       {},
+	}
+
+	cloneHeadersToRemove = map[string]struct{}{
+		"CF-IPCountry":     {},
+		"CF-RAY":           {},
+		"CF-Visitor":       {},
+		"CF-Connecting-IP": {},
+		"CF-EW-Via":        {},
+		"CDN-Loop":         {},
+	}
+)
+
+// 预定义headers
+var (
+	defaultHeaders = map[string]string{
+		"Accept":            "*/*",
+		"Accept-Encoding":   "gzip",
+		"Transfer-Encoding": "chunked",
+		"User-Agent":        "GHProxy/1.0",
+	}
+)
+
+func setRequestHeaders(c *app.RequestContext, req *http.Request, cfg *config.Config, matcher string) {
+	if matcher == "raw" && cfg.Httpc.UseCustomRawHeaders {
+		// 使用预定义Header
+		for key, value := range defaultHeaders {
+			req.Header.Set(key, value)
 		}
-	})
+	} else if matcher == "clone" {
+		c.Request.Header.VisitAll(func(key, value []byte) {
+			headerKey := string(key)
+			headerValue := string(value)
+			if _, shouldRemove := cloneHeadersToRemove[headerKey]; !shouldRemove {
+				req.Header.Set(headerKey, headerValue)
+			}
+		})
+	} else {
+		c.Request.Header.VisitAll(func(key, value []byte) {
+			headerKey := string(key)
+			headerValue := string(value)
+			if _, shouldRemove := reqHeadersToRemove[headerKey]; !shouldRemove {
+				req.Header.Set(headerKey, headerValue)
+			}
+		})
+	}
 }