From 55769d9a400bf06ebd369ed4c686667aa14ca19d Mon Sep 17 00:00:00 2001 From: wjqserver <114663932+WJQSERVER@users.noreply.github.com> Date: Tue, 29 Apr 2025 19:51:23 +0800 Subject: [PATCH 1/6] use custom headers for raw --- DEV-VERSION | 2 +- proxy/chunkreq.go | 2 +- proxy/gitreq.go | 4 ++-- proxy/reqheader.go | 31 ++++++++++++++++++++++++------- 4 files changed, 28 insertions(+), 11 deletions(-) diff --git a/DEV-VERSION b/DEV-VERSION index ca629c1..f9c9056 100644 --- a/DEV-VERSION +++ b/DEV-VERSION @@ -1 +1 @@ -25w32a \ No newline at end of file +25w33t-1 \ No newline at end of file diff --git a/proxy/chunkreq.go b/proxy/chunkreq.go index 13be45a..e766a06 100644 --- a/proxy/chunkreq.go +++ b/proxy/chunkreq.go @@ -51,7 +51,7 @@ func ChunkedProxyRequest(ctx context.Context, c *app.RequestContext, u string, c return } - setRequestHeaders(c, req) + setRequestHeaders(c, req, matcher) AuthPassThrough(c, cfg, req) resp, err = client.Do(req) diff --git a/proxy/gitreq.go b/proxy/gitreq.go index b8ba66d..2a4b57b 100644 --- a/proxy/gitreq.go +++ b/proxy/gitreq.go @@ -35,7 +35,7 @@ func GitReq(ctx context.Context, c *app.RequestContext, u string, cfg *config.Co HandleError(c, fmt.Sprintf("Failed to create request: %v", err)) return } - setRequestHeaders(c, req) + setRequestHeaders(c, req, "clone") //removeWSHeader(req) AuthPassThrough(c, cfg, req) @@ -50,7 +50,7 @@ func GitReq(ctx context.Context, c *app.RequestContext, u string, cfg *config.Co HandleError(c, fmt.Sprintf("Failed to create request: %v", err)) return } - setRequestHeaders(c, req) + setRequestHeaders(c, req, "clone") //removeWSHeader(req) AuthPassThrough(c, cfg, req) diff --git a/proxy/reqheader.go b/proxy/reqheader.go index 99bef3a..24baaae 100644 --- a/proxy/reqheader.go +++ b/proxy/reqheader.go @@ -6,12 +6,29 @@ import ( "github.com/cloudwego/hertz/pkg/app" ) -func setRequestHeaders(c *app.RequestContext, req *http.Request) { - c.Request.Header.VisitAll(func(key, value []byte) { - headerKey := string(key) - headerValue := string(value) - if _, shouldRemove := reqHeadersToRemove[headerKey]; !shouldRemove { - req.Header.Set(headerKey, headerValue) +// 预定义headers +var ( + defaultHeaders = map[string]string{ + "Accept": "*/*", + "Accept-Encoding": "gzip", + "Transfer-Encoding": "chunked", + "User-Agent": "GHProxy/1.0", + } +) + +func setRequestHeaders(c *app.RequestContext, req *http.Request, matcher string) { + if matcher == "raw" { + // 使用预定义Header + for key, value := range defaultHeaders { + req.Header.Set(key, value) } - }) + } else { + c.Request.Header.VisitAll(func(key, value []byte) { + headerKey := string(key) + headerValue := string(value) + if _, shouldRemove := reqHeadersToRemove[headerKey]; !shouldRemove { + req.Header.Set(headerKey, headerValue) + } + }) + } } From 3f51e5319ad1cf10aa5beb7d9ad4884122b1ea16 Mon Sep 17 00:00:00 2001 From: wjqserver <114663932+WJQSERVER@users.noreply.github.com> Date: Tue, 29 Apr 2025 20:39:41 +0800 Subject: [PATCH 2/6] 25w33a --- CHANGELOG.md | 5 +++++ DEV-VERSION | 2 +- config/config.go | 2 ++ config/config.toml | 1 + proxy/chunkreq.go | 25 +------------------------ proxy/gitreq.go | 8 ++------ proxy/reqheader.go | 45 +++++++++++++++++++++++++++++++++++++++++++-- 7 files changed, 55 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f1aeac..5a8c609 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # 更新日志 +25w33a - 2025-04-29 +--- +- ADD: 实验性的raw Header处置, 用于应对Github对zh-CN的限制 +- FIX: 修正Header部分的一些处理问题 + 3.2.1 - 2025-04-29 --- - FIX: 修复在`HertZ`路由匹配器下`matcher`键值不一致的问题 diff --git a/DEV-VERSION b/DEV-VERSION index f9c9056..23f75d9 100644 --- a/DEV-VERSION +++ b/DEV-VERSION @@ -1 +1 @@ -25w33t-1 \ No newline at end of file +25w33a \ No newline at end of file diff --git a/config/config.go b/config/config.go index ed62121..14c9301 100644 --- a/config/config.go +++ b/config/config.go @@ -50,12 +50,14 @@ mode = "auto" # "auto" or "advanced" maxIdleConns = 100 # only for advanced mode maxIdleConnsPerHost = 60 # only for advanced mode maxConnsPerHost = 0 # only for advanced mode +useCustomRawHeaders = false */ type HttpcConfig struct { Mode string `toml:"mode"` MaxIdleConns int `toml:"maxIdleConns"` MaxIdleConnsPerHost int `toml:"maxIdleConnsPerHost"` MaxConnsPerHost int `toml:"maxConnsPerHost"` + UseCustomRawHeaders bool `toml:"useCustomRawHeaders"` } /* diff --git a/config/config.toml b/config/config.toml index fc47ef1..b43ebf3 100644 --- a/config/config.toml +++ b/config/config.toml @@ -13,6 +13,7 @@ mode = "auto" # "auto" or "advanced" maxIdleConns = 100 # only for advanced mode maxIdleConnsPerHost = 60 # only for advanced mode maxConnsPerHost = 0 # only for advanced mode +useCustomRawHeaders = false [gitclone] mode = "bypass" # bypass / cache diff --git a/proxy/chunkreq.go b/proxy/chunkreq.go index e766a06..22c7b61 100644 --- a/proxy/chunkreq.go +++ b/proxy/chunkreq.go @@ -11,29 +11,6 @@ import ( "github.com/cloudwego/hertz/pkg/app" ) -var ( - respHeadersToRemove = map[string]struct{}{ - "Content-Security-Policy": {}, - "Referrer-Policy": {}, - "Strict-Transport-Security": {}, - "X-Github-Request-Id": {}, - "X-Timer": {}, - "X-Served-By": {}, - "X-Fastly-Request-Id": {}, - } - - reqHeadersToRemove = map[string]struct{}{ - "CF-IPCountry": {}, - "CF-RAY": {}, - "CF-Visitor": {}, - "CF-Connecting-IP": {}, - "CF-EW-Via": {}, - "CDN-Loop": {}, - "Upgrade": {}, - "Connection": {}, - } -) - func ChunkedProxyRequest(ctx context.Context, c *app.RequestContext, u string, cfg *config.Config, matcher string) { var ( @@ -51,7 +28,7 @@ func ChunkedProxyRequest(ctx context.Context, c *app.RequestContext, u string, c return } - setRequestHeaders(c, req, matcher) + setRequestHeaders(c, req, cfg, matcher) AuthPassThrough(c, cfg, req) resp, err = client.Do(req) diff --git a/proxy/gitreq.go b/proxy/gitreq.go index 2a4b57b..e48598f 100644 --- a/proxy/gitreq.go +++ b/proxy/gitreq.go @@ -13,7 +13,6 @@ import ( func GitReq(ctx context.Context, c *app.RequestContext, u string, cfg *config.Config, mode string) { method := string(c.Request.Method()) - logDump("Url Before FMT:%s", u) if cfg.GitClone.Mode == "cache" { userPath, repoPath, remainingPath, queryParams, err := extractParts(u) if err != nil { @@ -22,7 +21,6 @@ func GitReq(ctx context.Context, c *app.RequestContext, u string, cfg *config.Co } // 构建新url u = cfg.GitClone.SmartGitAddr + userPath + repoPath + remainingPath + "?" + queryParams.Encode() - logDump("New Url After FMT:%s", u) } var ( @@ -35,8 +33,7 @@ func GitReq(ctx context.Context, c *app.RequestContext, u string, cfg *config.Co HandleError(c, fmt.Sprintf("Failed to create request: %v", err)) return } - setRequestHeaders(c, req, "clone") - //removeWSHeader(req) + setRequestHeaders(c, req, cfg, "clone") AuthPassThrough(c, cfg, req) resp, err = gitclient.Do(req) @@ -50,8 +47,7 @@ func GitReq(ctx context.Context, c *app.RequestContext, u string, cfg *config.Co HandleError(c, fmt.Sprintf("Failed to create request: %v", err)) return } - setRequestHeaders(c, req, "clone") - //removeWSHeader(req) + setRequestHeaders(c, req, cfg, "clone") AuthPassThrough(c, cfg, req) resp, err = client.Do(req) diff --git a/proxy/reqheader.go b/proxy/reqheader.go index 24baaae..8400821 100644 --- a/proxy/reqheader.go +++ b/proxy/reqheader.go @@ -1,11 +1,44 @@ package proxy import ( + "ghproxy/config" "net/http" "github.com/cloudwego/hertz/pkg/app" ) +var ( + respHeadersToRemove = map[string]struct{}{ + "Content-Security-Policy": {}, + "Referrer-Policy": {}, + "Strict-Transport-Security": {}, + "X-Github-Request-Id": {}, + "X-Timer": {}, + "X-Served-By": {}, + "X-Fastly-Request-Id": {}, + } + + reqHeadersToRemove = map[string]struct{}{ + "CF-IPCountry": {}, + "CF-RAY": {}, + "CF-Visitor": {}, + "CF-Connecting-IP": {}, + "CF-EW-Via": {}, + "CDN-Loop": {}, + "Upgrade": {}, + "Connection": {}, + } + + cloneHeadersToRemove = map[string]struct{}{ + "CF-IPCountry": {}, + "CF-RAY": {}, + "CF-Visitor": {}, + "CF-Connecting-IP": {}, + "CF-EW-Via": {}, + "CDN-Loop": {}, + } +) + // 预定义headers var ( defaultHeaders = map[string]string{ @@ -16,12 +49,20 @@ var ( } ) -func setRequestHeaders(c *app.RequestContext, req *http.Request, matcher string) { - if matcher == "raw" { +func setRequestHeaders(c *app.RequestContext, req *http.Request, cfg *config.Config, matcher string) { + if matcher == "raw" && cfg.Httpc.UseCustomRawHeaders { // 使用预定义Header for key, value := range defaultHeaders { req.Header.Set(key, value) } + } else if matcher == "clone" { + c.Request.Header.VisitAll(func(key, value []byte) { + headerKey := string(key) + headerValue := string(value) + if _, shouldRemove := cloneHeadersToRemove[headerKey]; !shouldRemove { + req.Header.Set(headerKey, headerValue) + } + }) } else { c.Request.Header.VisitAll(func(key, value []byte) { headerKey := string(key) From ace795fe9df55689ed529696dd9e9d127d1308bb Mon Sep 17 00:00:00 2001 From: wjqserver <114663932+WJQSERVER@users.noreply.github.com> Date: Tue, 29 Apr 2025 22:13:11 +0800 Subject: [PATCH 3/6] revert gitreq body stream --- CHANGELOG.md | 4 ++++ DEV-VERSION | 2 +- proxy/gitreq.go | 28 +++++++++++++++++++++++++--- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a8c609..f6bdb26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # 更新日志 +25w33b - 2025-04-29 +--- +- REVERT: 为`git clone`部分回滚 3.1.0中的 "使用`bodystream`进行req方向的body复制, 而不是使用额外的`buffer reader`" 修改 + 25w33a - 2025-04-29 --- - ADD: 实验性的raw Header处置, 用于应对Github对zh-CN的限制 diff --git a/DEV-VERSION b/DEV-VERSION index 23f75d9..58d1f65 100644 --- a/DEV-VERSION +++ b/DEV-VERSION @@ -1 +1 @@ -25w33a \ No newline at end of file +25w33b \ No newline at end of file diff --git a/proxy/gitreq.go b/proxy/gitreq.go index e48598f..a050e29 100644 --- a/proxy/gitreq.go +++ b/proxy/gitreq.go @@ -1,6 +1,7 @@ package proxy import ( + "bytes" "context" "fmt" "ghproxy/config" @@ -13,6 +14,10 @@ import ( func GitReq(ctx context.Context, c *app.RequestContext, u string, cfg *config.Config, mode string) { method := string(c.Request.Method()) + bodyReader := bytes.NewBuffer(c.Request.Body()) + + //bodyReader := c.Request.BodyStream() + if cfg.GitClone.Mode == "cache" { userPath, repoPath, remainingPath, queryParams, err := extractParts(u) if err != nil { @@ -28,11 +33,16 @@ func GitReq(ctx context.Context, c *app.RequestContext, u string, cfg *config.Co ) if cfg.GitClone.Mode == "cache" { - req, err := gitclient.NewRequest(method, u, c.Request.BodyStream()) + rb := gitclient.NewRequestBuilder(method, u) + rb.NoDefaultHeaders() + rb.SetBody(bodyReader) + + req, err := rb.Build() if err != nil { HandleError(c, fmt.Sprintf("Failed to create request: %v", err)) return } + setRequestHeaders(c, req, cfg, "clone") AuthPassThrough(c, cfg, req) @@ -42,11 +52,16 @@ func GitReq(ctx context.Context, c *app.RequestContext, u string, cfg *config.Co return } } else { - req, err := client.NewRequest(method, u, c.Request.BodyStream()) + rb := client.NewRequestBuilder(string(c.Request.Method()), u) + rb.NoDefaultHeaders() + rb.SetBody(bodyReader) + + req, err := rb.Build() if err != nil { HandleError(c, fmt.Sprintf("Failed to create request: %v", err)) return } + setRequestHeaders(c, req, cfg, "clone") AuthPassThrough(c, cfg, req) @@ -74,7 +89,8 @@ func GitReq(ctx context.Context, c *app.RequestContext, u string, cfg *config.Co for key, values := range resp.Header { for _, value := range values { - c.Header(key, value) + //c.Header(key, value) + c.Response.Header.Add(key, value) } } @@ -106,5 +122,11 @@ func GitReq(ctx context.Context, c *app.RequestContext, u string, cfg *config.Co c.Response.Header.Set("Expires", "0") } + bodySize, _ := strconv.Atoi(contentLength) + + if contentLength != "" { + c.SetBodyStream(resp.Body, bodySize) + return + } c.SetBodyStream(resp.Body, -1) } From e57432a01c32b1a0e2cfcb3f006f670dddd8d429 Mon Sep 17 00:00:00 2001 From: wjqserver <114663932+WJQSERVER@users.noreply.github.com> Date: Tue, 29 Apr 2025 22:21:08 +0800 Subject: [PATCH 4/6] 3.2.2 --- CHANGELOG.md | 6 ++++++ VERSION | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f6bdb26..09baf1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # 更新日志 +3.2.2 - 2025-04-29 +--- +- ADD: 实验性的raw Header处置, 用于应对Github对zh-CN的限制 +- FIX: 修正Header部分的一些处理问题 +- REVERT: 为`git clone`部分回滚 3.1.0中的 "使用`bodystream`进行req方向的body复制, 而不是使用额外的`buffer reader`" 修改 + 25w33b - 2025-04-29 --- - REVERT: 为`git clone`部分回滚 3.1.0中的 "使用`bodystream`进行req方向的body复制, 而不是使用额外的`buffer reader`" 修改 diff --git a/VERSION b/VERSION index 0444f32..acf9bf0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.2.1 \ No newline at end of file +3.2.2 \ No newline at end of file From 030f0d12a923fc8b44d1173e8eba7dca0f32beaf Mon Sep 17 00:00:00 2001 From: wjqserver <114663932+WJQSERVER@users.noreply.github.com> Date: Tue, 29 Apr 2025 22:24:11 +0800 Subject: [PATCH 5/6] update docs --- docs/config.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/config.md b/docs/config.md index 1a91225..c146ca4 100644 --- a/docs/config.md +++ b/docs/config.md @@ -24,6 +24,7 @@ mode = "auto" # "auto" or "advanced" maxIdleConns = 100 # only for advanced mode maxIdleConnsPerHost = 60 # only for advanced mode maxConnsPerHost = 0 # only for advanced mode +useCustomRawHeaders = false [gitclone] mode = "bypass" # bypass / cache @@ -138,6 +139,10 @@ target = "ghcr" # ghcr/dockerhub * 类型: 整数 (`int`) * 默认值: `0` (不限制) * 说明: 设置 HTTP 客户端连接池中，每个主机允许建立的最大连接数。设置为 `0` 表示不限制。 + * `useCustomRawHeaders`: 使用预定义header避免github waf对应zh-CN的封锁 + * 类型: 布尔值(`bool`) + * 默认值: `false`(停用) + * 说明: 启用后, 拉取raw文件会使用程序预定义的固定headers, 而不是原先的复制行为 * **`[gitclone]` - Git 克隆配置** From ad4d8eb670b6ed93d34c117cfcc14fe36202df1a Mon Sep 17 00:00:00 2001 From: wjqserver <114663932+WJQSERVER@users.noreply.github.com> Date: Tue, 29 Apr 2025 22:25:20 +0800 Subject: [PATCH 6/6] remove unuse code --- proxy/gitreq.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/proxy/gitreq.go b/proxy/gitreq.go index a050e29..81293cd 100644 --- a/proxy/gitreq.go +++ b/proxy/gitreq.go @@ -122,11 +122,5 @@ func GitReq(ctx context.Context, c *app.RequestContext, u string, cfg *config.Co c.Response.Header.Set("Expires", "0") } - bodySize, _ := strconv.Atoi(contentLength) - - if contentLength != "" { - c.SetBodyStream(resp.Body, bodySize) - return - } c.SetBodyStream(resp.Body, -1) }