This commit is contained in:
wjqserver 2025-03-11 18:07:17 +08:00
parent e2faa497ab
commit 498266e08e
12 changed files with 207 additions and 166 deletions

View file

@ -126,11 +126,7 @@ func ChunkedProxyRequest(c *gin.Context, u string, cfg *config.Config, mode stri
c.Status(resp.StatusCode)
// 使用固定32KB缓冲池
buffer := BufferPool.Get().([]byte)
defer BufferPool.Put(buffer)
_, err = io.CopyBuffer(c.Writer, resp.Body, buffer)
_, err = io.CopyBuffer(c.Writer, resp.Body, nil)
if err != nil {
logError("%s %s %s %s %s Failed to copy response body: %v", c.ClientIP(), method, u, c.Request.Header.Get("User-Agent"), c.Request.Proto, err)
return

View file

@ -6,7 +6,9 @@ import (
"ghproxy/config"
"io"
"net/http"
"net/url"
"strconv"
"strings"
"github.com/gin-gonic/gin"
)
@ -15,38 +17,15 @@ func GitReq(c *gin.Context, u string, cfg *config.Config, mode string, runMode s
method := c.Request.Method
logInfo("%s %s %s %s %s", c.ClientIP(), method, u, c.Request.Header.Get("User-Agent"), c.Request.Proto)
// 发送HEAD请求, 预获取Content-Length
headReq, err := client.NewRequest("HEAD", u, nil)
if err != nil {
HandleError(c, fmt.Sprintf("Failed to create request: %v", err))
return
}
setRequestHeaders(c, headReq)
AuthPassThrough(c, cfg, headReq)
headResp, err := client.Do(headReq)
if err != nil {
HandleError(c, fmt.Sprintf("Failed to send request: %v", err))
return
}
// defer headResp.Body.Close()
defer func(Body io.ReadCloser) {
if err := Body.Close(); err != nil {
logError("Failed to close response body: %v", err)
}
}(headResp.Body)
contentLength := headResp.Header.Get("Content-Length")
sizelimit := cfg.Server.SizeLimit * 1024 * 1024
if contentLength != "" {
size, err := strconv.Atoi(contentLength)
if err == nil && size > sizelimit {
finalURL := headResp.Request.URL.String()
c.Redirect(http.StatusMovedPermanently, finalURL)
logWarning("%s %s %s %s %s Final-URL: %s Size-Limit-Exceeded: %d", c.ClientIP(), c.Request.Method, c.Request.URL.String(), c.Request.Header.Get("User-Agent"), c.Request.Proto, finalURL, size)
logInfo("U:%s", u)
if cfg.GitClone.Mode == "cache" {
userPath, repoPath, remainingPath, err := extractParts(u)
if err != nil {
HandleError(c, fmt.Sprintf("Failed to extract parts from URL: %v", err))
return
}
// 构建新url
u = cfg.GitClone.SmartGitAddr + userPath + repoPath + remainingPath
}
body, err := readRequestBody(c)
@ -77,9 +56,10 @@ func GitReq(c *gin.Context, u string, cfg *config.Config, mode string, runMode s
}
}(resp.Body)
contentLength = resp.Header.Get("Content-Length")
contentLength := resp.Header.Get("Content-Length")
if contentLength != "" {
size, err := strconv.Atoi(contentLength)
sizelimit := cfg.Server.SizeLimit * 1024 * 1024
if err == nil && size > sizelimit {
finalURL := resp.Request.URL.String()
c.Redirect(http.StatusMovedPermanently, finalURL)
@ -104,14 +84,6 @@ func GitReq(c *gin.Context, u string, cfg *config.Config, mode string, runMode s
resp.Header.Del(header)
}
/*
if cfg.CORS.Enabled {
c.Header("Access-Control-Allow-Origin", "*")
} else {
c.Header("Access-Control-Allow-Origin", "")
}
*/
switch cfg.Server.Cors {
case "*":
c.Header("Access-Control-Allow-Origin", "*")
@ -136,4 +108,43 @@ func GitReq(c *gin.Context, u string, cfg *config.Config, mode string, runMode s
} else {
c.Writer.Flush() // 确保刷入
}
/*
_, err = copyb.CopyBuffer(c.Writer, resp.Body, nil)
if err != nil {
logError("%s %s %s %s %s Failed to copy response body: %v", c.ClientIP(), method, u, c.Request.Header.Get("User-Agent"), c.Request.Proto, err)
return
} else {
c.Writer.Flush() // 确保刷入
}
*/
}
// extractParts 从给定的 URL 中提取所需的部分
func extractParts(rawURL string) (string, string, string, error) {
// 解析 URL
parsedURL, err := url.Parse(rawURL)
if err != nil {
return "", "", "", err
}
// 获取路径部分并分割
pathParts := strings.Split(parsedURL.Path, "/")
// 提取所需的部分
if len(pathParts) < 3 {
return "", "", "", fmt.Errorf("URL path is too short")
}
// 提取 /WJQSERVER-STUDIO 和 /go-utils.git
repoOwner := "/" + pathParts[1]
repoName := "/" + pathParts[2]
// 剩余部分
remainingPath := strings.Join(pathParts[3:], "/")
if remainingPath != "" {
remainingPath = "/" + remainingPath
}
return repoOwner, repoName, remainingPath, nil
}

View file

@ -12,121 +12,159 @@ import (
"github.com/gin-gonic/gin"
)
var exps = []*regexp.Regexp{
regexp.MustCompile(`^(?:https?://)?github\.com/([^/]+)/([^/]+)/(?:releases|archive)/.*`), // 匹配 GitHub Releases 或 Archive 链接
regexp.MustCompile(`^(?:https?://)?github\.com/([^/]+)/([^/]+)/(?:blob|raw)/.*`), // 匹配 GitHub Blob 或 Raw 链接
regexp.MustCompile(`^(?:https?://)?github\.com/([^/]+)/([^/]+)/(?:info|git-).*`), // 匹配 GitHub Info 或 Git 相关链接 (例如 .gitattributes, .gitignore)
regexp.MustCompile(`^(?:https?://)?raw\.github(?:usercontent|)\.com/([^/]+)/([^/]+)/.+?/.+`), // 匹配 raw.githubusercontent.com 链接
regexp.MustCompile(`^(?:https?://)?gist\.github(?:usercontent|)\.com/([^/]+)/.+?/.+`), // 匹配 gist.githubusercontent.com 链接
regexp.MustCompile(`^(?:https?://)?api\.github\.com/repos/([^/]+)/([^/]+)/.*`), // 匹配 api.github.com/repos 链接 (GitHub API)
}
// NoRouteHandler 是 Gin 框架的 NoRoute 处理器函数,用于处理所有未匹配到预定义路由的请求
// 此函数实现了请求的频率限制、URL 路径解析、白名单/黑名单检查、URL 类型匹配和最终的代理请求处理
func NoRouteHandler(cfg *config.Config, limiter *rate.RateLimiter, iplimiter *rate.IPRateLimiter, runMode string) gin.HandlerFunc {
return func(c *gin.Context) {
// 限制访问频率
if cfg.RateLimit.Enabled {
// **频率限制处理**
if cfg.RateLimit.Enabled { // 检查是否启用频率限制
var allowed bool
var allowed bool // 用于标记是否允许请求
switch cfg.RateLimit.RateMethod {
case "ip":
allowed = iplimiter.Allow(c.ClientIP())
case "total":
allowed = limiter.Allow()
default:
logWarning("Invalid RateLimit Method")
return
switch cfg.RateLimit.RateMethod { // 根据配置的频率限制方法选择
case "ip": // 基于 IP 地址的频率限制
allowed = iplimiter.Allow(c.ClientIP()) // 使用 IPRateLimiter 检查客户端 IP 是否允许请求
case "total": // 基于总请求量的频率限制
allowed = limiter.Allow() // 使用 RateLimiter 检查总请求量是否允许请求
default: // 无效的频率限制方法
logWarning("Invalid RateLimit Method") // 记录警告日志
return // 中断请求处理
}
if !allowed {
c.JSON(http.StatusTooManyRequests, gin.H{"error": "Too Many Requests"})
logWarning("%s %s %s %s %s 429-TooManyRequests", c.ClientIP(), c.Request.Method, c.Request.URL.RequestURI(), c.Request.Header.Get("User-Agent"), c.Request.Proto)
return
if !allowed { // 如果请求被频率限制阻止
c.JSON(http.StatusTooManyRequests, gin.H{"error": "Too Many Requests"}) // 返回 429 状态码和错误信息
logWarning("%s %s %s %s %s 429-TooManyRequests", c.ClientIP(), c.Request.Method, c.Request.URL.RequestURI(), c.Request.Header.Get("User-Agent"), c.Request.Proto) // 记录警告日志
return // 中断请求处理
}
}
//rawPath := strings.TrimPrefix(c.Request.URL.Path, "/") // 去掉前缀/
rawPath := strings.TrimPrefix(c.Request.URL.RequestURI(), "/") // 去掉前缀/
re := regexp.MustCompile(`^(http:|https:)?/?/?(.*)`) // 匹配http://或https://开头的路径
matches := re.FindStringSubmatch(rawPath) // 匹配路径
rawPath := strings.TrimPrefix(c.Request.URL.RequestURI(), "/") // 去掉 URL 前缀的斜杠 '/', 获取原始路径 (例如: /https://github.com/user/repo -> https://github.com/user/repo)
re := regexp.MustCompile(`^(http:|https:)?/?/?(.*)`) // 定义正则表达式,匹配以 http:// 或 https:// 开头的路径,并捕获协议和剩余部分
matches := re.FindStringSubmatch(rawPath) // 使用正则表达式匹配原始路径
// 匹配路径错误处理
if len(matches) < 3 {
errMsg := fmt.Sprintf("%s %s %s %s %s Invalid URL", c.ClientIP(), c.Request.Method, rawPath, c.Request.Header.Get("User-Agent"), c.Request.Proto)
logWarning(errMsg)
c.String(http.StatusForbidden, "Invalid URL Format. Path: %s", rawPath)
return
// **路径匹配错误处理**
if len(matches) < 3 { // 如果匹配结果少于 3 个子串 (完整匹配 + 协议 + 剩余部分),则说明 URL 格式无效
errMsg := fmt.Sprintf("%s %s %s %s %s Invalid URL", c.ClientIP(), c.Request.Method, rawPath, c.Request.Header.Get("User-Agent"), c.Request.Proto) // 构建错误日志信息
logWarning(errMsg) // 记录警告日志
c.String(http.StatusForbidden, "Invalid URL Format. Path: %s", rawPath) // 返回 403 状态码和错误信息,提示 URL 格式无效
return // 中断请求处理
}
// 制作url
rawPath = "https://" + matches[2]
// **构建完整的 URL**
rawPath = "https://" + matches[2] // 从匹配结果中提取 URL 的剩余部分,并添加 https:// 协议头,构建完整的 URL
username, repo := MatchUserRepo(rawPath, cfg, c, matches) // 匹配用户名和仓库名
username, repo := MatchUserRepo(rawPath, cfg, c, matches) // 调用 MatchUserRepo 函数,从 URL 中提取用户名和仓库名
logInfo("%s %s %s %s %s Matched-Username: %s, Matched-Repo: %s", c.ClientIP(), c.Request.Method, rawPath, c.Request.Header.Get("User-Agent"), c.Request.Proto, username, repo)
logInfo("%s %s %s %s %s Matched-Username: %s, Matched-Repo: %s", c.ClientIP(), c.Request.Method, rawPath, c.Request.Header.Get("User-Agent"), c.Request.Proto, username, repo) // 记录 info 日志,包含匹配到的用户名和仓库名
// dump log 记录详细信息 c.ClientIP(), c.Request.Method, rawPath, c.Request.Header.Get("User-Agent"), c.Request.Proto, full Header
LogDump("%s %s %s %s %s %s", c.ClientIP(), c.Request.Method, rawPath, c.Request.Header.Get("User-Agent"), c.Request.Proto, c.Request.Header)
repouser := fmt.Sprintf("%s/%s", username, repo)
LogDump("%s %s %s %s %s %s", c.ClientIP(), c.Request.Method, rawPath, c.Request.Header.Get("User-Agent"), c.Request.Proto, c.Request.Header) // 记录 dump 日志,包含更详细的请求头信息
repouser := fmt.Sprintf("%s/%s", username, repo) // 构建 "用户名/仓库名" 格式的字符串
// 白名单检查
if cfg.Whitelist.Enabled {
whitelist := auth.CheckWhitelist(username, repo)
if !whitelist {
logErrMsg := fmt.Sprintf("%s %s %s %s %s Whitelist Blocked repo: %s", c.ClientIP(), c.Request.Method, rawPath, c.Request.Header.Get("User-Agent"), c.Request.Proto, repouser)
errMsg := fmt.Sprintf("Whitelist Blocked repo: %s", repouser)
c.JSON(http.StatusForbidden, gin.H{"error": errMsg})
logWarning(logErrMsg)
return
// **白名单检查**
if cfg.Whitelist.Enabled { // 检查是否启用白名单
whitelist := auth.CheckWhitelist(username, repo) // 调用 CheckWhitelist 函数检查当前仓库是否在白名单中
if !whitelist { // 如果仓库不在白名单中
logErrMsg := fmt.Sprintf("%s %s %s %s %s Whitelist Blocked repo: %s", c.ClientIP(), c.Request.Method, rawPath, c.Request.Header.Get("User-Agent"), c.Request.Proto, repouser) // 构建错误日志信息
errMsg := fmt.Sprintf("Whitelist Blocked repo: %s", repouser) // 构建返回给客户端的错误信息
c.JSON(http.StatusForbidden, gin.H{"error": errMsg}) // 返回 403 状态码和 JSON 错误信息
logWarning(logErrMsg) // 记录警告日志
return // 中断请求处理
}
}
// 黑名单检查
if cfg.Blacklist.Enabled {
blacklist := auth.CheckBlacklist(username, repo)
if blacklist {
logErrMsg := fmt.Sprintf("%s %s %s %s %s Blacklist Blocked repo: %s", c.ClientIP(), c.Request.Method, rawPath, c.Request.Header.Get("User-Agent"), c.Request.Proto, repouser)
errMsg := fmt.Sprintf("Blacklist Blocked repo: %s", repouser)
c.JSON(http.StatusForbidden, gin.H{"error": errMsg})
logWarning(logErrMsg)
return
// **黑名单检查**
if cfg.Blacklist.Enabled { // 检查是否启用黑名单
blacklist := auth.CheckBlacklist(username, repo) // 调用 CheckBlacklist 函数检查当前仓库是否在黑名单中
if blacklist { // 如果仓库在黑名单中
logErrMsg := fmt.Sprintf("%s %s %s %s %s Blacklist Blocked repo: %s", c.ClientIP(), c.Request.Method, rawPath, c.Request.Header.Get("User-Agent"), c.Request.Proto, repouser) // 构建错误日志信息
errMsg := fmt.Sprintf("Blacklist Blocked repo: %s", repouser) // 构建返回给客户端的错误信息
c.JSON(http.StatusForbidden, gin.H{"error": errMsg}) // 返回 403 状态码和 JSON 错误信息
logWarning(logErrMsg) // 记录警告日志
return // 中断请求处理
}
}
matches = CheckURL(rawPath, c)
if matches == nil {
c.AbortWithStatus(http.StatusNotFound)
logWarning("%s %s %s %s %s 404-NOMATCH", c.ClientIP(), c.Request.Method, rawPath, c.Request.Header.Get("User-Agent"), c.Request.Proto)
return
}
// 若匹配api.github.com/repos/用户名/仓库名/路径, 则检查是否开启HeaderAuth
if exps[5].MatchString(rawPath) {
if cfg.Auth.AuthMethod != "header" || !cfg.Auth.Enabled {
c.JSON(http.StatusForbidden, gin.H{"error": "HeaderAuth is not enabled."})
logError("%s %s %s %s %s HeaderAuth-Error: HeaderAuth is not enabled.", c.ClientIP(), c.Request.Method, rawPath, c.Request.Header.Get("User-Agent"), c.Request.Proto)
return
}
}
// 处理blob/raw路径
if exps[1].MatchString(rawPath) {
rawPath = strings.Replace(rawPath, "/blob/", "/raw/", 1)
}
// 鉴权
authcheck, err := auth.AuthHandler(c, cfg)
if !authcheck {
c.AbortWithStatusJSON(401, gin.H{"error": "Unauthorized"})
logWarning("%s %s %s %s %s Auth-Error: %v", c.ClientIP(), c.Request.Method, rawPath, c.Request.Header.Get("User-Agent"), c.Request.Proto, err)
return
}
// IP METHOD URL USERAGENT PROTO MATCHES
logDebug("%s %s %s %s %s Matches: %v", c.ClientIP(), c.Request.Method, rawPath, c.Request.Header.Get("User-Agent"), c.Request.Proto, matches)
var matchedIndex = -1 // 用于存储匹配到的正则表达式索引,初始化为 -1 表示未匹配
// **优化的 URL 匹配逻辑:基于关键词分类匹配**
switch {
case exps[0].MatchString(rawPath), exps[1].MatchString(rawPath), exps[3].MatchString(rawPath), exps[4].MatchString(rawPath):
//ProxyRequest(c, rawPath, cfg, "chrome", runMode)
ChunkedProxyRequest(c, rawPath, cfg, "chrome", runMode) // dev test chunk
case exps[2].MatchString(rawPath):
//ProxyRequest(c, rawPath, cfg, "git", runMode)
GitReq(c, rawPath, cfg, "git", runMode)
default:
c.String(http.StatusForbidden, "Invalid input.")
fmt.Println("Invalid input.")
case strings.Contains(rawPath, "/releases/") || strings.Contains(rawPath, "/archive/"): // 检查 URL 中是否包含 "/releases/" 或 "/archive/" 关键词
matchedIndex = 0 // 如果包含,则匹配 exps[0] (GitHub Releases/Archive 链接)
case strings.Contains(rawPath, "/blob/") || strings.Contains(rawPath, "/raw/"): // 检查 URL 中是否包含 "/blob/" 或 "/raw/" 关键词
matchedIndex = 1 // 如果包含,则匹配 exps[1] (GitHub Blob/Raw 链接)
case strings.Contains(rawPath, "/info/") || strings.Contains(rawPath, "/git-"): // 检查 URL 中是否包含 "/info/" 或 "/git-" 关键词
matchedIndex = 2 // 如果包含,则匹配 exps[2] (GitHub Info/Git 相关链接)
case strings.Contains(rawPath, "raw.githubusercontent.com"): // 检查 URL 中是否包含 "raw.githubusercontent.com" 域名
matchedIndex = 3 // 如果包含,则匹配 exps[3] (raw.githubusercontent.com 链接)
case strings.Contains(rawPath, "gist.githubusercontent.com"): // 检查 URL 中是否包含 "gist.githubusercontent.com" 域名
matchedIndex = 4 // 如果包含,则匹配 exps[4] (gist.githubusercontent.com 链接)
case strings.Contains(rawPath, "api.github.com/repos/"): // 检查 URL 中是否包含 "api.github.com/repos/" 路径前缀
matchedIndex = 5 // 如果包含,则匹配 exps[5] (api.github.com/repos 链接)
}
if matchedIndex == -1 { // 如果没有任何关键词匹配到,则说明 URL 类型无法识别
c.AbortWithStatus(http.StatusNotFound) // 返回 404 状态码
logWarning("%s %s %s %s %s 404-NOMATCH", c.ClientIP(), c.Request.Method, rawPath, c.Request.Header.Get("User-Agent"), c.Request.Proto) // 记录警告日志
return // 中断请求处理
}
// **使用分类匹配到的正则表达式进行精确匹配**
exp := exps[matchedIndex]
matches = exp.FindStringSubmatch(rawPath)
if len(matches) == 0 {
// 如果精确匹配失败 (例如,关键词匹配到 releases但实际 URL 格式不符合 releases 的正则)
c.AbortWithStatus(http.StatusNotFound)
logWarning("%s %s %s %s %s 404-NOMATCH-ExpSpecific", c.ClientIP(), c.Request.Method, rawPath, c.Request.Header.Get("User-Agent"), c.Request.Proto) // 记录警告日志,表明是特定正则匹配失败
return
}
// **HeaderAuth 鉴权检查 (仅针对 api.github.com/repos 链接)**
if matchedIndex == 5 { // 如果匹配的是 api.github.com/repos 链接 (对应 exps[5])
if cfg.Auth.AuthMethod != "header" || !cfg.Auth.Enabled { // 检查是否启用了 HeaderAuth 并且 AuthMethod 配置为 "header"
c.JSON(http.StatusForbidden, gin.H{"error": "HeaderAuth is not enabled."}) // 返回 403 状态码和错误信息,提示 HeaderAuth 未启用
logError("%s %s %s %s %s HeaderAuth-Error: HeaderAuth is not enabled.", c.ClientIP(), c.Request.Method, rawPath, c.Request.Header.Get("User-Agent"), c.Request.Proto) // 记录错误日志
return // 中断请求处理
}
}
// **处理 blob/raw 路径**
if matchedIndex == 1 { // 如果匹配的是 GitHub Blob/Raw 链接 (对应 exps[1])
rawPath = strings.Replace(rawPath, "/blob/", "/raw/", 1) // 将 URL 中的 "/blob/" 替换为 "/raw/",获取 raw 链接 (用于下载原始文件内容)
}
// **通用鉴权处理**
authcheck, err := auth.AuthHandler(c, cfg) // 调用 AuthHandler 函数进行通用鉴权检查 (例如,基于 Cookie 或 Header 的鉴权)
if !authcheck { // 如果鉴权失败
c.AbortWithStatusJSON(401, gin.H{"error": "Unauthorized"}) // 返回 401 状态码和 JSON 错误信息,提示未授权
logWarning("%s %s %s %s %s Auth-Error: %v", c.ClientIP(), c.Request.Method, rawPath, c.Request.Header.Get("User-Agent"), c.Request.Proto, err) // 记录警告日志,包含鉴权错误信息
return // 中断请求处理
}
// **Debug 日志记录匹配结果**
logDebug("%s %s %s %s %s Matches: %v", c.ClientIP(), c.Request.Method, rawPath, c.Request.Header.Get("User-Agent"), c.Request.Proto, matches) // 记录 debug 日志,包含匹配结果信息
// **根据匹配到的 URL 类型,进行不同的代理请求处理**
switch matchedIndex {
case 0, 1, 3, 4: // 如果匹配的是 Releases/Archive, Blob/Raw, raw.githubusercontent.com 或 gist.githubusercontent.com 链接 (对应 exps[0], exps[1], exps[3], exps[4])
//ProxyRequest(c, rawPath, cfg, "chrome", runMode) // 原始的 ProxyRequest 函数 (可能一次性读取全部响应)
ChunkedProxyRequest(c, rawPath, cfg, "chrome", runMode) // 使用 ChunkedProxyRequest 函数进行分块代理 (更高效,特别是对于大文件)
case 2: // 如果匹配的是 Info/Git 相关链接 (对应 exps[2])
//ProxyRequest(c, rawPath, cfg, "git", runMode) // 原始的 ProxyRequest 函数
GitReq(c, rawPath, cfg, "git", runMode) // 使用 GitReq 函数处理 Git 相关请求 (针对 .gitattributes, .gitignore 等)
default: // 如果匹配到其他类型 (理论上不应该发生,因为前面的 matchedIndex == -1 已经处理了未识别类型)
c.String(http.StatusForbidden, "Invalid input.") // 返回 403 状态码和错误信息,提示无效输入
fmt.Println("Invalid input.") // 打印错误信息到控制台
return // 中断请求处理
}
}
}

View file

@ -4,7 +4,6 @@ import (
"fmt"
"io"
"net/http"
"regexp"
"github.com/WJQSERVER-STUDIO/go-utils/logger"
"github.com/gin-gonic/gin"
@ -20,15 +19,6 @@ var (
logError = logger.LogError
)
var exps = []*regexp.Regexp{
regexp.MustCompile(`^(?:https?://)?github\.com/([^/]+)/([^/]+)/(?:releases|archive)/.*`),
regexp.MustCompile(`^(?:https?://)?github\.com/([^/]+)/([^/]+)/(?:blob|raw)/.*`),
regexp.MustCompile(`^(?:https?://)?github\.com/([^/]+)/([^/]+)/(?:info|git-).*`),
regexp.MustCompile(`^(?:https?://)?raw\.github(?:usercontent|)\.com/([^/]+)/([^/]+)/.+?/.+`),
regexp.MustCompile(`^(?:https?://)?gist\.github(?:usercontent|)\.com/([^/]+)/.+?/.+`),
regexp.MustCompile(`^(?:https?://)?api\.github\.com/repos/([^/]+)/([^/]+)/.*`),
}
// 读取请求体
func readRequestBody(c *gin.Context) ([]byte, error) {
body, err := io.ReadAll(c.Request.Body)