diff --git a/.gitignore b/.gitignore index 0ad54a9..1834d15 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ demo.toml list.json repos pages -*_test \ No newline at end of file +*_test +.* \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index a9a1114..427ec1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # 更新日志 +3.3.3 - 2025-05-20 +--- +- CHANGE: 加入`senseClientDisconnection`与`async`配置项 + +25w39a - 2025-05-19 +--- +- PRE-RELEASE: 此版本是v3.3.3预发布版本,请勿在生产环境中使用; +- CHANGE: 加入`senseClientDisconnection`与`async`配置项 + 3.3.2 - 2025-05-18 --- - CHANGE: 默认主题改为`design` diff --git a/DEV-VERSION b/DEV-VERSION index 5c77f30..fdef6bc 100644 --- a/DEV-VERSION +++ b/DEV-VERSION @@ -1 +1 @@ -25w38a \ No newline at end of file +25w39a \ No newline at end of file diff --git a/VERSION b/VERSION index 5436ea0..3f09e91 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.3.2 \ No newline at end of file +3.3.3 \ No newline at end of file diff --git a/config/config.go b/config/config.go index c5b8aa0..c1300dc 100644 --- a/config/config.go +++ b/config/config.go @@ -34,14 +34,15 @@ debug = false */ type ServerConfig struct { - Port int `toml:"port"` - Host string `toml:"host"` - NetLib string `toml:"netlib"` - SizeLimit int `toml:"sizeLimit"` - MemLimit int64 `toml:"memLimit"` - H2C bool `toml:"H2C"` - Cors string `toml:"cors"` - Debug bool `toml:"debug"` + Port int `toml:"port"` + Host string `toml:"host"` + NetLib string `toml:"netlib"` + SenseClientDisconnection bool `toml:"senseClientDisconnection"` + SizeLimit int `toml:"sizeLimit"` + MemLimit int64 `toml:"memLimit"` + H2C bool `toml:"H2C"` + Cors string `toml:"cors"` + Debug bool `toml:"debug"` } /* @@ -98,6 +99,7 @@ type LogConfig struct { LogFilePath string `toml:"logFilePath"` MaxLogSize int `toml:"maxLogSize"` Level string `toml:"level"` + Async bool `toml:"async"` HertZLogPath string `toml:"hertzLogPath"` } diff --git a/config/config.toml b/config/config.toml index ca7b80a..c8ea1f5 100644 --- a/config/config.toml +++ b/config/config.toml @@ -2,6 +2,7 @@ host = "0.0.0.0" port = 8080 netlib = "netpoll" # "netpoll" / "std" "standard" "net/http" "net" +senseClientDisconnection = false sizeLimit = 125 # MB memLimit = 0 # MB H2C = true @@ -33,6 +34,7 @@ staticDir = "/data/www" logFilePath = "/data/ghproxy/log/ghproxy.log" maxLogSize = 5 # MB level = "info" # dump, debug, info, warn, error, none +async = false hertzLogPath = "/data/ghproxy/log/hertz.log" [auth] diff --git a/go.mod b/go.mod index e798fb1..d4a930a 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.24.3 require ( github.com/BurntSushi/toml v1.5.0 github.com/WJQSERVER-STUDIO/httpc v0.5.1 - github.com/WJQSERVER-STUDIO/logger v1.6.0 + github.com/WJQSERVER-STUDIO/logger v1.7.1 github.com/cloudwego/hertz v0.10.0 github.com/hertz-contrib/http2 v0.1.8 golang.org/x/net v0.40.0 @@ -16,7 +16,7 @@ require github.com/WJQSERVER-STUDIO/go-utils/limitreader v0.0.2 require ( github.com/WJQSERVER-STUDIO/go-utils/copyb v0.0.4 // indirect - github.com/WJQSERVER-STUDIO/go-utils/log v0.0.2 // indirect + github.com/WJQSERVER-STUDIO/go-utils/log v0.0.3 // indirect github.com/bytedance/gopkg v0.1.2 // indirect github.com/bytedance/sonic v1.13.2 // indirect github.com/bytedance/sonic/loader v0.2.4 // indirect @@ -40,3 +40,4 @@ require ( ) //replace github.com/WJQSERVER-STUDIO/httpc v0.5.1 => /data/github/WJQSERVER-STUDIO/httpc +//replace github.com/WJQSERVER-STUDIO/logger v1.6.0 => /data/github/WJQSERVER-STUDIO/logger diff --git a/go.sum b/go.sum index 68f4222..a614fc3 100644 --- a/go.sum +++ b/go.sum @@ -4,12 +4,12 @@ github.com/WJQSERVER-STUDIO/go-utils/copyb v0.0.4 h1:JLtFd00AdFg/TP+dtvIzLkdHwKU github.com/WJQSERVER-STUDIO/go-utils/copyb v0.0.4/go.mod h1:FZ6XE+4TKy4MOfX1xWKe6Rwsg0ucYFCdNh1KLvyKTfc= github.com/WJQSERVER-STUDIO/go-utils/limitreader v0.0.2 h1:8bBkKk6E2Zr+I5szL7gyc5f0DK8N9agIJCpM1Cqw2NE= github.com/WJQSERVER-STUDIO/go-utils/limitreader v0.0.2/go.mod h1:yPX8xuZH+py7eLJwOYj3VVI/4/Yuy5+x8Mhq8qezcPg= -github.com/WJQSERVER-STUDIO/go-utils/log v0.0.2 h1:9CSf+V0ZQPl2ijC/g6v/ObemmhpKcikKVIodsaLExTA= -github.com/WJQSERVER-STUDIO/go-utils/log v0.0.2/go.mod h1:j9Q+xnwpOfve7/uJnZ2izRQw6NNoXjvJHz7vUQAaLZE= +github.com/WJQSERVER-STUDIO/go-utils/log v0.0.3 h1:t6nyLhmo9pSfVHm1Wu1WyLsTpXFSjSpQtVKqEDpiZ5Q= +github.com/WJQSERVER-STUDIO/go-utils/log v0.0.3/go.mod h1:j9Q+xnwpOfve7/uJnZ2izRQw6NNoXjvJHz7vUQAaLZE= github.com/WJQSERVER-STUDIO/httpc v0.5.1 h1:+TKCPYBuj7PAHuiduGCGAqsHAa4QtsUfoVwRN777q64= github.com/WJQSERVER-STUDIO/httpc v0.5.1/go.mod h1:M7KNUZjjhCkzzcg9lBPs9YfkImI+7vqjAyjdA19+joE= -github.com/WJQSERVER-STUDIO/logger v1.6.0 h1:xK2xV7hlkMXaWzvj4+cNoNWA+JfnJaHX6VU+RrPnr7Q= -github.com/WJQSERVER-STUDIO/logger v1.6.0/go.mod h1:TICMsR7geROHBg6rxwkqUNGydo34XVsX93yeoxyfuyY= +github.com/WJQSERVER-STUDIO/logger v1.7.1 h1:sAFsF3umimY0Vmue5WnGf1Qxvm/vlhK2srZakWVtlFU= +github.com/WJQSERVER-STUDIO/logger v1.7.1/go.mod h1:cvP0XdFIMLtDWOZeKhklshzipkVU1zufsU4rKNfoM24= github.com/bytedance/gopkg v0.1.1/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= github.com/bytedance/gopkg v0.1.2 h1:8o2feYuxknDpN+O7kPwvSXfMEKfYvJYiA2K7aonoMEQ= github.com/bytedance/gopkg v0.1.2/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= diff --git a/main.go b/main.go index a9c37ae..484d59f 100644 --- a/main.go +++ b/main.go @@ -121,6 +121,7 @@ func loadConfig() { func setupLogger(cfg *config.Config) { var err error + err = logger.Init(cfg.Log.LogFilePath, cfg.Log.MaxLogSize) if err != nil { fmt.Printf("Failed to initialize logger: %v\n", err) @@ -131,6 +132,8 @@ func setupLogger(cfg *config.Config) { fmt.Printf("Logger Level Error: %v\n", err) os.Exit(1) } + logger.SetAsync(cfg.Log.Async) + fmt.Printf("Log Level: %s\n", cfg.Log.Level) logDebug("Config File Path: ", cfgfile) logDebug("Loaded config: %v\n", cfg) @@ -401,13 +404,13 @@ func main() { r = server.New( server.WithH2C(true), server.WithHostPorts(addr), - server.WithSenseClientDisconnection(true), + server.WithSenseClientDisconnection(cfg.Server.SenseClientDisconnection), ) r.AddProtocol("h2", factory.NewServerFactory()) } else { r = server.New( server.WithHostPorts(addr), - server.WithSenseClientDisconnection(true), + server.WithSenseClientDisconnection(cfg.Server.SenseClientDisconnection), ) } } else { @@ -465,6 +468,51 @@ func main() { proxy.RoutingHandler(cfg, limiter, iplimiter)(ctx, c) }) + // for 3.4.0 + + /* + r.GET("/v2/", func(ctx context.Context, c *app.RequestContext) { + proxy.GhcrRouting(cfg)(ctx, c) + + /* + //proxy.GhcrRouting(cfg)(ctx, c) + // 返回200与空json + //c.JSON(200, map[string]interface{}{}) + emptyJSON := "{}" + //emptyJSON := `{"name":"disable-list-tags","tags":[]}` + c.Header("Content-Type", "application/json") + c.Header("Content-Length", fmt.Sprint(len(emptyJSON))) + c.String(200, emptyJSON) + */ + /* + emptyJSON := "{}" + c.Header("Content-Type", "application/json") + c.Header("Content-Length", fmt.Sprint(len(emptyJSON))) + + c.Header("Docker-Distribution-API-Version", "registry/2.0") + + c.Status(200) + c.Write([]byte(emptyJSON)) + */ + + /* + w := adaptor.GetCompatResponseWriter(&c.Response) + + const emptyJSON = "{}" + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Length", fmt.Sprint(len(emptyJSON))) + w.Header().Del("Server") + + fmt.Fprint(w, emptyJSON) + */ + /* + }) + + r.Any("/v2/:target/*filepath", func(ctx context.Context, c *app.RequestContext) { + proxy.GhcrRouting(cfg)(ctx, c) + }) + */ + r.Any("/v2/*filepath", func(ctx context.Context, c *app.RequestContext) { proxy.GhcrRouting(cfg)(ctx, c) }) diff --git a/proxy/match.go b/proxy/match.go index 8c64623..9db0c23 100644 --- a/proxy/match.go +++ b/proxy/match.go @@ -1,11 +1,8 @@ package proxy import ( - "bufio" - "compress/gzip" "fmt" "ghproxy/config" - "io" "net/url" "regexp" "strings" @@ -104,62 +101,6 @@ func Matcher(rawPath string, cfg *config.Config) (string, string, string, *GHPro return "", "", "", NewErrorWithStatusLookup(404, errMsg) } -func EditorMatcher(rawPath string, cfg *config.Config) (bool, error) { - // 匹配 "https://github.com"开头的链接 - if strings.HasPrefix(rawPath, "https://github.com") { - return true, nil - } - // 匹配 "https://raw.githubusercontent.com"开头的链接 - if strings.HasPrefix(rawPath, "https://raw.githubusercontent.com") { - return true, nil - } - // 匹配 "https://raw.github.com"开头的链接 - if strings.HasPrefix(rawPath, "https://raw.github.com") { - return true, nil - } - // 匹配 "https://gist.githubusercontent.com"开头的链接 - if strings.HasPrefix(rawPath, "https://gist.githubusercontent.com") { - return true, nil - } - // 匹配 "https://gist.github.com"开头的链接 - if strings.HasPrefix(rawPath, "https://gist.github.com") { - return true, nil - } - if cfg.Shell.RewriteAPI { - // 匹配 "https://api.github.com/"开头的链接 - if strings.HasPrefix(rawPath, "https://api.github.com") { - return true, nil - } - } - return false, nil -} - -// 匹配文件扩展名是sh的rawPath -func MatcherShell(rawPath string) bool { - return strings.HasSuffix(rawPath, ".sh") -} - -// LinkProcessor 是一个函数类型,用于处理提取到的链接。 -type LinkProcessor func(string) string - -// 自定义 URL 修改函数 -func modifyURL(url string, host string, cfg *config.Config) string { - // 去除url内的https://或http:// - matched, err := EditorMatcher(url, cfg) - if err != nil { - logDump("Invalid URL: %s", url) - return url - } - if matched { - var u = url - u = strings.TrimPrefix(u, "https://") - u = strings.TrimPrefix(u, "http://") - logDump("Modified URL: %s", "https://"+host+"/"+u) - return "https://" + host + "/" + u - } - return url -} - var ( matchedMatchers = []string{ "blob", @@ -211,118 +152,3 @@ func extractParts(rawURL string) (string, string, string, url.Values, error) { } var urlPattern = regexp.MustCompile(`https?://[^\s'"]+`) - -// processLinks 处理链接,返回包含处理后数据的 io.Reader -func processLinks(input io.ReadCloser, compress string, host string, cfg *config.Config) (readerOut io.Reader, written int64, err error) { - pipeReader, pipeWriter := io.Pipe() // 创建 io.Pipe - readerOut = pipeReader - - go func() { // 在 Goroutine 中执行写入操作 - defer func() { - if pipeWriter != nil { // 确保 pipeWriter 关闭,即使发生错误 - if err != nil { - if closeErr := pipeWriter.CloseWithError(err); closeErr != nil { // 如果有错误,传递错误给 reader - logError("pipeWriter close with error failed: %v, original error: %v", closeErr, err) - } - } else { - if closeErr := pipeWriter.Close(); closeErr != nil { // 没有错误,正常关闭 - logError("pipeWriter close failed: %v", closeErr) - if err == nil { // 如果之前没有错误,记录关闭错误 - err = closeErr - } - } - } - } - }() - - defer func() { - if err := input.Close(); err != nil { - logError("input close failed: %v", err) - } - - }() - - var bufReader *bufio.Reader - - if compress == "gzip" { - // 解压gzip - gzipReader, gzipErr := gzip.NewReader(input) - if gzipErr != nil { - err = fmt.Errorf("gzip解压错误: %v", gzipErr) - return // Goroutine 中使用 return 返回错误 - } - defer gzipReader.Close() - bufReader = bufio.NewReader(gzipReader) - } else { - bufReader = bufio.NewReader(input) - } - - var bufWriter *bufio.Writer - var gzipWriter *gzip.Writer - - // 根据是否gzip确定 writer 的创建 - if compress == "gzip" { - gzipWriter = gzip.NewWriter(pipeWriter) // 使用 pipeWriter - bufWriter = bufio.NewWriterSize(gzipWriter, 4096) //设置缓冲区大小 - } else { - bufWriter = bufio.NewWriterSize(pipeWriter, 4096) // 使用 pipeWriter - } - - //确保writer关闭 - defer func() { - var closeErr error // 局部变量,用于保存defer中可能发生的错误 - - if gzipWriter != nil { - if closeErr = gzipWriter.Close(); closeErr != nil { - logError("gzipWriter close failed %v", closeErr) - // 如果已经存在错误,则保留。否则,记录此错误。 - if err == nil { - err = closeErr - } - } - } - if flushErr := bufWriter.Flush(); flushErr != nil { - logError("writer flush failed %v", flushErr) - // 如果已经存在错误,则保留。否则,记录此错误。 - if err == nil { - err = flushErr - } - } - }() - - // 使用正则表达式匹配 http 和 https 链接 - for { - line, readErr := bufReader.ReadString('\n') - if readErr != nil { - if readErr == io.EOF { - break // 文件结束 - } - err = fmt.Errorf("读取行错误: %v", readErr) // 传递错误 - return // Goroutine 中使用 return 返回错误 - } - - // 替换所有匹配的 URL - modifiedLine := urlPattern.ReplaceAllStringFunc(line, func(originalURL string) string { - logDump("originalURL: %s", originalURL) - return modifyURL(originalURL, host, cfg) // 假设 modifyURL 函数已定义 - }) - - n, writeErr := bufWriter.WriteString(modifiedLine) - written += int64(n) // 更新写入的字节数 - if writeErr != nil { - err = fmt.Errorf("写入文件错误: %v", writeErr) // 传递错误 - return // Goroutine 中使用 return 返回错误 - } - } - - // 在返回之前,再刷新一次 (虽然 defer 中已经有 flush,但这里再加一次确保及时刷新) - if flushErr := bufWriter.Flush(); flushErr != nil { - if err == nil { // 避免覆盖之前的错误 - err = flushErr - } - return // Goroutine 中使用 return 返回错误 - } - }() - - return readerOut, written, nil // 返回 reader 和 written,error 由 Goroutine 通过 pipeWriter.CloseWithError 传递 -} diff --git a/proxy/nest.go b/proxy/nest.go new file mode 100644 index 0000000..84f55cd --- /dev/null +++ b/proxy/nest.go @@ -0,0 +1,185 @@ +// Copyright 2025 WJQSERVER, WJQSERVER-STUDIO. All rights reserved. +// 使用本源代码受 WSL 2.0(WJQserver Studio License v2.0)与MPL 2.0(Mozilla Public License v2.0)许可协议的约束 +// 此段代码使用双重授权许可, 允许用户选择其中一种许可证 + +package proxy + +import ( + "bufio" + "compress/gzip" + "fmt" + "ghproxy/config" + "io" + "strings" +) + +func EditorMatcher(rawPath string, cfg *config.Config) (bool, error) { + // 匹配 "https://github.com"开头的链接 + if strings.HasPrefix(rawPath, "https://github.com") { + return true, nil + } + // 匹配 "https://raw.githubusercontent.com"开头的链接 + if strings.HasPrefix(rawPath, "https://raw.githubusercontent.com") { + return true, nil + } + // 匹配 "https://raw.github.com"开头的链接 + if strings.HasPrefix(rawPath, "https://raw.github.com") { + return true, nil + } + // 匹配 "https://gist.githubusercontent.com"开头的链接 + if strings.HasPrefix(rawPath, "https://gist.githubusercontent.com") { + return true, nil + } + // 匹配 "https://gist.github.com"开头的链接 + if strings.HasPrefix(rawPath, "https://gist.github.com") { + return true, nil + } + if cfg.Shell.RewriteAPI { + // 匹配 "https://api.github.com/"开头的链接 + if strings.HasPrefix(rawPath, "https://api.github.com") { + return true, nil + } + } + return false, nil +} + +// 匹配文件扩展名是sh的rawPath +func MatcherShell(rawPath string) bool { + return strings.HasSuffix(rawPath, ".sh") +} + +// LinkProcessor 是一个函数类型,用于处理提取到的链接。 +type LinkProcessor func(string) string + +// 自定义 URL 修改函数 +func modifyURL(url string, host string, cfg *config.Config) string { + // 去除url内的https://或http:// + matched, err := EditorMatcher(url, cfg) + if err != nil { + logDump("Invalid URL: %s", url) + return url + } + if matched { + var u = url + u = strings.TrimPrefix(u, "https://") + u = strings.TrimPrefix(u, "http://") + logDump("Modified URL: %s", "https://"+host+"/"+u) + return "https://" + host + "/" + u + } + return url +} + +// processLinks 处理链接,返回包含处理后数据的 io.Reader +func processLinks(input io.ReadCloser, compress string, host string, cfg *config.Config) (readerOut io.Reader, written int64, err error) { + pipeReader, pipeWriter := io.Pipe() // 创建 io.Pipe + readerOut = pipeReader + + go func() { // 在 Goroutine 中执行写入操作 + defer func() { + if pipeWriter != nil { // 确保 pipeWriter 关闭,即使发生错误 + if err != nil { + if closeErr := pipeWriter.CloseWithError(err); closeErr != nil { // 如果有错误,传递错误给 reader + logError("pipeWriter close with error failed: %v, original error: %v", closeErr, err) + } + } else { + if closeErr := pipeWriter.Close(); closeErr != nil { // 没有错误,正常关闭 + logError("pipeWriter close failed: %v", closeErr) + if err == nil { // 如果之前没有错误,记录关闭错误 + err = closeErr + } + } + } + } + }() + + defer func() { + if err := input.Close(); err != nil { + logError("input close failed: %v", err) + } + + }() + + var bufReader *bufio.Reader + + if compress == "gzip" { + // 解压gzip + gzipReader, gzipErr := gzip.NewReader(input) + if gzipErr != nil { + err = fmt.Errorf("gzip解压错误: %v", gzipErr) + return // Goroutine 中使用 return 返回错误 + } + defer gzipReader.Close() + bufReader = bufio.NewReader(gzipReader) + } else { + bufReader = bufio.NewReader(input) + } + + var bufWriter *bufio.Writer + var gzipWriter *gzip.Writer + + // 根据是否gzip确定 writer 的创建 + if compress == "gzip" { + gzipWriter = gzip.NewWriter(pipeWriter) // 使用 pipeWriter + bufWriter = bufio.NewWriterSize(gzipWriter, 4096) //设置缓冲区大小 + } else { + bufWriter = bufio.NewWriterSize(pipeWriter, 4096) // 使用 pipeWriter + } + + //确保writer关闭 + defer func() { + var closeErr error // 局部变量,用于保存defer中可能发生的错误 + + if gzipWriter != nil { + if closeErr = gzipWriter.Close(); closeErr != nil { + logError("gzipWriter close failed %v", closeErr) + // 如果已经存在错误,则保留。否则,记录此错误。 + if err == nil { + err = closeErr + } + } + } + if flushErr := bufWriter.Flush(); flushErr != nil { + logError("writer flush failed %v", flushErr) + // 如果已经存在错误,则保留。否则,记录此错误。 + if err == nil { + err = flushErr + } + } + }() + + // 使用正则表达式匹配 http 和 https 链接 + for { + line, readErr := bufReader.ReadString('\n') + if readErr != nil { + if readErr == io.EOF { + break // 文件结束 + } + err = fmt.Errorf("读取行错误: %v", readErr) // 传递错误 + return // Goroutine 中使用 return 返回错误 + } + + // 替换所有匹配的 URL + modifiedLine := urlPattern.ReplaceAllStringFunc(line, func(originalURL string) string { + logDump("originalURL: %s", originalURL) + return modifyURL(originalURL, host, cfg) // 假设 modifyURL 函数已定义 + }) + + n, writeErr := bufWriter.WriteString(modifiedLine) + written += int64(n) // 更新写入的字节数 + if writeErr != nil { + err = fmt.Errorf("写入文件错误: %v", writeErr) // 传递错误 + return // Goroutine 中使用 return 返回错误 + } + } + + // 在返回之前,再刷新一次 (虽然 defer 中已经有 flush,但这里再加一次确保及时刷新) + if flushErr := bufWriter.Flush(); flushErr != nil { + if err == nil { // 避免覆盖之前的错误 + err = flushErr + } + return // Goroutine 中使用 return 返回错误 + } + }() + + return readerOut, written, nil // 返回 reader 和 written,error 由 Goroutine 通过 pipeWriter.CloseWithError 传递 +}