[feat][meta] Modelscope download

config/config.yaml

@@ -75,3 +75,9 @@ dynamicProxy:
     timePeriod: 60 #定期检测代理是否可用时间周期，单位秒（S）
     maxContinuousFails: 5 #连续失败次数超过该值，则认为代理不可用
     webhook: https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=73662ac1-1055-48a7-8c89-37964b5f4fdc111 # 企业微信机器人Webhook地址
+
+modelscope:
+    officialBaseURL: https://www.modelscope.cn  # ModelScope官方基础地址
+    chunkSize: 8388608  # 8MB分块，16*1024*1024的数值结果
+    maxRetry: 5  # 超时重试次数
+    retryDelay: 3  # 重试间隔，单位秒（S）（原配置为5*time.Second，YAML中简化为数值+注释）

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/bytedance/sonic v1.13.2
 	github.com/go-playground/validator/v10 v10.27.0
 	github.com/go-redsync/redsync/v4 v4.13.0
+	github.com/gofrs/flock v0.8.1
 	github.com/google/uuid v1.6.0
 	github.com/google/wire v0.6.0
 	github.com/klauspost/compress v1.18.0

go.sum

@@ -46,6 +46,10 @@ github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC
 github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
 github.com/go-redsync/redsync/v4 v4.13.0 h1:49X6GJfnbLGaIpBBREM/zA4uIMDXKAh1NDkvQ1EkZKA=
 github.com/go-redsync/redsync/v4 v4.13.0/go.mod h1:HMW4Q224GZQz6x1Xc7040Yfgacukdzu7ifTDAKiyErQ=
+github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
+github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
+github.com/gofrs/flock v0.12.1 h1:MTLVXXHf8ekldpJk3AKicLij9MdwOWkZ+a/jHHZby9E=
+github.com/gofrs/flock v0.12.1/go.mod h1:9zxTsyu5xtJ9DK+1tFZyibEV7y3uwDxPPfbxeeHCoD0=
 github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
 github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
 github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws=

internal/handler/handler.go

@@ -18,4 +18,4 @@ import (
 	"github.com/google/wire"
 )
 
-var HandlerProvider = wire.NewSet(NewFileHandler, NewMetaHandler, NewSysHandler, NewCacheJobHandler)
+var HandlerProvider = wire.NewSet(NewFileHandler, NewMetaHandler, NewSysHandler, NewCacheJobHandler, NewModelscopeHandler)

internal/handler/modelscope_handler.go

@@ -0,0 +1,119 @@
+package handler
+
+import (
+	"fmt"
+	"strings"
+
+	"dingospeed/internal/service"
+	"dingospeed/pkg/util"
+
+	"github.com/labstack/echo/v4"
+)
+
+// ModelscopeHandler 模型代理请求处理器
+type ModelscopeHandler struct {
+	modelscopeService *service.ModelscopeService
+}
+
+// NewModelscopeHandler 创建模型代理处理器实例
+func NewModelscopeHandler(modelscopeService *service.ModelscopeService) *ModelscopeHandler {
+	return &ModelscopeHandler{
+		modelscopeService: modelscopeService,
+	}
+}
+
+// ModelInfoHandler 处理模型信息查询请求
+func (m *ModelscopeHandler) ModelInfoHandler(c echo.Context) error {
+	parts := strings.Split(strings.Trim(c.Request().URL.Path, "/"), "/")
+
+	if len(parts) < 5 {
+		err := fmt.Errorf("请求路径格式非法")
+		return util.ResponseError(c, err)
+	}
+
+	org, repo, repoType := parts[3], parts[4], parts[2]
+	if err := m.modelscopeService.ForwardModelInfo(c, org, repo, repoType); err != nil {
+		return util.ResponseError(c, err)
+	}
+	return nil
+}
+
+// RevisionsHandler 处理模型版本查询请求
+func (m *ModelscopeHandler) RevisionsHandler(c echo.Context) error {
+	parts := strings.Split(strings.Trim(c.Request().URL.Path, "/"), "/")
+
+	if len(parts) < 5 {
+		err := fmt.Errorf("请求路径格式非法")
+		return util.ResponseError(c, err)
+	}
+
+	org, repo, repoType := parts[3], parts[4], parts[2]
+	if err := m.modelscopeService.ForwardRevisions(c, org, repo, repoType); err != nil {
+		return util.ResponseError(c, err)
+	}
+	return nil
+}
+
+// FileListHandler 处理模型文件列表请求
+func (m *ModelscopeHandler) FileListHandler(c echo.Context) error {
+	parts := strings.Split(strings.Trim(c.Request().URL.Path, "/"), "/")
+
+	if len(parts) < 5 {
+		err := fmt.Errorf("请求路径格式非法")
+		return util.ResponseError(c, err)
+	}
+
+	org, repo, repoType := parts[3], parts[4], parts[2]
+	if err := m.modelscopeService.ForwardFileList(c, org, repo, repoType); err != nil {
+		return util.ResponseError(c, err)
+	}
+	return nil
+}
+
+// FileDownloadHandler 处理模型文件下载请求（支持续传）
+func (m *ModelscopeHandler) FileDownloadHandler(c echo.Context) error {
+	parts := strings.Split(strings.Trim(c.Request().URL.Path, "/"), "/")
+
+	if len(parts) < 5 {
+		err := fmt.Errorf("请求路径格式非法")
+		return util.ResponseError(c, err)
+	}
+
+	org, repo, repoType := parts[3], parts[4], parts[2]
+	if err := m.modelscopeService.HandleFileDownload(c, org, repo, repoType); err != nil {
+		return util.ResponseError(c, err)
+	}
+	return nil
+}
+
+// FileTreeHandler 处理数据集文件列表请求
+func (m *ModelscopeHandler) FileTreeHandler(c echo.Context) error {
+	parts := strings.Split(strings.Trim(c.Request().URL.Path, "/"), "/")
+
+	if len(parts) < 5 {
+		err := fmt.Errorf("请求路径格式非法")
+		return util.ResponseError(c, err)
+	}
+
+	org, repo, repoType := parts[3], parts[4], parts[2]
+	if err := m.modelscopeService.ForwardRepoTree(c, org, repo, repoType); err != nil {
+		return util.ResponseError(c, err)
+	}
+	return nil
+}
+
+// DatasetFileTreeHandler 处理数据集文件列表请求
+func (m *ModelscopeHandler) DatasetFileTreeHandler(c echo.Context) error {
+	parts := strings.Split(strings.Trim(c.Request().URL.Path, "/"), "/")
+
+	if len(parts) < 4 {
+		err := fmt.Errorf("请求路径格式非法")
+		return util.ResponseError(c, err)
+	}
+
+	datasetId := parts[3]
+	if err := m.modelscopeService.ForwardRepoTreeByDatasetId(c, datasetId); err != nil {
+		return util.ResponseError(c, err)
+	}
+	return nil
+}

internal/router/http_router.go

@@ -23,21 +23,23 @@ import (
 )
 
 type HttpRouter struct {
-	echo            *echo.Echo
-	fileHandler     *handler.FileHandler
-	metaHandler     *handler.MetaHandler
-	sysHandler      *handler.SysHandler
-	cacheJobHandler *handler.CacheJobHandler
+	echo              *echo.Echo
+	fileHandler       *handler.FileHandler
+	metaHandler       *handler.MetaHandler
+	sysHandler        *handler.SysHandler
+	cacheJobHandler   *handler.CacheJobHandler
+	modelscopeHandler *handler.ModelscopeHandler
 }
 
 func NewHttpRouter(echo *echo.Echo, fileHandler *handler.FileHandler, metaHandler *handler.MetaHandler,
-	sysHandler *handler.SysHandler, cacheJobHandler *handler.CacheJobHandler) *HttpRouter {
+	sysHandler *handler.SysHandler, cacheJobHandler *handler.CacheJobHandler, modelscopeHandler *handler.ModelscopeHandler) *HttpRouter {
 	r := &HttpRouter{
-		echo:            echo,
-		fileHandler:     fileHandler,
-		metaHandler:     metaHandler,
-		sysHandler:      sysHandler,
-		cacheJobHandler: cacheJobHandler,
+		echo:              echo,
+		fileHandler:       fileHandler,
+		metaHandler:       metaHandler,
+		sysHandler:        sysHandler,
+		cacheJobHandler:   cacheJobHandler,
+		modelscopeHandler: modelscopeHandler,
 	}
 	r.initRouter()
 	return r
@@ -54,6 +56,7 @@ func (r *HttpRouter) initRouter() {
 	r.routerForCacheJob()
 
 	r.routerForSpeed()
+	r.routerForModelscope()
 }
 
 func (r *HttpRouter) routerForSpeed() { // alayanew
@@ -91,3 +94,12 @@ func (r *HttpRouter) routerForCacheJob() { // alayanew
 	r.echo.POST("/api/cacheJob/resume", r.cacheJobHandler.ResumeCacheJobHandler)
 	r.echo.POST("/api/cacheJob/realtime", r.cacheJobHandler.RealtimeCacheJobHandler)
 }
+
+func (r *HttpRouter) routerForModelscope() { // modelscope
+	r.echo.GET("/api/v1/:repoType/:org/:repo", r.modelscopeHandler.ModelInfoHandler)
+	r.echo.GET("/api/v1/:repoType/:org/:repo/revisions", r.modelscopeHandler.RevisionsHandler)
+	r.echo.GET("/api/v1/:repoType/:org/:repo/repo/files", r.modelscopeHandler.FileListHandler)
+	r.echo.GET("/api/v1/:repoType/:org/:repo/repo", r.modelscopeHandler.FileDownloadHandler)
+	r.echo.GET("/api/v1/:repoType/:org/:repo/repo/tree", r.modelscopeHandler.FileTreeHandler)
+	r.echo.GET("/api/v1/datasets/:datasetId/repo/tree", r.modelscopeHandler.DatasetFileTreeHandler)
+}

internal/server/http.go

@@ -80,6 +80,7 @@ func NewEngine() *echo.Echo {
 	r := echo.New()
 	middleware.InitMiddlewareConfig()
 	r.Use(middleware.QueueLimitMiddleware)
+	r.Use(middleware.CORSMiddleware())
 
 	t := &Template{
 		templates: template.Must(template.ParseFS(templatesFS, "templates/*.html")),