chore: 初始化 Cloudreve 源码仓库

pkg/request/options.go

@@ -0,0 +1,176 @@
+package request
+
+import (
+	"context"
+	"github.com/cloudreve/Cloudreve/v4/pkg/auth"
+	"github.com/cloudreve/Cloudreve/v4/pkg/logging"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+)
+
+// Option 发送请求的额外设置
+type Option interface {
+	apply(*options)
+}
+
+type options struct {
+	timeout           time.Duration
+	header            http.Header
+	sign              auth.Auth
+	signTTL           int64
+	ctx               context.Context
+	contentLength     int64
+	masterMeta        bool
+	siteID            string
+	siteURL           string
+	endpoint          *url.URL
+	slaveNodeID       int
+	tpsLimiterToken   string
+	tps               float64
+	tpsBurst          int
+	logger            logging.Logger
+	withCorrelationID bool
+	cookieJar         http.CookieJar
+	transport         *http.Transport
+}
+
+type optionFunc func(*options)
+
+func (f optionFunc) apply(o *options) {
+	f(o)
+}
+
+func newDefaultOption() *options {
+	return &options{
+		header:        http.Header{},
+		timeout:       0,
+		contentLength: -1,
+		ctx:           context.Background(),
+	}
+}
+
+func (o *options) clone() options {
+	newOptions := *o
+	newOptions.header = o.header.Clone()
+	return newOptions
+}
+
+// WithTransport 设置请求Transport
+func WithTransport(transport *http.Transport) Option {
+	return optionFunc(func(o *options) {
+		o.transport = transport
+	})
+}
+
+// WithTimeout 设置请求超时
+func WithTimeout(t time.Duration) Option {
+	return optionFunc(func(o *options) {
+		o.timeout = t
+	})
+}
+
+// WithContext 设置请求上下文
+func WithContext(c context.Context) Option {
+	return optionFunc(func(o *options) {
+		o.ctx = c
+	})
+}
+
+// WithCredential 对请求进行签名
+func WithCredential(instance auth.Auth, ttl int64) Option {
+	return optionFunc(func(o *options) {
+		o.sign = instance
+		if ttl > 0 {
+			o.signTTL = ttl
+		}
+	})
+}
+
+// WithHeader 设置请求Header
+func WithHeader(header http.Header) Option {
+	return optionFunc(func(o *options) {
+		for k, v := range header {
+			o.header[k] = v
+		}
+	})
+}
+
+// WithoutHeader 设置清除请求Header
+func WithoutHeader(header []string) Option {
+	return optionFunc(func(o *options) {
+		for _, v := range header {
+			delete(o.header, v)
+		}
+
+	})
+}
+
+// WithContentLength 设置请求大小
+func WithContentLength(s int64) Option {
+	return optionFunc(func(o *options) {
+		o.contentLength = s
+	})
+}
+
+// WithMasterMeta 请求时携带主机信息
+func WithMasterMeta(siteID string, siteURL string) Option {
+	return optionFunc(func(o *options) {
+		o.masterMeta = true
+		o.siteID = siteID
+		o.siteURL = siteURL
+	})
+}
+
+// WithSlaveMeta set slave node ID in master's request header
+func WithSlaveMeta(s int) Option {
+	return optionFunc(func(o *options) {
+		o.slaveNodeID = s
+	})
+}
+
+// Endpoint 使用同一的请求Endpoint
+func WithEndpoint(endpoint string) Option {
+	if !strings.HasSuffix(endpoint, "/") {
+		endpoint += "/"
+	}
+
+	endpointURL, _ := url.Parse(endpoint)
+	return optionFunc(func(o *options) {
+		o.endpoint = endpointURL
+	})
+}
+
+// WithTPSLimit 请求时使用全局流量限制
+func WithTPSLimit(token string, tps float64, burst int) Option {
+	return optionFunc(func(o *options) {
+		o.tpsLimiterToken = token
+		o.tps = tps
+		if burst < 1 {
+			burst = 1
+		}
+		o.tpsBurst = burst
+	})
+}
+
+// WithLogger set logger for logging requests
+func WithLogger(logger logging.Logger) Option {
+	return optionFunc(func(o *options) {
+		o.logger = logger
+	})
+}
+
+// WithCorrelationID set correlation ID for logging requests
+func WithCorrelationID() Option {
+	return optionFunc(func(o *options) {
+		o.withCorrelationID = true
+	})
+}
+
+// WithCookieJar set cookie jar for request
+func WithCookieJar(jar http.CookieJar) Option {
+	return optionFunc(func(o *options) {
+		o.cookieJar = jar
+	})
+}

pkg/request/request.go

@@ -0,0 +1,352 @@
+package request
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/cloudreve/Cloudreve/v4/application/constants"
+	"github.com/cloudreve/Cloudreve/v4/pkg/auth"
+	"github.com/cloudreve/Cloudreve/v4/pkg/conf"
+	"github.com/cloudreve/Cloudreve/v4/pkg/logging"
+	"github.com/cloudreve/Cloudreve/v4/pkg/serializer"
+	"github.com/samber/lo"
+)
+
+// GeneralClient 通用 HTTP Client
+var GeneralClient Client = NewClientDeprecated()
+
+const (
+	CorrelationHeader = constants.CrHeaderPrefix + "Correlation-Id"
+	SiteURLHeader     = constants.CrHeaderPrefix + "Site-Url"
+	SiteVersionHeader = constants.CrHeaderPrefix + "Version"
+	SiteIDHeader      = constants.CrHeaderPrefix + "Site-Id"
+	SlaveNodeIDHeader = constants.CrHeaderPrefix + "Node-Id"
+	ClientIDHeader    = constants.CrHeaderPrefix + "Client-Id"
+	LocalIP           = "localhost"
+)
+
+// Response 请求的响应或错误信息
+type Response struct {
+	Err      error
+	Response *http.Response
+	l        logging.Logger
+}
+
+// Client 请求客户端
+type Client interface {
+	// Apply applies the given options to this client.
+	Apply(opts ...Option)
+	// Request send a HTTP request
+	Request(method, target string, body io.Reader, opts ...Option) *Response
+}
+
+// HTTPClient 实现 Client 接口
+type HTTPClient struct {
+	mu         sync.Mutex
+	options    *options
+	tpsLimiter TPSLimiter
+	l          logging.Logger
+	config     conf.ConfigProvider
+}
+
+func NewClient(config conf.ConfigProvider, opts ...Option) Client {
+	client := &HTTPClient{
+		options:    newDefaultOption(),
+		tpsLimiter: globalTPSLimiter,
+		config:     config,
+	}
+
+	for _, o := range opts {
+		o.apply(client.options)
+	}
+
+	return client
+}
+
+// Deprecated
+func NewClientDeprecated(opts ...Option) Client {
+	client := &HTTPClient{
+		options:    newDefaultOption(),
+		tpsLimiter: globalTPSLimiter,
+	}
+
+	for _, o := range opts {
+		o.apply(client.options)
+	}
+
+	return client
+}
+
+func (c *HTTPClient) Apply(opts ...Option) {
+	for _, o := range opts {
+		o.apply(c.options)
+	}
+}
+
+// Request 发送HTTP请求
+func (c *HTTPClient) Request(method, target string, body io.Reader, opts ...Option) *Response {
+	// 应用额外设置
+	c.mu.Lock()
+	options := c.options.clone()
+	c.mu.Unlock()
+	for _, o := range opts {
+		o.apply(&options)
+	}
+
+	// 创建请求客户端
+	client := &http.Client{
+		Timeout: options.timeout,
+		Jar:     options.cookieJar,
+	}
+
+	if options.transport != nil {
+		client.Transport = options.transport
+	}
+
+	// size为0时将body设为nil
+	if options.contentLength == 0 {
+		body = nil
+	}
+
+	// 确定请求URL
+	if options.endpoint != nil {
+		targetPath, err := url.Parse(target)
+		if err != nil {
+			return &Response{Err: err}
+		}
+
+		targetURL := *options.endpoint
+		target = targetURL.ResolveReference(targetPath).String()
+	}
+
+	// 创建请求
+	var (
+		req *http.Request
+		err error
+	)
+	start := time.Now()
+	if options.ctx != nil {
+		req, err = http.NewRequestWithContext(options.ctx, method, target, body)
+	} else {
+		req, err = http.NewRequest(method, target, body)
+	}
+	if err != nil {
+		return &Response{Err: err}
+	}
+
+	// 添加请求相关设置
+	if options.header != nil {
+		for k, v := range options.header {
+			req.Header.Add(k, strings.Join(v, " "))
+		}
+	}
+
+	req.Header.Set("User-Agent", "Cloudreve/"+constants.BackendVersion)
+
+	if options.ctx != nil && options.withCorrelationID {
+		req.Header.Add(CorrelationHeader, logging.CorrelationID(options.ctx).String())
+	}
+
+	mode := c.config.System().Mode
+	if options.masterMeta && mode == conf.MasterMode {
+		req.Header.Add(SiteURLHeader, options.siteURL)
+		req.Header.Add(SiteIDHeader, options.siteID)
+		req.Header.Add(SiteVersionHeader, constants.BackendVersion)
+	}
+
+	if options.slaveNodeID > 0 {
+		req.Header.Add(SlaveNodeIDHeader, strconv.Itoa(options.slaveNodeID))
+	}
+
+	if options.contentLength != -1 {
+		req.ContentLength = options.contentLength
+	}
+
+	// 签名请求
+	if options.sign != nil {
+		ctx := options.ctx
+		if options.ctx == nil {
+			ctx = context.Background()
+		}
+		expire := time.Now().Add(time.Second * time.Duration(options.signTTL))
+		switch method {
+		case "PUT", "POST", "PATCH":
+			auth.SignRequest(ctx, options.sign, req, &expire)
+		default:
+			if resURL, err := auth.SignURI(ctx, options.sign, req.URL.String(), &expire); err == nil {
+				req.URL = resURL
+			}
+		}
+	}
+
+	if options.tps > 0 {
+		c.tpsLimiter.Limit(options.ctx, options.tpsLimiterToken, options.tps, options.tpsBurst)
+	}
+
+	// 发送请求
+	resp, err := client.Do(req)
+
+	// Logging request
+	if options.logger != nil {
+		statusCode := 0
+		errStr := ""
+		if resp != nil {
+			statusCode = resp.StatusCode
+		}
+
+		if err != nil {
+			errStr = err.Error()
+		}
+
+		logging.Request(options.logger, false, statusCode, req.Method, LocalIP, req.URL.String(), errStr, start)
+	}
+
+	// Apply cookies
+	if resp != nil && resp.Cookies() != nil && options.cookieJar != nil {
+		options.cookieJar.SetCookies(req.URL, resp.Cookies())
+	}
+
+	if err != nil {
+		return &Response{Err: err}
+	}
+
+	return &Response{Err: nil, Response: resp, l: options.logger}
+}
+
+// GetResponse 检查响应并获取响应正文
+func (resp *Response) GetResponse() (string, error) {
+	if resp.Err != nil {
+		return "", resp.Err
+	}
+	respBody, err := io.ReadAll(resp.Response.Body)
+	_ = resp.Response.Body.Close()
+
+	return string(respBody), err
+}
+
+// GetResponseIgnoreErr 获取响应正文
+func (resp *Response) GetResponseIgnoreErr() (string, error) {
+	if resp.Response == nil {
+		return "", resp.Err
+	}
+
+	respBody, _ := io.ReadAll(resp.Response.Body)
+	_ = resp.Response.Body.Close()
+
+	return string(respBody), resp.Err
+}
+
+// CheckHTTPResponse 检查请求响应HTTP状态码
+func (resp *Response) CheckHTTPResponse(status ...int) *Response {
+	if resp.Err != nil {
+		return resp
+	}
+
+	// 检查HTTP状态码
+	if !lo.Contains(status, resp.Response.StatusCode) {
+		resp.Err = fmt.Errorf("Remote returns unexpected status code: %d", resp.Response.StatusCode)
+	}
+	return resp
+}
+
+// DecodeResponse 尝试解析为serializer.Response，并对状态码进行检查
+func (resp *Response) DecodeResponse() (*serializer.Response, error) {
+	if resp.Err != nil {
+		return nil, resp.Err
+	}
+
+	respString, err := resp.GetResponse()
+	if err != nil {
+		return nil, err
+	}
+
+	var res serializer.Response
+	err = json.Unmarshal([]byte(respString), &res)
+	if err != nil {
+		if resp.l != nil {
+			resp.l.Debug("Failed to parse response: %s", respString)
+		}
+
+		return nil, err
+	}
+	return &res, nil
+}
+
+// NopRSCloser 实现不完整seeker
+type NopRSCloser struct {
+	body   io.ReadCloser
+	status *rscStatus
+}
+
+type rscStatus struct {
+	// http.ServeContent 会读取一小块以决定内容类型，
+	// 但是响应body无法实现seek，所以此项为真时第一个read会返回假数据
+	IgnoreFirst bool
+
+	Size int64
+}
+
+// GetRSCloser 返回带有空seeker的RSCloser，供http.ServeContent使用
+func (resp *Response) GetRSCloser() (*NopRSCloser, error) {
+	if resp.Err != nil {
+		return nil, resp.Err
+	}
+
+	return &NopRSCloser{
+		body: resp.Response.Body,
+		status: &rscStatus{
+			Size: resp.Response.ContentLength,
+		},
+	}, resp.Err
+}
+
+// SetFirstFakeChunk 开启第一次read返回空数据
+// TODO 测试
+func (instance NopRSCloser) SetFirstFakeChunk() {
+	instance.status.IgnoreFirst = true
+}
+
+// SetContentLength 设置数据流大小
+func (instance NopRSCloser) SetContentLength(size int64) {
+	instance.status.Size = size
+}
+
+// Read 实现 NopRSCloser reader
+func (instance NopRSCloser) Read(p []byte) (n int, err error) {
+	if instance.status.IgnoreFirst && len(p) == 512 {
+		return 0, io.EOF
+	}
+	return instance.body.Read(p)
+}
+
+// Close 实现 NopRSCloser closer
+func (instance NopRSCloser) Close() error {
+	return instance.body.Close()
+}
+
+// Seek 实现 NopRSCloser seeker, 只实现seek开头/结尾以便http.ServeContent用于确定正文大小
+func (instance NopRSCloser) Seek(offset int64, whence int) (int64, error) {
+	// 进行第一次Seek操作后，取消忽略选项
+	if instance.status.IgnoreFirst {
+		instance.status.IgnoreFirst = false
+	}
+	if offset == 0 {
+		switch whence {
+		case io.SeekStart:
+			return 0, nil
+		case io.SeekEnd:
+			return instance.status.Size, nil
+		}
+	}
+	return 0, errors.New("not implemented")
+
+}

pkg/request/request_test.go

@@ -0,0 +1,275 @@
+package request
+
+import (
+	"context"
+	"errors"
+	"github.com/cloudreve/Cloudreve/v4/pkg/auth"
+	"github.com/cloudreve/Cloudreve/v4/pkg/cache"
+	"github.com/stretchr/testify/assert"
+	testMock "github.com/stretchr/testify/mock"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+)
+
+type ClientMock struct {
+	testMock.Mock
+}
+
+func (m ClientMock) Request(method, target string, body io.Reader, opts ...Option) *Response {
+	args := m.Called(method, target, body, opts)
+	return args.Get(0).(*Response)
+}
+
+func TestWithTimeout(t *testing.T) {
+	asserts := assert.New(t)
+	options := newDefaultOption()
+	WithTimeout(time.Duration(5) * time.Second).apply(options)
+	asserts.Equal(time.Duration(5)*time.Second, options.timeout)
+}
+
+func TestWithHeader(t *testing.T) {
+	asserts := assert.New(t)
+	options := newDefaultOption()
+	WithHeader(map[string][]string{"Origin": []string{"123"}}).apply(options)
+	asserts.Equal(http.Header{"Origin": []string{"123"}}, options.header)
+}
+
+func TestWithContentLength(t *testing.T) {
+	asserts := assert.New(t)
+	options := newDefaultOption()
+	WithContentLength(10).apply(options)
+	asserts.EqualValues(10, options.contentLength)
+}
+
+func TestWithContext(t *testing.T) {
+	asserts := assert.New(t)
+	options := newDefaultOption()
+	WithContext(context.Background()).apply(options)
+	asserts.NotNil(options.ctx)
+}
+
+func TestHTTPClient_Request(t *testing.T) {
+	asserts := assert.New(t)
+	client := NewClientDeprecated(WithSlaveMeta("test"))
+
+	// 正常
+	{
+		resp := client.Request(
+			"POST",
+			"/test",
+			strings.NewReader(""),
+			WithTimeout(time.Duration(1)*time.Microsecond),
+			WithCredential(auth.HMACAuth{SecretKey: []byte("123")}, 10),
+			WithoutHeader([]string{"origin", "origin"}),
+		)
+		asserts.Error(resp.Err)
+		asserts.Nil(resp.Response)
+	}
+
+	// 正常 带有ctx
+	{
+		resp := client.Request(
+			"GET",
+			"http://cloudreveisnotexist.com",
+			strings.NewReader(""),
+			WithContentLength(0),
+			WithEndpoint("http://cloudreveisnotexist.com"),
+			WithTimeout(time.Duration(1)*time.Microsecond),
+			WithCredential(auth.HMACAuth{SecretKey: []byte("123")}, 10),
+			WithContext(context.Background()),
+		)
+		asserts.Error(resp.Err)
+		asserts.Nil(resp.Response)
+	}
+
+}
+
+func TestResponse_GetResponse(t *testing.T) {
+	asserts := assert.New(t)
+
+	// 直接返回错误
+	{
+		resp := Response{
+			Err: errors.New("error"),
+		}
+		content, err := resp.GetResponse()
+		asserts.Empty(content)
+		asserts.Error(err)
+	}
+
+	// 正常
+	{
+		resp := Response{
+			Response: &http.Response{Body: ioutil.NopCloser(strings.NewReader("123"))},
+		}
+		content, err := resp.GetResponse()
+		asserts.Equal("123", content)
+		asserts.NoError(err)
+	}
+}
+
+func TestResponse_CheckHTTPResponse(t *testing.T) {
+	asserts := assert.New(t)
+
+	// 直接返回错误
+	{
+		resp := Response{
+			Err: errors.New("error"),
+		}
+		res := resp.CheckHTTPResponse(200)
+		asserts.Error(res.Err)
+	}
+
+	// 404错误
+	{
+		resp := Response{
+			Response: &http.Response{StatusCode: 404},
+		}
+		res := resp.CheckHTTPResponse(200)
+		asserts.Error(res.Err)
+	}
+
+	// 通过
+	{
+		resp := Response{
+			Response: &http.Response{StatusCode: 200},
+		}
+		res := resp.CheckHTTPResponse(200)
+		asserts.NoError(res.Err)
+	}
+}
+
+func TestResponse_GetRSCloser(t *testing.T) {
+	asserts := assert.New(t)
+
+	// 直接返回错误
+	{
+		resp := Response{
+			Err: errors.New("error"),
+		}
+		res, err := resp.GetRSCloser()
+		asserts.Error(err)
+		asserts.Nil(res)
+	}
+
+	// 正常
+	{
+		resp := Response{
+			Response: &http.Response{ContentLength: 3, Body: ioutil.NopCloser(strings.NewReader("123"))},
+		}
+		res, err := resp.GetRSCloser()
+		asserts.NoError(err)
+		content, err := ioutil.ReadAll(res)
+		asserts.NoError(err)
+		asserts.Equal("123", string(content))
+		offset, err := res.Seek(0, 0)
+		asserts.NoError(err)
+		asserts.Equal(int64(0), offset)
+		offset, err = res.Seek(0, 2)
+		asserts.NoError(err)
+		asserts.Equal(int64(3), offset)
+		_, err = res.Seek(1, 2)
+		asserts.Error(err)
+		asserts.NoError(res.Close())
+	}
+
+}
+
+func TestResponse_DecodeResponse(t *testing.T) {
+	asserts := assert.New(t)
+
+	// 直接返回错误
+	{
+		resp := Response{Err: errors.New("error")}
+		response, err := resp.DecodeResponse()
+		asserts.Error(err)
+		asserts.Nil(response)
+	}
+
+	// 无法解析响应
+	{
+		resp := Response{
+			Response: &http.Response{
+				Body: ioutil.NopCloser(strings.NewReader("test")),
+			},
+		}
+		response, err := resp.DecodeResponse()
+		asserts.Error(err)
+		asserts.Nil(response)
+	}
+
+	// 成功
+	{
+		resp := Response{
+			Response: &http.Response{
+				Body: ioutil.NopCloser(strings.NewReader("{\"code\":0}")),
+			},
+		}
+		response, err := resp.DecodeResponse()
+		asserts.NoError(err)
+		asserts.NotNil(response)
+		asserts.Equal(0, response.Code)
+	}
+}
+
+func TestNopRSCloser_SetFirstFakeChunk(t *testing.T) {
+	asserts := assert.New(t)
+	rsc := NopRSCloser{
+		status: &rscStatus{},
+	}
+	rsc.SetFirstFakeChunk()
+	asserts.True(rsc.status.IgnoreFirst)
+
+	rsc.SetContentLength(20)
+	asserts.EqualValues(20, rsc.status.Size)
+}
+
+func TestBlackHole(t *testing.T) {
+	a := assert.New(t)
+	cache.Set("setting_reset_after_upload_failed", "true", 0)
+	a.NotPanics(func() {
+		BlackHole(strings.NewReader("TestBlackHole"))
+	})
+}
+
+func TestHTTPClient_TPSLimit(t *testing.T) {
+	a := assert.New(t)
+	client := NewClientDeprecated()
+
+	finished := make(chan struct{})
+	go func() {
+		client.Request(
+			"POST",
+			"/test",
+			strings.NewReader(""),
+			WithTPSLimit("TestHTTPClient_TPSLimit", 1, 1),
+		)
+		close(finished)
+	}()
+	select {
+	case <-finished:
+	case <-time.After(10 * time.Second):
+		a.Fail("Request should be finished instantly.")
+	}
+
+	finished = make(chan struct{})
+	go func() {
+		client.Request(
+			"POST",
+			"/test",
+			strings.NewReader(""),
+			WithTPSLimit("TestHTTPClient_TPSLimit", 1, 1),
+		)
+		close(finished)
+	}()
+	select {
+	case <-finished:
+	case <-time.After(2 * time.Second):
+		a.Fail("Request should be finished in 1 second.")
+	}
+
+}

pkg/request/ssrf.go

@@ -0,0 +1,290 @@
+package request
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"net"
+	"net/url"
+	"strings"
+)
+
+// ErrUnsafeURL is returned when a user-supplied URL targets an address that
+// should not be reachable by server-side fetchers (loopback, private,
+// link-local, cloud metadata, multicast, etc.) or uses a non-allowed scheme.
+var ErrUnsafeURL = errors.New("URL is not allowed")
+
+var downloadAllowedSchemes = map[string]struct{}{
+	"http":   {},
+	"https":  {},
+	"ftp":    {},
+	"ftps":   {},
+	"sftp":   {},
+	"magnet": {},
+}
+
+// trackerAllowedSchemes are schemes valid inside magnet tr=/ws=/xs=/as= params
+// that we recognize and validate. Anything else is treated as non-network
+// (e.g. dht://) and skipped.
+var trackerAllowedSchemes = map[string]struct{}{
+	"http":  {},
+	"https": {},
+	"udp":   {},
+	"ws":    {},
+	"wss":   {},
+}
+
+var bannedHostnames = map[string]struct{}{
+	"localhost":             {},
+	"localhost.localdomain": {},
+	"ip6-localhost":         {},
+	"ip6-loopback":          {},
+}
+
+// 100.64.0.0/10 (RFC 6598 CGNAT) is not flagged by net.IP.IsPrivate.
+var cgnatNet = mustParseCIDR("100.64.0.0/10")
+
+// 169.254.169.254 (cloud instance metadata) is link-local and already covered
+// by IsLinkLocalUnicast, but we keep an explicit check for clarity.
+var cloudMetadataIP = net.ParseIP("169.254.169.254")
+
+func mustParseCIDR(s string) *net.IPNet {
+	_, n, err := net.ParseCIDR(s)
+	if err != nil {
+		panic(err)
+	}
+	return n
+}
+
+// SSRFOptions configures ValidateExternalURL.
+type SSRFOptions struct {
+	// Disabled short-circuits the entire check. Use when the operator has
+	// explicitly opted out (e.g. node-level URLValidation.Disabled).
+	Disabled bool
+	// AllowedHosts contains hostnames whose URLs bypass all subsequent checks
+	// (used to whitelist the operator-configured site URL host(s) and any
+	// operator-trusted internal hostnames). Compared case-insensitively to
+	// the URL's Hostname(). Port is ignored.
+	AllowedHosts []string
+	// AllowedCIDRs contains CIDR blocks whose IPs bypass the IP-class checks.
+	// Resolved IPs falling within any of these are treated as safe even if
+	// they would otherwise be rejected (private, link-local, ...). Malformed
+	// entries are ignored. Used to opt a LAN range back in.
+	AllowedCIDRs []string
+	// Resolver is used for DNS lookups; nil uses net.DefaultResolver.
+	Resolver *net.Resolver
+}
+
+// ValidateExternalURL returns an error wrapping ErrUnsafeURL if the URL would
+// reach an internal-only address when fetched by a server-side downloader.
+// Magnet links have their tr=/ws=/xs=/as= parameters validated as URLs.
+func ValidateExternalURL(ctx context.Context, raw string, opt SSRFOptions) error {
+	if opt.Disabled {
+		return nil
+	}
+
+	raw = strings.TrimSpace(raw)
+	if raw == "" {
+		return fmt.Errorf("empty URL: %w", ErrUnsafeURL)
+	}
+
+	u, err := url.Parse(raw)
+	if err != nil {
+		return fmt.Errorf("parse URL: %w", ErrUnsafeURL)
+	}
+
+	scheme := strings.ToLower(u.Scheme)
+	if _, ok := downloadAllowedSchemes[scheme]; !ok {
+		return fmt.Errorf("scheme %q not allowed: %w", u.Scheme, ErrUnsafeURL)
+	}
+
+	if scheme == "magnet" {
+		return validateMagnet(ctx, u, opt)
+	}
+
+	return validateHost(ctx, u.Hostname(), opt)
+}
+
+func validateMagnet(ctx context.Context, u *url.URL, opt SSRFOptions) error {
+	q := u.Query()
+	for _, key := range []string{"tr", "ws", "xs", "as"} {
+		for _, v := range q[key] {
+			sub, err := url.Parse(strings.TrimSpace(v))
+			if err != nil {
+				return fmt.Errorf("magnet %s=%q: %w", key, v, ErrUnsafeURL)
+			}
+			if sub.Hostname() == "" {
+				continue
+			}
+			if _, ok := trackerAllowedSchemes[strings.ToLower(sub.Scheme)]; !ok {
+				continue
+			}
+			if err := validateHost(ctx, sub.Hostname(), opt); err != nil {
+				return fmt.Errorf("magnet %s=%q: %w", key, v, err)
+			}
+		}
+	}
+	return nil
+}
+
+func validateHost(ctx context.Context, host string, opt SSRFOptions) error {
+	host = strings.TrimSpace(host)
+	if host == "" {
+		return fmt.Errorf("empty host: %w", ErrUnsafeURL)
+	}
+
+	for _, allowed := range opt.AllowedHosts {
+		if strings.EqualFold(strings.TrimSpace(allowed), host) {
+			return nil
+		}
+	}
+
+	lowered := strings.ToLower(host)
+	if _, banned := bannedHostnames[lowered]; banned {
+		return fmt.Errorf("hostname %q is local: %w", host, ErrUnsafeURL)
+	}
+	if strings.HasSuffix(lowered, ".localhost") {
+		return fmt.Errorf("hostname %q is local: %w", host, ErrUnsafeURL)
+	}
+
+	allowed := parseCIDRs(opt.AllowedCIDRs)
+
+	if ip := net.ParseIP(host); ip != nil {
+		return checkIPWithAllowlist(ip, allowed)
+	}
+
+	resolver := opt.Resolver
+	if resolver == nil {
+		resolver = net.DefaultResolver
+	}
+	addrs, err := resolver.LookupIPAddr(ctx, host)
+	if err != nil {
+		return fmt.Errorf("resolve %q: %w", host, ErrUnsafeURL)
+	}
+	if len(addrs) == 0 {
+		return fmt.Errorf("no addresses for %q: %w", host, ErrUnsafeURL)
+	}
+	for _, a := range addrs {
+		if err := checkIPWithAllowlist(a.IP, allowed); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// parseCIDRs parses CIDR strings, silently dropping malformed entries — the
+// admin gets validation feedback at config-save time, so noisy errors during a
+// download are not useful.
+func parseCIDRs(raw []string) []*net.IPNet {
+	out := make([]*net.IPNet, 0, len(raw))
+	for _, r := range raw {
+		_, n, err := net.ParseCIDR(strings.TrimSpace(r))
+		if err == nil {
+			out = append(out, n)
+		}
+	}
+	return out
+}
+
+func checkIPWithAllowlist(ip net.IP, allowed []*net.IPNet) error {
+	// Normalize IPv4-in-IPv6 transition forms so both the allowlist and the
+	// class check operate on the address the packet actually reaches.
+	ip = effectiveIP(ip)
+	for _, n := range allowed {
+		if n.Contains(ip) {
+			return nil
+		}
+	}
+	return checkIP(ip)
+}
+
+// nat64WellKnownPrefix is 64:ff9b::/96 (RFC 6052 §2.1), the 12-byte prefix
+// that wraps an IPv4 address in its last 32 bits for NAT64 gateways.
+var nat64WellKnownPrefix = []byte{
+	0x00, 0x64, 0xff, 0x9b,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+}
+
+// teredoPrefix is 2001:0000::/32 (RFC 4380). In Teredo, the client's IPv4 is
+// the last 32 bits of the address XORed with 0xff.
+var teredoPrefix = []byte{0x20, 0x01, 0x00, 0x00}
+
+// effectiveIP unwraps IPv4-in-IPv6 transition forms to the IPv4 address the
+// packet ultimately reaches, so checkIP classifies the real target rather
+// than the (often globally-routable) IPv6 wrapper. Go's net.IP builtins
+// (IsLoopback, IsPrivate, IsLinkLocalUnicast, ...) inspect only the outer
+// address and would otherwise accept, for example, 64:ff9b::a9fe:a9fe as a
+// safe public IPv6 even though it routes to 169.254.169.254 on a NAT64 host.
+//
+// Covers:
+//   - IPv4-mapped     ::ffff:a.b.c.d          (RFC 4291 §2.5.5.2)
+//   - NAT64 WKP       64:ff9b::a.b.c.d        (RFC 6052 §2.1)
+//   - 6to4            2002:AABB:CCDD::        (RFC 3056)
+//   - Teredo          2001:0000:...           (RFC 4380)
+//   - IPv4-compatible ::a.b.c.d               (RFC 4291 §2.5.5.1, deprecated)
+func effectiveIP(ip net.IP) net.IP {
+	if ip == nil {
+		return nil
+	}
+	// IPv4 literal or IPv4-mapped ::ffff:a.b.c.d — To4 returns the embedded v4.
+	if v4 := ip.To4(); v4 != nil {
+		return v4
+	}
+	v6 := ip.To16()
+	if v6 == nil {
+		return ip
+	}
+
+	// NAT64 well-known prefix 64:ff9b::/96.
+	if bytes.HasPrefix(v6, nat64WellKnownPrefix) {
+		return net.IPv4(v6[12], v6[13], v6[14], v6[15]).To4()
+	}
+
+	// 6to4 2002::/16 — bytes 2-5 encode the embedded IPv4.
+	if v6[0] == 0x20 && v6[1] == 0x02 {
+		return net.IPv4(v6[2], v6[3], v6[4], v6[5]).To4()
+	}
+
+	// Teredo 2001:0000::/32 — client IPv4 is last 4 bytes XOR 0xff.
+	if bytes.HasPrefix(v6, teredoPrefix) {
+		return net.IPv4(v6[12]^0xff, v6[13]^0xff, v6[14]^0xff, v6[15]^0xff).To4()
+	}
+
+	// IPv4-compatible ::a.b.c.d — top 12 bytes are zero. Skip :: and ::1 so
+	// the outer classifier still catches them as unspecified/loopback IPv6.
+	var zero [12]byte
+	if bytes.Equal(v6[:12], zero[:]) {
+		if last := binary.BigEndian.Uint32(v6[12:16]); last > 1 {
+			return net.IPv4(v6[12], v6[13], v6[14], v6[15]).To4()
+		}
+	}
+
+	return ip
+}
+
+func checkIP(ip net.IP) error {
+	ip = effectiveIP(ip)
+	if ip == nil {
+		return fmt.Errorf("invalid IP: %w", ErrUnsafeURL)
+	}
+	switch {
+	case ip.IsLoopback():
+		return fmt.Errorf("loopback address %s: %w", ip, ErrUnsafeURL)
+	case ip.IsUnspecified():
+		return fmt.Errorf("unspecified address %s: %w", ip, ErrUnsafeURL)
+	case ip.IsLinkLocalUnicast(), ip.IsLinkLocalMulticast():
+		return fmt.Errorf("link-local address %s: %w", ip, ErrUnsafeURL)
+	case ip.IsPrivate():
+		return fmt.Errorf("private address %s: %w", ip, ErrUnsafeURL)
+	case ip.IsMulticast():
+		return fmt.Errorf("multicast address %s: %w", ip, ErrUnsafeURL)
+	case ip.Equal(cloudMetadataIP):
+		return fmt.Errorf("cloud metadata address %s: %w", ip, ErrUnsafeURL)
+	case cgnatNet.Contains(ip):
+		return fmt.Errorf("CGNAT address %s: %w", ip, ErrUnsafeURL)
+	}
+	return nil
+}

pkg/request/ssrftest/ssrf_test.go

@@ -0,0 +1,163 @@
+// Package ssrftest contains tests for pkg/request SSRF validation. It lives
+// in a separate directory so it can compile even when the parent package's
+// pre-existing _test.go files do not.
+package ssrftest
+
+import (
+	"context"
+	"testing"
+
+	"github.com/cloudreve/Cloudreve/v4/pkg/request"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestValidateExternalURL_Scheme(t *testing.T) {
+	ctx := context.Background()
+	cases := []string{
+		"file:///etc/passwd",
+		"gopher://example.com/_GET",
+		"dict://127.0.0.1:11211/stat",
+		"ldap://x",
+		"javascript:alert(1)",
+		"",
+	}
+	for _, raw := range cases {
+		err := request.ValidateExternalURL(ctx, raw, request.SSRFOptions{})
+		assert.ErrorIs(t, err, request.ErrUnsafeURL, "raw=%q", raw)
+	}
+}
+
+func TestValidateExternalURL_LocalLiterals(t *testing.T) {
+	ctx := context.Background()
+	cases := []string{
+		"http://localhost/x",
+		"http://LOCALHOST:8080/x",
+		"http://ip6-localhost/x",
+		"http://service.localhost/x",
+		"https://localhost.localdomain/x",
+	}
+	for _, raw := range cases {
+		err := request.ValidateExternalURL(ctx, raw, request.SSRFOptions{})
+		assert.ErrorIs(t, err, request.ErrUnsafeURL, "raw=%q", raw)
+	}
+}
+
+func TestValidateExternalURL_IPLiterals(t *testing.T) {
+	ctx := context.Background()
+	cases := []string{
+		"http://127.0.0.1:7777/secret",
+		"http://[::1]:7780/v6secret",
+		"http://10.0.0.5/internal",
+		"http://172.16.5.5/internal",
+		"http://192.168.1.1/router",
+		"http://169.254.169.254/latest/meta",
+		"http://169.254.42.42/link-local",
+		"http://[fe80::1]/link-local-v6",
+		"http://[fc00::1]/ula",
+		"http://0.0.0.0/x",
+		"http://[::]/x",
+		"http://100.64.0.1/cgnat",
+		"http://[::ffff:127.0.0.1]/v4mapped",
+		"http://[::ffff:10.0.0.1]/v4mappedpriv",
+		"http://224.0.0.1/multicast",
+	}
+	for _, raw := range cases {
+		err := request.ValidateExternalURL(ctx, raw, request.SSRFOptions{})
+		assert.ErrorIs(t, err, request.ErrUnsafeURL, "raw=%q", raw)
+	}
+}
+
+func TestValidateExternalURL_PublicIP(t *testing.T) {
+	ctx := context.Background()
+	cases := []string{
+		"http://1.1.1.1/",
+		"https://8.8.8.8/",
+		"http://[2606:4700:4700::1111]/",
+	}
+	for _, raw := range cases {
+		err := request.ValidateExternalURL(ctx, raw, request.SSRFOptions{})
+		assert.NoError(t, err, "raw=%q", raw)
+	}
+}
+
+func TestValidateExternalURL_AllowedHostBypass(t *testing.T) {
+	ctx := context.Background()
+
+	opt := request.SSRFOptions{AllowedHosts: []string{"localhost"}}
+	for _, raw := range []string{
+		"http://localhost/file.zip",
+		"http://LOCALHOST:8080/file.zip",
+	} {
+		assert.NoError(t, request.ValidateExternalURL(ctx, raw, opt), "raw=%q", raw)
+	}
+
+	opt = request.SSRFOptions{AllowedHosts: []string{"cloudreve.example.com"}}
+	assert.NoError(t, request.ValidateExternalURL(ctx, "https://cloudreve.example.com/x", opt))
+
+	opt = request.SSRFOptions{AllowedHosts: []string{"127.0.0.1"}}
+	assert.NoError(t, request.ValidateExternalURL(ctx, "http://127.0.0.1:7777/secret", opt))
+}
+
+func TestValidateExternalURL_Disabled(t *testing.T) {
+	ctx := context.Background()
+	opt := request.SSRFOptions{Disabled: true}
+
+	// Everything passes when the operator has explicitly disabled validation.
+	for _, raw := range []string{
+		"http://127.0.0.1:7777/secret",
+		"http://localhost/x",
+		"file:///etc/passwd",
+		"http://169.254.169.254/latest",
+	} {
+		assert.NoError(t, request.ValidateExternalURL(ctx, raw, opt), "raw=%q", raw)
+	}
+}
+
+func TestValidateExternalURL_AllowedCIDR(t *testing.T) {
+	ctx := context.Background()
+
+	// LAN range allowlist lets a NAS at 192.168.10.50 through.
+	opt := request.SSRFOptions{AllowedCIDRs: []string{"192.168.10.0/24"}}
+	assert.NoError(t, request.ValidateExternalURL(ctx,
+		"http://192.168.10.50/share/file.zip", opt))
+
+	// IPs outside the allowlisted CIDR remain rejected.
+	err := request.ValidateExternalURL(ctx,
+		"http://192.168.11.50/share/file.zip", opt)
+	assert.ErrorIs(t, err, request.ErrUnsafeURL)
+
+	// IPv6 ULA allowlist.
+	opt = request.SSRFOptions{AllowedCIDRs: []string{"fd00::/8"}}
+	assert.NoError(t, request.ValidateExternalURL(ctx,
+		"http://[fd12:3456::1]/x", opt))
+
+	// Malformed CIDRs are silently dropped without affecting other entries.
+	opt = request.SSRFOptions{AllowedCIDRs: []string{"not-a-cidr", "10.0.0.0/8"}}
+	assert.NoError(t, request.ValidateExternalURL(ctx, "http://10.5.5.5/x", opt))
+}
+
+func TestValidateExternalURL_Magnet(t *testing.T) {
+	ctx := context.Background()
+
+	// DHT-only magnet (no tracker/web seed) is allowed.
+	assert.NoError(t, request.ValidateExternalURL(ctx,
+		"magnet:?xt=urn:btih:c12fe1c06bba254a9dc9f519b335aa7c1367a88a",
+		request.SSRFOptions{}))
+
+	// Internal tracker URL is rejected.
+	err := request.ValidateExternalURL(ctx,
+		"magnet:?xt=urn:btih:abc&tr=http://127.0.0.1:7777/announce",
+		request.SSRFOptions{})
+	assert.ErrorIs(t, err, request.ErrUnsafeURL)
+
+	// Internal web seed is rejected.
+	err = request.ValidateExternalURL(ctx,
+		"magnet:?xt=urn:btih:abc&ws=http://10.0.0.1/seed",
+		request.SSRFOptions{})
+	assert.ErrorIs(t, err, request.ErrUnsafeURL)
+
+	// Public tracker is allowed.
+	assert.NoError(t, request.ValidateExternalURL(ctx,
+		"magnet:?xt=urn:btih:abc&tr=udp://tracker.opentrackr.org:1337/announce",
+		request.SSRFOptions{}))
+}

pkg/request/tpslimiter.go

@@ -0,0 +1,39 @@
+package request
+
+import (
+	"context"
+	"golang.org/x/time/rate"
+	"sync"
+)
+
+var globalTPSLimiter = NewTPSLimiter()
+
+type TPSLimiter interface {
+	Limit(ctx context.Context, token string, tps float64, burst int)
+}
+
+func NewTPSLimiter() TPSLimiter {
+	return &multipleBucketLimiter{
+		buckets: make(map[string]*rate.Limiter),
+	}
+}
+
+// multipleBucketLimiter implements TPSLimiter with multiple bucket support.
+type multipleBucketLimiter struct {
+	mu      sync.Mutex
+	buckets map[string]*rate.Limiter
+}
+
+// Limit finds the given bucket, if bucket not exist or limit is changed,
+// a new bucket will be generated.
+func (m *multipleBucketLimiter) Limit(ctx context.Context, token string, tps float64, burst int) {
+	m.mu.Lock()
+	bucket, ok := m.buckets[token]
+	if !ok || float64(bucket.Limit()) != tps || bucket.Burst() != burst {
+		bucket = rate.NewLimiter(rate.Limit(tps), burst)
+		m.buckets[token] = bucket
+	}
+	m.mu.Unlock()
+
+	bucket.Wait(ctx)
+}

pkg/request/tpslimiter_test.go

@@ -0,0 +1,46 @@
+package request
+
+import (
+	"context"
+	"github.com/stretchr/testify/assert"
+	"testing"
+	"time"
+)
+
+func TestLimit(t *testing.T) {
+	a := assert.New(t)
+	l := NewTPSLimiter()
+	finished := make(chan struct{})
+	go func() {
+		l.Limit(context.Background(), "token", 1, 1)
+		close(finished)
+	}()
+	select {
+	case <-finished:
+	case <-time.After(10 * time.Second):
+		a.Fail("Limit should be finished instantly.")
+	}
+
+	finished = make(chan struct{})
+	go func() {
+		l.Limit(context.Background(), "token", 1, 1)
+		close(finished)
+	}()
+	select {
+	case <-finished:
+	case <-time.After(2 * time.Second):
+		a.Fail("Limit should be finished in 1 second.")
+	}
+
+	finished = make(chan struct{})
+	go func() {
+		l.Limit(context.Background(), "token", 10, 1)
+		close(finished)
+	}()
+	select {
+	case <-finished:
+	case <-time.After(1 * time.Second):
+		a.Fail("Limit should be finished instantly.")
+	}
+
+}

pkg/request/utils.go

@@ -0,0 +1,62 @@
+package request
+
+import (
+	"io"
+	"net/http"
+	"strconv"
+)
+
+var contentLengthHeaders = []string{
+	"Content-Length",
+	"X-Expected-Entity-Length", // DavFS on MacOS
+}
+
+// BlackHole 将客户端发来的数据放入黑洞
+func BlackHole(r io.Reader) {
+	io.Copy(io.Discard, r)
+}
+
+// SniffContentLength tries to get the content length from the request. It also returns
+// a reader that will limit to the sniffed content length.
+func SniffContentLength(r *http.Request) (LimitReaderCloser, int64, error) {
+	for _, header := range contentLengthHeaders {
+		if length := r.Header.Get(header); length != "" {
+			res, err := strconv.ParseInt(length, 10, 64)
+			if err != nil {
+				return nil, 0, err
+			}
+
+			return newLimitReaderCloser(r.Body, res), res, nil
+		}
+	}
+	return newLimitReaderCloser(r.Body, 0), 0, nil
+}
+
+type LimitReaderCloser interface {
+	io.Reader
+	io.Closer
+	Count() int64
+}
+
+type limitReaderCloser struct {
+	io.Reader
+	io.Closer
+	read int64
+}
+
+func newLimitReaderCloser(r io.ReadCloser, limit int64) LimitReaderCloser {
+	return &limitReaderCloser{
+		Reader: io.LimitReader(r, limit),
+		Closer: r,
+	}
+}
+
+func (l *limitReaderCloser) Read(p []byte) (n int, err error) {
+	n, err = l.Reader.Read(p)
+	l.read += int64(n)
+	return n, err
+}
+
+func (l *limitReaderCloser) Count() int64 {
+	return l.read
+}