chore: 初始化 Cloudreve 源码仓库

This commit is contained in:
2026-09-13 11:28:59 +08:00
commit 2a7b03ab1d
489 changed files with 163867 additions and 0 deletions
+36
View File
@@ -0,0 +1,36 @@
package cache
import (
"encoding/gob"
)
func init() {
gob.Register(map[string]itemWithTTL{})
}
// Driver 键值缓存存储容器
type Driver interface {
// 设置值,ttl为过期时间,单位为秒
Set(key string, value any, ttl int) error
// 取值,并返回是否成功
Get(key string) (any, bool)
// 批量取值,返回成功取值的map即不存在的值
Gets(keys []string, prefix string) (map[string]any, []string)
// 批量设置值,所有的key都会加上prefix前缀
Sets(values map[string]any, prefix string) error
// Delete values by [Prefix + key]. If no ket is presented, all keys with given prefix will be deleted.
Delete(prefix string, keys ...string) error
// Save in-memory cache to disk
Persist(path string) error
// Restore cache from disk
Restore(path string) error
// Remove all entries
DeleteAll() error
}
+61
View File
@@ -0,0 +1,61 @@
package cache
import (
"github.com/stretchr/testify/assert"
"testing"
)
func TestSet(t *testing.T) {
asserts := assert.New(t)
asserts.NoError(Set("123", "321", -1))
}
func TestGet(t *testing.T) {
asserts := assert.New(t)
asserts.NoError(Set("123", "321", -1))
value, ok := Get("123")
asserts.True(ok)
asserts.Equal("321", value)
value, ok = Get("not_exist")
asserts.False(ok)
}
func TestDeletes(t *testing.T) {
asserts := assert.New(t)
asserts.NoError(Set("123", "321", -1))
err := Deletes([]string{"123"}, "")
asserts.NoError(err)
_, exist := Get("123")
asserts.False(exist)
}
func TestGetSettings(t *testing.T) {
asserts := assert.New(t)
asserts.NoError(Set("test_1", "1", -1))
values, missed := GetSettings([]string{"1", "2"}, "test_")
asserts.Equal(map[string]string{"1": "1"}, values)
asserts.Equal([]string{"2"}, missed)
}
func TestSetSettings(t *testing.T) {
asserts := assert.New(t)
err := SetSettings(map[string]string{"3": "3", "4": "4"}, "test_")
asserts.NoError(err)
value1, _ := Get("test_3")
value2, _ := Get("test_4")
asserts.Equal("3", value1)
asserts.Equal("4", value2)
}
func TestInit(t *testing.T) {
asserts := assert.New(t)
asserts.NotPanics(func() {
Init()
})
}
+204
View File
@@ -0,0 +1,204 @@
package cache
import (
"encoding/gob"
"fmt"
"os"
"strings"
"sync"
"time"
"github.com/cloudreve/Cloudreve/v4/pkg/logging"
"github.com/cloudreve/Cloudreve/v4/pkg/util"
)
// MemoStore 内存存储驱动
type MemoStore struct {
Store *sync.Map
}
// item 存储的对象
type itemWithTTL struct {
Expires int64
Value interface{}
}
const DefaultCacheFile = "cache_persist.bin"
func newItem(value interface{}, expires int) itemWithTTL {
expires64 := int64(expires)
if expires > 0 {
expires64 = time.Now().Unix() + expires64
}
return itemWithTTL{
Value: value,
Expires: expires64,
}
}
// getValue 从itemWithTTL中取值
func getValue(item any, ok bool) (any, bool) {
if !ok {
return nil, ok
}
var itemObj itemWithTTL
if itemObj, ok = item.(itemWithTTL); !ok {
return item, true
}
if itemObj.Expires > 0 && itemObj.Expires < time.Now().Unix() {
return nil, false
}
return itemObj.Value, ok
}
// GarbageCollect 回收已过期的缓存
func (store *MemoStore) GarbageCollect(l logging.Logger) {
store.Store.Range(func(key, value any) bool {
if item, ok := value.(itemWithTTL); ok {
if item.Expires > 0 && item.Expires < time.Now().Unix() {
l.Debug("Cache %q is garbage collected.", key.(string))
store.Store.Delete(key)
}
}
return true
})
}
// NewMemoStore 新建内存存储
func NewMemoStore(persistFile string, l logging.Logger) *MemoStore {
store := &MemoStore{
Store: &sync.Map{},
}
if persistFile != "" {
if err := store.Restore(persistFile); err != nil {
l.Warning("Failed to restore cache from disk: %s", err)
}
}
return store
}
// Set 存储值
func (store *MemoStore) Set(key string, value any, ttl int) error {
store.Store.Store(key, newItem(value, ttl))
return nil
}
// Get 取值
func (store *MemoStore) Get(key string) (any, bool) {
return getValue(store.Store.Load(key))
}
// Gets 批量取值
func (store *MemoStore) Gets(keys []string, prefix string) (map[string]any, []string) {
var res = make(map[string]interface{})
var notFound = make([]string, 0, len(keys))
for _, key := range keys {
if value, ok := getValue(store.Store.Load(prefix + key)); ok {
res[key] = value
} else {
notFound = append(notFound, key)
}
}
return res, notFound
}
// Sets 批量设置值
func (store *MemoStore) Sets(values map[string]any, prefix string) error {
for key, value := range values {
store.Store.Store(prefix+key, newItem(value, 0))
}
return nil
}
// Delete 批量删除值
func (store *MemoStore) Delete(prefix string, keys ...string) error {
for _, key := range keys {
store.Store.Delete(prefix + key)
}
// No key is presented, delete all entries with given prefix.
if len(keys) == 0 {
store.Store.Range(func(key, value any) bool {
if k, ok := key.(string); ok && strings.HasPrefix(k, prefix) {
store.Store.Delete(key)
}
return true
})
}
return nil
}
// Persist write memory store into cache
func (store *MemoStore) Persist(path string) error {
persisted := make(map[string]itemWithTTL)
store.Store.Range(func(key, value any) bool {
v, ok := store.Store.Load(key)
if _, ok := getValue(v, ok); ok {
persisted[key.(string)] = v.(itemWithTTL)
}
return true
})
res, err := serializer(persisted)
if err != nil {
return fmt.Errorf("failed to serialize cache: %s", err)
}
err = os.WriteFile(path, res, 0644)
return err
}
// Restore memory cache from disk file
func (store *MemoStore) Restore(path string) error {
if !util.Exists(path) {
return nil
}
f, err := os.Open(path)
if err != nil {
return fmt.Errorf("failed to read cache file: %s", err)
}
defer func() {
f.Close()
os.Remove(path)
}()
persisted := &item{}
dec := gob.NewDecoder(f)
if err := dec.Decode(&persisted); err != nil {
return fmt.Errorf("unknown cache file format: %s", err)
}
items := persisted.Value.(map[string]itemWithTTL)
loaded := 0
for k, v := range items {
if _, ok := getValue(v, true); ok {
loaded++
store.Store.Store(k, v)
} else {
util.Log().Debug("Persisted cache %q is expired.", k)
}
}
util.Log().Info("Restored %d items from %q into memory cache.", loaded, path)
return nil
}
func (store *MemoStore) DeleteAll() error {
store.Store.Range(func(key any, value any) bool {
store.Store.Delete(key)
return true
})
return nil
}
+147
View File
@@ -0,0 +1,147 @@
package cache
import (
"github.com/stretchr/testify/assert"
"testing"
"time"
)
func TestNewMemoStore(t *testing.T) {
asserts := assert.New(t)
store := NewMemoStore()
asserts.NotNil(store)
asserts.NotNil(store.Store)
}
func TestMemoStore_Set(t *testing.T) {
asserts := assert.New(t)
store := NewMemoStore()
err := store.Set("KEY", "vAL", -1)
asserts.NoError(err)
val, ok := store.Store.Load("KEY")
asserts.True(ok)
asserts.Equal("vAL", val.(itemWithTTL).Value)
}
func TestMemoStore_Get(t *testing.T) {
asserts := assert.New(t)
store := NewMemoStore()
// 正常情况
{
_ = store.Set("string", "string_val", -1)
val, ok := store.Get("string")
asserts.Equal("string_val", val)
asserts.True(ok)
}
// Key不存在
{
val, ok := store.Get("something")
asserts.Equal(nil, val)
asserts.False(ok)
}
// 存储struct
{
type testStruct struct {
key int
}
test := testStruct{key: 233}
_ = store.Set("struct", test, -1)
val, ok := store.Get("struct")
asserts.True(ok)
res, ok := val.(testStruct)
asserts.True(ok)
asserts.Equal(test, res)
}
// 过期
{
_ = store.Set("string", "string_val", 1)
time.Sleep(time.Duration(2) * time.Second)
val, ok := store.Get("string")
asserts.Nil(val)
asserts.False(ok)
}
}
func TestMemoStore_Gets(t *testing.T) {
asserts := assert.New(t)
store := NewMemoStore()
err := store.Set("1", "1,val", -1)
err = store.Set("2", "2,val", -1)
err = store.Set("3", "3,val", -1)
err = store.Set("4", "4,val", -1)
asserts.NoError(err)
// 全部命中
{
values, miss := store.Gets([]string{"1", "2", "3", "4"}, "")
asserts.Len(values, 4)
asserts.Len(miss, 0)
}
// 命中一半
{
values, miss := store.Gets([]string{"1", "2", "9", "10"}, "")
asserts.Len(values, 2)
asserts.Equal([]string{"9", "10"}, miss)
}
}
func TestMemoStore_Sets(t *testing.T) {
asserts := assert.New(t)
store := NewMemoStore()
err := store.Sets(map[string]interface{}{
"1": "1.val",
"2": "2.val",
"3": "3.val",
"4": "4.val",
}, "test_")
asserts.NoError(err)
vals, miss := store.Gets([]string{"1", "2", "3", "4"}, "test_")
asserts.Len(miss, 0)
asserts.Equal(map[string]interface{}{
"1": "1.val",
"2": "2.val",
"3": "3.val",
"4": "4.val",
}, vals)
}
func TestMemoStore_Delete(t *testing.T) {
asserts := assert.New(t)
store := NewMemoStore()
err := store.Sets(map[string]interface{}{
"1": "1.val",
"2": "2.val",
"3": "3.val",
"4": "4.val",
}, "test_")
asserts.NoError(err)
err = store.Delete([]string{"1", "2"}, "test_")
asserts.NoError(err)
values, miss := store.Gets([]string{"1", "2", "3", "4"}, "test_")
asserts.Equal([]string{"1", "2"}, miss)
asserts.Equal(map[string]interface{}{"3": "3.val", "4": "4.val"}, values)
}
func TestMemoStore_GarbageCollect(t *testing.T) {
asserts := assert.New(t)
store := NewMemoStore()
store.Set("test", 1, 1)
time.Sleep(time.Duration(2000) * time.Millisecond)
store.GarbageCollect()
_, ok := store.Get("test")
asserts.False(ok)
}
+245
View File
@@ -0,0 +1,245 @@
package cache
import (
"bytes"
"encoding/gob"
"strconv"
"time"
"github.com/cloudreve/Cloudreve/v4/pkg/conf"
"github.com/cloudreve/Cloudreve/v4/pkg/logging"
"github.com/gomodule/redigo/redis"
)
// RedisStore redis存储驱动
type RedisStore struct {
pool *redis.Pool
}
type item struct {
Value interface{}
}
func serializer(value any) ([]byte, error) {
var buffer bytes.Buffer
enc := gob.NewEncoder(&buffer)
storeValue := item{
Value: value,
}
err := enc.Encode(storeValue)
if err != nil {
return nil, err
}
return buffer.Bytes(), nil
}
func deserializer(value []byte) (any, error) {
var res item
buffer := bytes.NewReader(value)
dec := gob.NewDecoder(buffer)
err := dec.Decode(&res)
if err != nil {
return nil, err
}
return res.Value, nil
}
// NewRedisStore 创建新的redis存储
func NewRedisStore(l logging.Logger, size int, redisConfig *conf.Redis) *RedisStore {
return &RedisStore{
pool: &redis.Pool{
MaxIdle: size,
IdleTimeout: 240 * time.Second,
TestOnBorrow: func(c redis.Conn, t time.Time) error {
_, err := c.Do("PING")
return err
},
Dial: func() (redis.Conn, error) {
db, err := strconv.Atoi(redisConfig.DB)
if err != nil {
return nil, err
}
c, err := redis.Dial(
redisConfig.Network,
redisConfig.Server,
redis.DialDatabase(db),
redis.DialPassword(redisConfig.Password),
redis.DialUsername(redisConfig.User),
redis.DialUseTLS(redisConfig.UseTLS),
redis.DialTLSSkipVerify(redisConfig.TLSSkipVerify),
)
if err != nil {
l.Panic("Failed to create Redis connection: %s", err)
}
return c, nil
},
},
}
}
// Set 存储值
func (store *RedisStore) Set(key string, value any, ttl int) error {
rc := store.pool.Get()
defer rc.Close()
serialized, err := serializer(value)
if err != nil {
return err
}
if rc.Err() != nil {
return rc.Err()
}
if ttl > 0 {
_, err = rc.Do("SETEX", key, ttl, serialized)
} else {
_, err = rc.Do("SET", key, serialized)
}
if err != nil {
return err
}
return nil
}
// Get 取值
func (store *RedisStore) Get(key string) (any, bool) {
rc := store.pool.Get()
defer rc.Close()
if rc.Err() != nil {
return nil, false
}
v, err := redis.Bytes(rc.Do("GET", key))
if err != nil || v == nil {
return nil, false
}
finalValue, err := deserializer(v)
if err != nil {
return nil, false
}
return finalValue, true
}
// Gets 批量取值
func (store *RedisStore) Gets(keys []string, prefix string) (map[string]any, []string) {
rc := store.pool.Get()
defer rc.Close()
if rc.Err() != nil {
return nil, keys
}
var queryKeys = make([]string, len(keys))
for key, value := range keys {
queryKeys[key] = prefix + value
}
v, err := redis.ByteSlices(rc.Do("MGET", redis.Args{}.AddFlat(queryKeys)...))
if err != nil {
return nil, keys
}
var res = make(map[string]any)
var missed = make([]string, 0, len(keys))
for key, value := range v {
decoded, err := deserializer(value)
if err != nil || decoded == nil {
missed = append(missed, keys[key])
} else {
res[keys[key]] = decoded
}
}
// 解码所得值
return res, missed
}
// Sets 批量设置值
func (store *RedisStore) Sets(values map[string]any, prefix string) error {
rc := store.pool.Get()
defer rc.Close()
if rc.Err() != nil {
return rc.Err()
}
var setValues = make(map[string]any)
// 编码待设置值
for key, value := range values {
serialized, err := serializer(value)
if err != nil {
return err
}
setValues[prefix+key] = serialized
}
_, err := rc.Do("MSET", redis.Args{}.AddFlat(setValues)...)
if err != nil {
return err
}
return nil
}
// Delete 批量删除给定的键
func (store *RedisStore) Delete(prefix string, keys ...string) error {
rc := store.pool.Get()
defer rc.Close()
if rc.Err() != nil {
return rc.Err()
}
// 处理前缀
for i := 0; i < len(keys); i++ {
keys[i] = prefix + keys[i]
}
// No key is presented, delete all keys with given prefix
if len(keys) == 0 {
// Fetch all key with given prefix
allPrefixKeys, err := redis.Strings(rc.Do("KEYS", prefix+"*"))
if err != nil {
return err
}
keys = allPrefixKeys
}
if len(keys) > 0 {
_, err := rc.Do("DEL", redis.Args{}.AddFlat(keys)...)
if err != nil {
return err
}
}
return nil
}
// DeleteAll 批量所有键
func (store *RedisStore) DeleteAll() error {
rc := store.pool.Get()
defer rc.Close()
if rc.Err() != nil {
return rc.Err()
}
_, err := rc.Do("FLUSHDB")
return err
}
// Persist Dummy implementation
func (store *RedisStore) Persist(path string) error {
return nil
}
// Restore dummy implementation
func (store *RedisStore) Restore(path string) error {
return nil
}
+324
View File
@@ -0,0 +1,324 @@
package cache
import (
"errors"
"fmt"
"github.com/gomodule/redigo/redis"
"github.com/rafaeljusto/redigomock"
"github.com/stretchr/testify/assert"
"testing"
"time"
)
func TestNewRedisStore(t *testing.T) {
asserts := assert.New(t)
store := NewRedisStore(10, "tcp", "", "", "0")
asserts.NotNil(store)
conn, err := store.pool.Dial()
asserts.Nil(conn)
asserts.Error(err)
testConn := redigomock.NewConn()
cmd := testConn.Command("PING").Expect("PONG")
err = store.pool.TestOnBorrow(testConn, time.Now())
if testConn.Stats(cmd) != 1 {
fmt.Println("Command was not used")
return
}
asserts.NoError(err)
}
func TestRedisStore_Set(t *testing.T) {
asserts := assert.New(t)
conn := redigomock.NewConn()
pool := &redis.Pool{
Dial: func() (redis.Conn, error) { return conn, nil },
MaxIdle: 10,
}
store := &RedisStore{pool: pool}
// 正常情况
{
cmd := conn.Command("SET", "test", redigomock.NewAnyData()).ExpectStringSlice("OK")
err := store.Set("test", "test val", -1)
asserts.NoError(err)
if conn.Stats(cmd) != 1 {
fmt.Println("Command was not used")
return
}
}
// 带有TTL
// 正常情况
{
cmd := conn.Command("SETEX", "test", 10, redigomock.NewAnyData()).ExpectStringSlice("OK")
err := store.Set("test", "test val", 10)
asserts.NoError(err)
if conn.Stats(cmd) != 1 {
fmt.Println("Command was not used")
return
}
}
// 序列化出错
{
value := struct {
Key string
}{
Key: "123",
}
err := store.Set("test", value, -1)
asserts.Error(err)
}
// 命令执行失败
{
conn.Clear()
cmd := conn.Command("SET", "test", redigomock.NewAnyData()).ExpectError(errors.New("error"))
err := store.Set("test", "test val", -1)
asserts.Error(err)
if conn.Stats(cmd) != 1 {
fmt.Println("Command was not used")
return
}
}
// 获取连接失败
{
store.pool = &redis.Pool{
Dial: func() (redis.Conn, error) { return nil, errors.New("error") },
MaxIdle: 10,
}
err := store.Set("test", "123", -1)
asserts.Error(err)
}
}
func TestRedisStore_Get(t *testing.T) {
asserts := assert.New(t)
conn := redigomock.NewConn()
pool := &redis.Pool{
Dial: func() (redis.Conn, error) { return conn, nil },
MaxIdle: 10,
}
store := &RedisStore{pool: pool}
// 正常情况
{
expectVal, _ := serializer("test val")
cmd := conn.Command("GET", "test").Expect(expectVal)
val, ok := store.Get("test")
if conn.Stats(cmd) != 1 {
fmt.Println("Command was not used")
return
}
asserts.True(ok)
asserts.Equal("test val", val.(string))
}
// Key不存在
{
conn.Clear()
cmd := conn.Command("GET", "test").Expect(nil)
val, ok := store.Get("test")
if conn.Stats(cmd) != 1 {
fmt.Println("Command was not used")
return
}
asserts.False(ok)
asserts.Nil(val)
}
// 解码错误
{
conn.Clear()
cmd := conn.Command("GET", "test").Expect([]byte{0x20})
val, ok := store.Get("test")
if conn.Stats(cmd) != 1 {
fmt.Println("Command was not used")
return
}
asserts.False(ok)
asserts.Nil(val)
}
// 获取连接失败
{
store.pool = &redis.Pool{
Dial: func() (redis.Conn, error) { return nil, errors.New("error") },
MaxIdle: 10,
}
val, ok := store.Get("test")
asserts.False(ok)
asserts.Nil(val)
}
}
func TestRedisStore_Gets(t *testing.T) {
asserts := assert.New(t)
conn := redigomock.NewConn()
pool := &redis.Pool{
Dial: func() (redis.Conn, error) { return conn, nil },
MaxIdle: 10,
}
store := &RedisStore{pool: pool}
// 全部命中
{
conn.Clear()
value1, _ := serializer("1")
value2, _ := serializer("2")
cmd := conn.Command("MGET", "test_1", "test_2").ExpectSlice(
value1, value2)
res, missed := store.Gets([]string{"1", "2"}, "test_")
if conn.Stats(cmd) != 1 {
fmt.Println("Command was not used")
return
}
asserts.Len(missed, 0)
asserts.Len(res, 2)
asserts.Equal("1", res["1"].(string))
asserts.Equal("2", res["2"].(string))
}
// 命中一个
{
conn.Clear()
value2, _ := serializer("2")
cmd := conn.Command("MGET", "test_1", "test_2").ExpectSlice(
nil, value2)
res, missed := store.Gets([]string{"1", "2"}, "test_")
if conn.Stats(cmd) != 1 {
fmt.Println("Command was not used")
return
}
asserts.Len(missed, 1)
asserts.Len(res, 1)
asserts.Equal("1", missed[0])
asserts.Equal("2", res["2"].(string))
}
// 命令出错
{
conn.Clear()
cmd := conn.Command("MGET", "test_1", "test_2").ExpectError(errors.New("error"))
res, missed := store.Gets([]string{"1", "2"}, "test_")
if conn.Stats(cmd) != 1 {
fmt.Println("Command was not used")
return
}
asserts.Len(missed, 2)
asserts.Len(res, 0)
}
// 连接出错
{
conn.Clear()
store.pool = &redis.Pool{
Dial: func() (redis.Conn, error) { return nil, errors.New("error") },
MaxIdle: 10,
}
res, missed := store.Gets([]string{"1", "2"}, "test_")
asserts.Len(missed, 2)
asserts.Len(res, 0)
}
}
func TestRedisStore_Sets(t *testing.T) {
asserts := assert.New(t)
conn := redigomock.NewConn()
pool := &redis.Pool{
Dial: func() (redis.Conn, error) { return conn, nil },
MaxIdle: 10,
}
store := &RedisStore{pool: pool}
// 正常
{
cmd := conn.Command("MSET", redigomock.NewAnyData(), redigomock.NewAnyData(), redigomock.NewAnyData(), redigomock.NewAnyData()).ExpectSlice("OK")
err := store.Sets(map[string]interface{}{"1": "1", "2": "2"}, "test_")
asserts.NoError(err)
if conn.Stats(cmd) != 1 {
fmt.Println("Command was not used")
return
}
}
// 序列化失败
{
conn.Clear()
value := struct {
Key string
}{
Key: "123",
}
err := store.Sets(map[string]interface{}{"1": value, "2": "2"}, "test_")
asserts.Error(err)
}
// 执行失败
{
cmd := conn.Command("MSET", redigomock.NewAnyData(), redigomock.NewAnyData(), redigomock.NewAnyData(), redigomock.NewAnyData()).ExpectError(errors.New("error"))
err := store.Sets(map[string]interface{}{"1": "1", "2": "2"}, "test_")
asserts.Error(err)
if conn.Stats(cmd) != 1 {
fmt.Println("Command was not used")
return
}
}
// 连接失败
{
conn.Clear()
store.pool = &redis.Pool{
Dial: func() (redis.Conn, error) { return nil, errors.New("error") },
MaxIdle: 10,
}
err := store.Sets(map[string]interface{}{"1": "1", "2": "2"}, "test_")
asserts.Error(err)
}
}
func TestRedisStore_Delete(t *testing.T) {
asserts := assert.New(t)
conn := redigomock.NewConn()
pool := &redis.Pool{
Dial: func() (redis.Conn, error) { return conn, nil },
MaxIdle: 10,
}
store := &RedisStore{pool: pool}
// 正常
{
cmd := conn.Command("DEL", redigomock.NewAnyData(), redigomock.NewAnyData(), redigomock.NewAnyData(), redigomock.NewAnyData()).ExpectSlice("OK")
err := store.Delete([]string{"1", "2", "3", "4"}, "test_")
asserts.NoError(err)
if conn.Stats(cmd) != 1 {
fmt.Println("Command was not used")
return
}
}
// 命令执行失败
{
conn.Clear()
cmd := conn.Command("DEL", redigomock.NewAnyData(), redigomock.NewAnyData(), redigomock.NewAnyData(), redigomock.NewAnyData()).ExpectError(errors.New("error"))
err := store.Delete([]string{"1", "2", "3", "4"}, "test_")
asserts.Error(err)
if conn.Stats(cmd) != 1 {
fmt.Println("Command was not used")
return
}
}
// 连接失败
{
conn.Clear()
store.pool = &redis.Pool{
Dial: func() (redis.Conn, error) { return nil, errors.New("error") },
MaxIdle: 10,
}
err := store.Delete([]string{"1", "2", "3", "4"}, "test_")
asserts.Error(err)
}
}