Henry_Du/BYWCloudreve
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: 初始化 Cloudreve 源码仓库

Browse Source
This commit is contained in:
Henry_Du
2026-09-13 11:28:59 +08:00
commit 2a7b03ab1d
489 changed files with 163867 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/filemanager/manager/archive.go
+317
View File
@@ -0,0 +1,317 @@
package manager
import (
"archive/zip"
"context"
"encoding/gob"
"fmt"
"io"
"path"
"path/filepath"
"strings"
"time"
"github.com/bodgit/sevenzip"
"github.com/cloudreve/Cloudreve/v4/inventory/types"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/fs"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/fs/dbfs"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/manager/entitysource"
"github.com/cloudreve/Cloudreve/v4/pkg/util"
"golang.org/x/text/encoding"
"golang.org/x/text/encoding/charmap"
"golang.org/x/text/encoding/japanese"
"golang.org/x/text/encoding/korean"
"golang.org/x/text/encoding/simplifiedchinese"
"golang.org/x/text/encoding/traditionalchinese"
"golang.org/x/text/encoding/unicode"
"golang.org/x/tools/container/intsets"
)
type (
ArchivedFile struct {
Name string `json:"name"`
Size int64 `json:"size"`
UpdatedAt *time.Time `json:"updated_at"`
IsDirectory bool `json:"is_directory"`
}
)
const (
ArchiveListCacheTTL = 3600 // 1 hour
)
func init() {
gob.Register([]ArchivedFile{})
}
var ZipEncodings = map[string]encoding.Encoding{
"ibm866": charmap.CodePage866,
"iso8859_2": charmap.ISO8859_2,
"iso8859_3": charmap.ISO8859_3,
"iso8859_4": charmap.ISO8859_4,
"iso8859_5": charmap.ISO8859_5,
"iso8859_6": charmap.ISO8859_6,
"iso8859_7": charmap.ISO8859_7,
"iso8859_8": charmap.ISO8859_8,
"iso8859_8I": charmap.ISO8859_8I,
"iso8859_10": charmap.ISO8859_10,
"iso8859_13": charmap.ISO8859_13,
"iso8859_14": charmap.ISO8859_14,
"iso8859_15": charmap.ISO8859_15,
"iso8859_16": charmap.ISO8859_16,
"koi8r": charmap.KOI8R,
"koi8u": charmap.KOI8U,
"macintosh": charmap.Macintosh,
"windows874": charmap.Windows874,
"windows1250": charmap.Windows1250,
"windows1251": charmap.Windows1251,
"windows1252": charmap.Windows1252,
"windows1253": charmap.Windows1253,
"windows1254": charmap.Windows1254,
"windows1255": charmap.Windows1255,
"windows1256": charmap.Windows1256,
"windows1257": charmap.Windows1257,
"windows1258": charmap.Windows1258,
"macintoshcyrillic": charmap.MacintoshCyrillic,
"gbk": simplifiedchinese.GBK,
"gb18030": simplifiedchinese.GB18030,
"big5": traditionalchinese.Big5,
"eucjp": japanese.EUCJP,
"iso2022jp": japanese.ISO2022JP,
"shiftjis": japanese.ShiftJIS,
"euckr": korean.EUCKR,
"utf16be": unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM),
"utf16le": unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM),
}
func (m *manager) ListArchiveFiles(ctx context.Context, uri *fs.URI, entity, zipEncoding string) ([]ArchivedFile, error) {
file, err := m.fs.Get(ctx, uri, dbfs.WithFileEntities(), dbfs.WithRequiredCapabilities(dbfs.NavigatorCapabilityDownloadFile))
if err != nil {
return nil, fmt.Errorf("failed to get file: %w", err)
}
if file.Type() != types.FileTypeFile {
return nil, fs.ErrNotSupportedAction.WithError(fmt.Errorf("path %s is not a file", uri))
}
// Validate file size
if m.user.Edges.Group.Settings.DecompressSize > 0 && file.Size() > m.user.Edges.Group.Settings.DecompressSize {
return nil, fs.ErrFileSizeTooBig.WithError(fmt.Errorf("file size %d exceeds the limit %d", file.Size(), m.user.Edges.Group.Settings.DecompressSize))
}
found, targetEntity := fs.FindDesiredEntity(file, entity, m.hasher, nil)
if !found {
return nil, fs.ErrEntityNotExist
}
var (
enc encoding.Encoding
ok bool
)
if zipEncoding != "" {
enc, ok = ZipEncodings[strings.ToLower(zipEncoding)]
if !ok {
return nil, fs.ErrNotSupportedAction.WithError(fmt.Errorf("not supported zip encoding: %s", zipEncoding))
}
}
cacheKey := getArchiveListCacheKey(targetEntity.ID(), zipEncoding)
kv := m.kv
res, found := kv.Get(cacheKey)
if found {
return res.([]ArchivedFile), nil
}
es, err := m.GetEntitySource(ctx, 0, fs.WithEntity(targetEntity))
if err != nil {
return nil, fmt.Errorf("failed to get entity source: %w", err)
}
es.Apply(entitysource.WithContext(ctx))
defer es.Close()
var readerFunc func(ctx context.Context, file io.ReaderAt, size int64, textEncoding encoding.Encoding) ([]ArchivedFile, error)
switch file.Ext() {
case "zip":
readerFunc = getZipFileList
case "7z":
readerFunc = get7zFileList
default:
return nil, fs.ErrNotSupportedAction.WithError(fmt.Errorf("not supported archive format: %s", file.Ext()))
}
sr := io.NewSectionReader(es, 0, targetEntity.Size())
fileList, err := readerFunc(ctx, sr, targetEntity.Size(), enc)
if err != nil {
return nil, fmt.Errorf("failed to read file list: %w", err)
}
kv.Set(cacheKey, fileList, ArchiveListCacheTTL)
return fileList, nil
}
func (m *manager) CreateArchive(ctx context.Context, uris []*fs.URI, writer io.Writer, opts ...fs.Option) (int, error) {
o := newOption()
for _, opt := range opts {
opt.Apply(o)
}
failed := 0
// List all top level files
files := make([]fs.File, 0, len(uris))
for _, uri := range uris {
file, err := m.Get(ctx, uri, dbfs.WithFileEntities(), dbfs.WithRequiredCapabilities(dbfs.NavigatorCapabilityDownloadFile), dbfs.WithNotRoot())
if err != nil {
return 0, fmt.Errorf("failed to get file %s: %w", uri, err)
}
files = append(files, file)
}
zipWriter := zip.NewWriter(writer)
defer zipWriter.Close()
var compressed int64
for _, file := range files {
if file.Type() == types.FileTypeFile {
if err := m.compressFileToArchive(ctx, "/", file, zipWriter, o.ArchiveCompression, o.DryRun); err != nil {
failed++
m.l.Warning("Failed to compress file %s: %s, skipping it...", file.Uri(false), err)
}
compressed += file.Size()
if o.ProgressFunc != nil {
o.ProgressFunc(compressed, file.Size(), 0)
}
if o.MaxArchiveSize > 0 && compressed > o.MaxArchiveSize {
return 0, fs.ErrArchiveSrcSizeTooBig
}
} else {
if err := m.Walk(ctx, file.Uri(false), intsets.MaxInt, func(f fs.File, level int) error {
if f.Type() == types.FileTypeFolder || f.IsSymbolic() {
return nil
}
if err := m.compressFileToArchive(ctx, strings.TrimPrefix(f.Uri(false).Dir(),
file.Uri(false).Dir()), f, zipWriter, o.ArchiveCompression, o.DryRun); err != nil {
failed++
m.l.Warning("Failed to compress file %s: %s, skipping it...", f.Uri(false), err)
}
compressed += f.Size()
if o.ProgressFunc != nil {
o.ProgressFunc(compressed, f.Size(), 0)
}
if o.MaxArchiveSize > 0 && compressed > o.MaxArchiveSize {
return fs.ErrArchiveSrcSizeTooBig
}
return nil
}); err != nil {
m.l.Warning("Failed to walk folder %s: %s, skipping it...", file.Uri(false), err)
failed++
}
}
}
return failed, nil
}
func (m *manager) compressFileToArchive(ctx context.Context, parent string, file fs.File, zipWriter *zip.Writer,
compression bool, dryrun fs.CreateArchiveDryRunFunc) error {
es, err := m.GetEntitySource(ctx, file.PrimaryEntityID())
if err != nil {
return fmt.Errorf("failed to get entity source for file %s: %w", file.Uri(false), err)
}
zipName := filepath.FromSlash(path.Join(parent, file.DisplayName()))
if dryrun != nil {
dryrun(zipName, es.Entity())
return nil
}
m.l.Debug("Compressing %s to archive...", file.Uri(false))
header := &zip.FileHeader{
Name: zipName,
Modified: file.UpdatedAt(),
UncompressedSize64: uint64(file.Size()),
}
if !compression {
header.Method = zip.Store
} else {
header.Method = zip.Deflate
}
writer, err := zipWriter.CreateHeader(header)
if err != nil {
return fmt.Errorf("failed to create zip header for %s: %w", file.Uri(false), err)
}
es.Apply(entitysource.WithContext(ctx))
_, err = io.Copy(writer, es)
return err
}
func getZipFileList(ctx context.Context, file io.ReaderAt, size int64, textEncoding encoding.Encoding) ([]ArchivedFile, error) {
zr, err := zip.NewReader(file, size)
if err != nil {
return nil, fmt.Errorf("failed to create zip reader: %w", err)
}
fileList := make([]ArchivedFile, 0, len(zr.File))
for _, f := range zr.File {
hdr := f.FileHeader
if hdr.NonUTF8 && textEncoding != nil {
dec := textEncoding.NewDecoder()
filename, err := dec.String(hdr.Name)
if err == nil {
hdr.Name = filename
}
if hdr.Comment != "" {
comment, err := dec.String(hdr.Comment)
if err == nil {
hdr.Comment = comment
}
}
}
info := f.FileInfo()
modTime := info.ModTime()
fileList = append(fileList, ArchivedFile{
Name: util.FormSlash(hdr.Name),
Size: info.Size(),
UpdatedAt: &modTime,
IsDirectory: info.IsDir(),
})
}
return fileList, nil
}
func get7zFileList(ctx context.Context, file io.ReaderAt, size int64, extEncoding encoding.Encoding) ([]ArchivedFile, error) {
zr, err := sevenzip.NewReader(file, size)
if err != nil {
return nil, fmt.Errorf("failed to create 7z reader: %w", err)
}
fileList := make([]ArchivedFile, 0, len(zr.File))
for _, f := range zr.File {
info := f.FileInfo()
modTime := info.ModTime()
fileList = append(fileList, ArchivedFile{
Name: util.FormSlash(f.Name),
Size: info.Size(),
UpdatedAt: &modTime,
IsDirectory: info.IsDir(),
})
}
return fileList, nil
}
func getArchiveListCacheKey(entity int, encoding string) string {
return fmt.Sprintf("archive_list_%d_%s", entity, encoding)
}
pkg/filemanager/manager/entity.go
+412
View File
@@ -0,0 +1,412 @@
package manager
import (
"context"
"crypto/sha1"
"encoding/hex"
"fmt"
"time"
"github.com/cloudreve/Cloudreve/v4/ent"
"github.com/cloudreve/Cloudreve/v4/inventory/types"
"github.com/cloudreve/Cloudreve/v4/pkg/cluster/routes"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/driver"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/fs"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/fs/dbfs"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/manager/entitysource"
"github.com/cloudreve/Cloudreve/v4/pkg/hashid"
"github.com/cloudreve/Cloudreve/v4/pkg/serializer"
"github.com/gofrs/uuid"
)
type (
EntityManagement interface {
// GetEntityUrls gets download urls of given entities, return URLs and the earliest expiry time
GetEntityUrls(ctx context.Context, args []GetEntityUrlArgs, opts ...fs.Option) ([]EntityUrl, *time.Time, error)
// GetUrlForRedirectedDirectLink gets redirected direct download link of given direct link
GetUrlForRedirectedDirectLink(ctx context.Context, dl *ent.DirectLink, opts ...fs.Option) (string, *time.Time, error)
// GetDirectLink gets permanent direct download link of given files
GetDirectLink(ctx context.Context, urls ...*fs.URI) ([]DirectLink, error)
// GetEntitySource gets source of given entity
GetEntitySource(ctx context.Context, entityID int, opts ...fs.Option) (entitysource.EntitySource, error)
// Thumbnail gets thumbnail entity of given file
Thumbnail(ctx context.Context, uri *fs.URI) (entitysource.EntitySource, error)
// SubmitAndAwaitThumbnailTask submits a thumbnail task and waits for result
SubmitAndAwaitThumbnailTask(ctx context.Context, uri *fs.URI, ext string, entity fs.Entity) (fs.Entity, error)
// SetCurrentVersion sets current version of given file
SetCurrentVersion(ctx context.Context, path *fs.URI, version int) error
// DeleteVersion deletes a version of given file
DeleteVersion(ctx context.Context, path *fs.URI, version int) error
// ExtractAndSaveMediaMeta extracts and saves media meta into file metadata of given file.
ExtractAndSaveMediaMeta(ctx context.Context, uri *fs.URI, entityID int) error
// RecycleEntities recycles a group of entities
RecycleEntities(ctx context.Context, force bool, entityIDs ...int) error
// ListPhysical lists physical files in a path
ListPhysical(ctx context.Context, path string, policyID int, recursive bool, progress driver.ListProgressFunc) ([]fs.PhysicalObject, error)
// ImportPhysical imports a physical file to a Cloudreve file
ImportPhysical(ctx context.Context, dst *fs.URI, policyId int, src fs.PhysicalObject, completeHook bool) error
}
DirectLink struct {
File fs.File
Url string
}
EntityUrl struct {
Url string `json:"url"`
BrowserDownloadDisplayName string `json:"stream_saver_display_name,omitempty"`
}
)
func (m *manager) GetDirectLink(ctx context.Context, urls ...*fs.URI) ([]DirectLink, error) {
ae := serializer.NewAggregateError()
res := make([]DirectLink, 0, len(urls))
useRedirect := m.user.Edges.Group.Settings.RedirectedSource
fileClient := m.dep.FileClient()
siteUrl := m.settings.SiteURL(ctx)
for _, url := range urls {
file, err := m.fs.Get(
ctx, url,
dbfs.WithFileEntities(),
dbfs.WithRequiredCapabilities(dbfs.NavigatorCapabilityDownloadFile),
)
if err != nil {
ae.Add(url.String(), err)
continue
}
if file.OwnerID() != m.user.ID && !m.user.Edges.Group.Permissions.Enabled(int(types.GroupPermissionIsAdmin)) {
ae.Add(url.String(), fs.ErrOwnerOnly)
continue
}
if file.Type() != types.FileTypeFile {
ae.Add(url.String(), fs.ErrEntityNotExist)
continue
}
target := file.PrimaryEntity()
if target == nil {
ae.Add(url.String(), fs.ErrEntityNotExist)
continue
}
// Hooks for entity download
if err := m.fs.ExecuteNavigatorHooks(ctx, fs.HookTypeBeforeDownload, file); err != nil {
m.l.Warning("Failed to execute navigator hooks: %s", err)
}
if useRedirect {
reuseExisting := !m.user.Edges.Group.Permissions.Enabled(int(types.GroupPermissionUniqueRedirectDirectLink))
// Use redirect source
link, err := fileClient.CreateDirectLink(ctx, file.ID(), file.Name(), m.user.Edges.Group.SpeedLimit, reuseExisting)
if err != nil {
ae.Add(url.String(), err)
continue
}
linkHashID := hashid.EncodeSourceLinkID(m.hasher, link.ID)
res = append(res, DirectLink{
File: file,
Url: routes.MasterDirectLink(siteUrl, linkHashID, link.Name).String(),
})
} else {
// Use direct source
policy, d, err := m.getEntityPolicyDriver(ctx, target, nil)
if err != nil {
ae.Add(url.String(), err)
continue
}
source := entitysource.NewEntitySource(target, d, policy, m.auth, m.settings, m.hasher, m.dep.RequestClient(),
m.l, m.config, m.dep.MimeDetector(ctx), m.dep.EncryptorFactory(ctx))
sourceUrl, err := source.Url(ctx,
entitysource.WithSpeedLimit(int64(m.user.Edges.Group.SpeedLimit)),
entitysource.WithDisplayName(file.Name()),
)
if err != nil {
ae.Add(url.String(), err)
continue
}
res = append(res, DirectLink{
File: file,
Url: sourceUrl.Url,
})
}
}
return res, ae.Aggregate()
}
func (m *manager) GetUrlForRedirectedDirectLink(ctx context.Context, dl *ent.DirectLink, opts ...fs.Option) (string, *time.Time, error) {
o := newOption()
for _, opt := range opts {
opt.Apply(o)
}
file, err := m.fs.GetFileFromDirectLink(ctx, dl)
if err != nil {
return "", nil, err
}
// Find primary entity
primaryEntity := file.PrimaryEntity()
// Generate url
var (
res string
expire *time.Time
)
// Try to read from cache.
cacheKey := entityUrlCacheKey(primaryEntity.ID(), int64(dl.Speed), dl.Name, o.IsDownload,
m.settings.SiteURL(ctx).String())
if cached, ok := m.kv.Get(cacheKey); ok {
cachedItem := cached.(EntityUrlCache)
res = cachedItem.Url
expire = cachedItem.ExpireAt
} else {
// Cache miss, Generate new url
policy, d, err := m.getEntityPolicyDriver(ctx, primaryEntity, nil)
if err != nil {
return "", nil, err
}
source := entitysource.NewEntitySource(primaryEntity, d, policy, m.auth, m.settings, m.hasher, m.dep.RequestClient(),
m.l, m.config, m.dep.MimeDetector(ctx), m.dep.EncryptorFactory(ctx))
downloadUrl, err := source.Url(ctx,
entitysource.WithExpire(o.Expire),
entitysource.WithDownload(o.IsDownload),
entitysource.WithSpeedLimit(int64(dl.Speed)),
entitysource.WithDisplayName(dl.Name),
)
if err != nil {
return "", nil, err
}
// Save into kv
cacheValidDuration := expireTimeToTTL(o.Expire) - m.settings.EntityUrlCacheMargin(ctx)
if cacheValidDuration > 0 {
m.kv.Set(cacheKey, EntityUrlCache{
Url: downloadUrl.Url,
ExpireAt: downloadUrl.ExpireAt,
}, cacheValidDuration)
}
res = downloadUrl.Url
expire = downloadUrl.ExpireAt
}
return res, expire, nil
}
func (m *manager) GetEntityUrls(ctx context.Context, args []GetEntityUrlArgs, opts ...fs.Option) ([]EntityUrl, *time.Time, error) {
o := newOption()
for _, opt := range opts {
opt.Apply(o)
}
var earliestExpireAt *time.Time
res := make([]EntityUrl, len(args))
ae := serializer.NewAggregateError()
for i, arg := range args {
file, err := m.fs.Get(
ctx, arg.URI,
dbfs.WithFileEntities(),
dbfs.WithRequiredCapabilities(dbfs.NavigatorCapabilityDownloadFile),
)
if err != nil {
ae.Add(arg.URI.String(), err)
continue
}
if file.Type() != types.FileTypeFile {
ae.Add(arg.URI.String(), fs.ErrEntityNotExist)
continue
}
var (
target fs.Entity
found bool
)
if arg.PreferredEntityID != "" {
found, target = fs.FindDesiredEntity(file, arg.PreferredEntityID, m.hasher, nil)
if !found {
ae.Add(arg.URI.String(), fs.ErrEntityNotExist)
continue
}
} else {
// No preferred entity ID, use the primary version entity
target = file.PrimaryEntity()
if target == nil {
ae.Add(arg.URI.String(), fs.ErrEntityNotExist)
continue
}
}
// Hooks for entity download
if err := m.fs.ExecuteNavigatorHooks(ctx, fs.HookTypeBeforeDownload, file); err != nil {
m.l.Warning("Failed to execute navigator hooks: %s", err)
}
policy, d, err := m.getEntityPolicyDriver(ctx, target, nil)
if err != nil {
ae.Add(arg.URI.String(), err)
continue
}
// Try to read from cache.
cacheKey := entityUrlCacheKey(target.ID(), o.DownloadSpeed, getEntityDisplayName(file, target), o.IsDownload,
m.settings.SiteURL(ctx).String())
if cached, ok := m.kv.Get(cacheKey); ok && !o.NoCache {
cachedItem := cached.(EntityUrlCache)
// Find the earliest expiry time
if cachedItem.ExpireAt != nil && (earliestExpireAt == nil || cachedItem.ExpireAt.Before(*earliestExpireAt)) {
earliestExpireAt = cachedItem.ExpireAt
}
res[i] = EntityUrl{
Url: cachedItem.Url,
BrowserDownloadDisplayName: cachedItem.BrowserDownloadDisplayName,
}
continue
}
// Cache miss, Generate new url
source := entitysource.NewEntitySource(target, d, policy, m.auth, m.settings, m.hasher, m.dep.RequestClient(),
m.l, m.config, m.dep.MimeDetector(ctx), m.dep.EncryptorFactory(ctx))
downloadUrl, err := source.Url(ctx,
entitysource.WithExpire(o.Expire),
entitysource.WithDownload(o.IsDownload),
entitysource.WithSpeedLimit(o.DownloadSpeed),
entitysource.WithDisplayName(getEntityDisplayName(file, target)),
)
if err != nil {
ae.Add(arg.URI.String(), err)
continue
}
// Find the earliest expiry time
if downloadUrl.ExpireAt != nil && (earliestExpireAt == nil || downloadUrl.ExpireAt.Before(*earliestExpireAt)) {
earliestExpireAt = downloadUrl.ExpireAt
}
// Save into kv
cacheValidDuration := expireTimeToTTL(o.Expire) - m.settings.EntityUrlCacheMargin(ctx)
if cacheValidDuration > 0 {
m.kv.Set(cacheKey, EntityUrlCache{
Url: downloadUrl.Url,
ExpireAt: downloadUrl.ExpireAt,
}, cacheValidDuration)
}
res[i] = EntityUrl{
Url: downloadUrl.Url,
}
if d.Capabilities().BrowserRelayedDownload {
res[i].BrowserDownloadDisplayName = getEntityDisplayName(file, target)
}
}
return res, earliestExpireAt, ae.Aggregate()
}
func (m *manager) GetEntitySource(ctx context.Context, entityID int, opts ...fs.Option) (entitysource.EntitySource, error) {
o := newOption()
for _, opt := range opts {
opt.Apply(o)
}
var (
entity fs.Entity
err error
)
if o.Entity != nil {
entity = o.Entity
} else {
entity, err = m.fs.GetEntity(ctx, entityID)
if err != nil {
return nil, err
}
if entity.ReferenceCount() == 0 {
return nil, fs.ErrEntityNotExist
}
}
policy, handler, err := m.getEntityPolicyDriver(ctx, entity, o.Policy)
if err != nil {
return nil, err
}
return entitysource.NewEntitySource(entity, handler, policy, m.auth, m.settings, m.hasher, m.dep.RequestClient(), m.l,
m.config, m.dep.MimeDetector(ctx), m.dep.EncryptorFactory(ctx), entitysource.WithContext(ctx), entitysource.WithThumb(o.IsThumb)), nil
}
func (l *manager) SetCurrentVersion(ctx context.Context, path *fs.URI, version int) error {
indexDiff, err := l.fs.VersionControl(ctx, path, version, false)
l.processIndexDiff(ctx, indexDiff)
return err
}
func (l *manager) DeleteVersion(ctx context.Context, path *fs.URI, version int) error {
_, err := l.fs.VersionControl(ctx, path, version, true)
return err
}
func (l *manager) ListPhysical(ctx context.Context, path string, policyID int, recursive bool, progress driver.ListProgressFunc) ([]fs.PhysicalObject, error) {
policy, err := l.dep.StoragePolicyClient().GetPolicyByID(ctx, policyID)
if err != nil {
return nil, err
}
driver, err := l.GetStorageDriver(ctx, policy)
if err != nil {
return nil, err
}
return driver.List(ctx, path, progress, recursive)
}
func (l *manager) ImportPhysical(ctx context.Context, dst *fs.URI, policyId int, src fs.PhysicalObject, completeHook bool) error {
targetUri := dst.Join(src.RelativePath)
req := &fs.UploadRequest{
Props: &fs.UploadProps{
Uri: targetUri,
UploadSessionID: uuid.Must(uuid.NewV4()).String(),
Size: src.Size,
PreferredStoragePolicy: policyId,
SavePath: src.Source,
LastModified: &src.LastModify,
},
ImportFrom: &src,
}
// Prepare for upload
uploadSession, err := l.fs.PrepareUpload(ctx, req)
if err != nil {
return fmt.Errorf("faield to prepare uplaod: %w", err)
}
if completeHook {
d, err := l.GetStorageDriver(ctx, l.CastStoragePolicyOnSlave(ctx, uploadSession.Policy))
if err != nil {
return err
}
l.onNewEntityUploaded(ctx, uploadSession, d, l.user.ID)
}
return nil
}
func entityUrlCacheKey(id int, speed int64, displayName string, download bool, siteUrl string) string {
hash := sha1.New()
hash.Write([]byte(fmt.Sprintf("%d_%d_%s_%t_%s", id,
speed, displayName, download, siteUrl)))
hashRes := hex.EncodeToString(hash.Sum(nil))
return fmt.Sprintf("%s_%s", EntityUrlCacheKeyPrefix, hashRes)
}
pkg/filemanager/manager/entitysource/entitysource.go
+1123
View File
File diff suppressed because it is too large Load Diff
pkg/filemanager/manager/fs.go
+117
View File
@@ -0,0 +1,117 @@
package manager
import (
"context"
"github.com/cloudreve/Cloudreve/v4/ent"
"github.com/cloudreve/Cloudreve/v4/inventory/types"
"github.com/cloudreve/Cloudreve/v4/pkg/cluster"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/driver"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/driver/cos"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/driver/ks3"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/driver/local"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/driver/obs"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/driver/onedrive"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/driver/oss"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/driver/qiniu"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/driver/remote"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/driver/s3"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/driver/upyun"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/fs"
"github.com/cloudreve/Cloudreve/v4/pkg/serializer"
)
func (m *manager) LocalDriver(policy *ent.StoragePolicy) driver.Handler {
if policy == nil {
policy = &ent.StoragePolicy{Type: types.PolicyTypeLocal, Settings: &types.PolicySetting{}}
}
return local.New(policy, m.l, m.config)
}
func (m *manager) CastStoragePolicyOnSlave(ctx context.Context, policy *ent.StoragePolicy) *ent.StoragePolicy {
if !m.stateless {
return policy
}
nodeId := cluster.NodeIdFromContext(ctx)
if policy.Type == types.PolicyTypeRemote {
if nodeId != policy.NodeID {
return policy
}
policyCopy := *policy
policyCopy.Type = types.PolicyTypeLocal
return &policyCopy
} else if policy.Type == types.PolicyTypeLocal {
policyCopy := *policy
policyCopy.NodeID = nodeId
policyCopy.Type = types.PolicyTypeRemote
policyCopy.SetNode(&ent.Node{
ID: nodeId,
Server: cluster.MasterSiteUrlFromContext(ctx),
SlaveKey: m.config.Slave().Secret,
})
return &policyCopy
} else if policy.Type == types.PolicyTypeOss {
policyCopy := *policy
if policyCopy.Settings != nil {
policyCopy.Settings.ServerSideEndpoint = ""
}
}
return policy
}
func (m *manager) GetStorageDriver(ctx context.Context, policy *ent.StoragePolicy) (driver.Handler, error) {
switch policy.Type {
case types.PolicyTypeLocal:
return local.New(policy, m.l, m.config), nil
case types.PolicyTypeRemote:
return remote.New(ctx, policy, m.settings, m.config, m.l)
case types.PolicyTypeOss:
return oss.New(ctx, policy, m.settings, m.config, m.l, m.dep.MimeDetector(ctx))
case types.PolicyTypeCos:
return cos.New(ctx, policy, m.settings, m.config, m.l, m.dep.MimeDetector(ctx))
case types.PolicyTypeS3:
return s3.New(ctx, policy, m.settings, m.config, m.l, m.dep.MimeDetector(ctx))
case types.PolicyTypeKs3:
return ks3.New(ctx, policy, m.settings, m.config, m.l, m.dep.MimeDetector(ctx))
case types.PolicyTypeObs:
return obs.New(ctx, policy, m.settings, m.config, m.l, m.dep.MimeDetector(ctx))
case types.PolicyTypeQiniu:
return qiniu.New(ctx, policy, m.settings, m.config, m.l, m.dep.MimeDetector(ctx))
case types.PolicyTypeUpyun:
return upyun.New(ctx, policy, m.settings, m.config, m.l, m.dep.MimeDetector(ctx))
case types.PolicyTypeOd:
return onedrive.New(ctx, policy, m.settings, m.config, m.l, m.dep.CredManager())
default:
return nil, ErrUnknownPolicyType
}
}
func (m *manager) getEntityPolicyDriver(cxt context.Context, e fs.Entity, policyOverwrite *ent.StoragePolicy) (*ent.StoragePolicy, driver.Handler, error) {
policyID := e.PolicyID()
var (
policy *ent.StoragePolicy
err error
)
if policyID == 0 {
policy = &ent.StoragePolicy{Type: types.PolicyTypeLocal, Settings: &types.PolicySetting{}}
} else {
if policyOverwrite != nil && policyOverwrite.ID == policyID {
policy = policyOverwrite
} else {
policy, err = m.policyClient.GetPolicyByID(cxt, e.PolicyID())
if err != nil {
return nil, nil, serializer.NewError(serializer.CodeDBError, "failed to get policy", err)
}
}
}
d, err := m.GetStorageDriver(cxt, policy)
if err != nil {
return nil, nil, err
}
return policy, d, nil
}
pkg/filemanager/manager/fulltextindex.go
+540
View File
@@ -0,0 +1,540 @@
package manager
import (
"context"
"encoding/json"
"fmt"
"github.com/cloudreve/Cloudreve/v4/application/dependency"
"github.com/cloudreve/Cloudreve/v4/ent"
"github.com/cloudreve/Cloudreve/v4/ent/task"
"github.com/cloudreve/Cloudreve/v4/inventory"
"github.com/cloudreve/Cloudreve/v4/inventory/types"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/fs"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/fs/dbfs"
"github.com/cloudreve/Cloudreve/v4/pkg/hashid"
"github.com/cloudreve/Cloudreve/v4/pkg/logging"
"github.com/cloudreve/Cloudreve/v4/pkg/queue"
"github.com/cloudreve/Cloudreve/v4/pkg/searcher"
"github.com/cloudreve/Cloudreve/v4/pkg/util"
"github.com/samber/lo"
)
type (
FullTextIndexTask struct {
*queue.DBTask
}
FullTextIndexTaskState struct {
Uri *fs.URI `json:"uri"`
EntityID int `json:"entity_id"`
FileID int `json:"file_id"`
OwnerID int `json:"owner_id"`
}
ftsFileInfo struct {
FileID int
OwnerID int
EntityID int
FileName string
}
)
func (m *manager) SearchFullText(ctx context.Context, query string, offset int) (*FullTextSearchResults, error) {
indexer := m.dep.SearchIndexer(ctx)
results, total, err := indexer.Search(ctx, m.user.ID, query, offset)
if err != nil {
return nil, fmt.Errorf("failed to search full text: %w", err)
}
if len(results) == 0 {
// No results.
return &FullTextSearchResults{}, nil
}
// Traverse each file in result
files := lo.FilterMap(results, func(result searcher.SearchResult, _ int) (FullTextSearchResult, bool) {
file, err := m.TraverseFile(ctx, result.FileID)
if err != nil {
m.l.Debug("Failed to traverse file %d for full text search: %s, skipping.", result.FileID, err)
return FullTextSearchResult{}, false
}
if file.InTrashBin() {
return FullTextSearchResult{}, false
}
return FullTextSearchResult{
File: file,
Content: result.Text,
}, true
})
if len(files) == 0 {
// No valid files, run next offset
return m.SearchFullText(ctx, query, offset+len(results))
}
return &FullTextSearchResults{
Hits: files,
Total: total,
}, nil
}
func init() {
queue.RegisterResumableTaskFactory(queue.FullTextIndexTaskType, NewFullTextIndexTaskFromModel)
queue.RegisterResumableTaskFactory(queue.FullTextCopyTaskType, NewFullTextCopyTaskFromModel)
queue.RegisterResumableTaskFactory(queue.FullTextChangeOwnerTaskType, NewFullTextChangeOwnerTaskFromModel)
queue.RegisterResumableTaskFactory(queue.FullTextDeleteTaskType, NewFullTextDeleteTaskFromModel)
}
func NewFullTextIndexTask(ctx context.Context, uri *fs.URI, entityID, fileID, ownerID int, creator *ent.User) (*FullTextIndexTask, error) {
state := &FullTextIndexTaskState{
Uri: uri,
EntityID: entityID,
FileID: fileID,
OwnerID: ownerID,
}
stateBytes, err := json.Marshal(state)
if err != nil {
return nil, fmt.Errorf("failed to marshal state: %w", err)
}
return &FullTextIndexTask{
DBTask: &queue.DBTask{
DirectOwner: creator,
Task: &ent.Task{
Type: queue.FullTextIndexTaskType,
CorrelationID: logging.CorrelationID(ctx),
PrivateState: string(stateBytes),
PublicState: &types.TaskPublicState{},
},
},
}, nil
}
func NewFullTextIndexTaskFromModel(t *ent.Task) queue.Task {
return &FullTextIndexTask{
DBTask: &queue.DBTask{
Task: t,
},
}
}
type (
FullTextCopyTask struct {
*queue.DBTask
}
FullTextCopyTaskState struct {
Uri *fs.URI `json:"uri"`
OriginalFileID int `json:"original_file_id"`
FileID int `json:"file_id"`
OwnerID int `json:"owner_id"`
EntityID int `json:"entity_id"`
}
)
func NewFullTextCopyTask(ctx context.Context, uri *fs.URI, originalFileID, fileID, ownerID, entityID int, creator *ent.User) (*FullTextCopyTask, error) {
state := &FullTextCopyTaskState{
Uri: uri,
OriginalFileID: originalFileID,
FileID: fileID,
OwnerID: ownerID,
EntityID: entityID,
}
stateBytes, err := json.Marshal(state)
if err != nil {
return nil, fmt.Errorf("failed to marshal state: %w", err)
}
return &FullTextCopyTask{
DBTask: &queue.DBTask{
DirectOwner: creator,
Task: &ent.Task{
Type: queue.FullTextCopyTaskType,
CorrelationID: logging.CorrelationID(ctx),
PrivateState: string(stateBytes),
PublicState: &types.TaskPublicState{},
},
},
}, nil
}
func NewFullTextCopyTaskFromModel(t *ent.Task) queue.Task {
return &FullTextCopyTask{
DBTask: &queue.DBTask{
Task: t,
},
}
}
func (t *FullTextCopyTask) Do(ctx context.Context) (task.Status, error) {
dep := dependency.FromContext(ctx)
l := dep.Logger()
fm := NewFileManager(dep, inventory.UserFromContext(ctx)).(*manager)
if !fm.settings.FTSEnabled(ctx) {
l.Debug("FTS disabled, skipping full text copy task.")
return task.StatusCompleted, nil
}
var state FullTextCopyTaskState
if err := json.Unmarshal([]byte(t.State()), &state); err != nil {
return task.StatusError, fmt.Errorf("failed to unmarshal state: %s (%w)", err, queue.CriticalErr)
}
// Get fresh file to make sure task is not stale.
file, err := fm.Get(ctx, state.Uri, dbfs.WithFilePublicMetadata())
if err != nil {
return task.StatusError, fmt.Errorf("failed to get latest file: %w", err)
}
if file.PrimaryEntityID() != state.EntityID {
l.Debug("File %d entity changed, skipping copy index.", state.FileID)
return task.StatusCompleted, nil
}
indexer := dep.SearchIndexer(ctx)
if err := indexer.CopyByFileID(ctx, state.OriginalFileID, state.FileID, state.OwnerID, state.EntityID); err != nil {
l.Warning("Failed to copy index from file %d to %d, falling back to full indexing: %s", state.OriginalFileID, state.FileID, err)
return performIndexing(ctx, fm, state.Uri, state.EntityID, state.FileID, state.OwnerID, file.Name(), false)
}
// Patch metadata to mark file as indexed.
if err := fm.fs.PatchMetadata(ctx, []*fs.URI{state.Uri}, fs.MetadataPatch{
Key: dbfs.FullTextIndexKey,
Value: hashid.EncodeEntityID(fm.hasher, state.EntityID),
}); err != nil {
return task.StatusError, fmt.Errorf("failed to patch metadata: %w", err)
}
l.Debug("Successfully copied index from file %d to %d.", state.OriginalFileID, state.FileID)
return task.StatusCompleted, nil
}
type (
FullTextChangeOwnerTask struct {
*queue.DBTask
}
FullTextChangeOwnerTaskState struct {
Uri *fs.URI `json:"uri"`
EntityID int `json:"entity_id"`
FileID int `json:"file_id"`
OriginalOwnerID int `json:"original_owner_id"`
NewOwnerID int `json:"new_owner_id"`
}
)
func NewFullTextChangeOwnerTask(ctx context.Context, uri *fs.URI, entityID, fileID, originalOwnerID, newOwnerID int, creator *ent.User) (*FullTextChangeOwnerTask, error) {
state := &FullTextChangeOwnerTaskState{
Uri: uri,
EntityID: entityID,
FileID: fileID,
OriginalOwnerID: originalOwnerID,
NewOwnerID: newOwnerID,
}
stateBytes, err := json.Marshal(state)
if err != nil {
return nil, fmt.Errorf("failed to marshal state: %w", err)
}
return &FullTextChangeOwnerTask{
DBTask: &queue.DBTask{
DirectOwner: creator,
Task: &ent.Task{
Type: queue.FullTextChangeOwnerTaskType,
CorrelationID: logging.CorrelationID(ctx),
PrivateState: string(stateBytes),
PublicState: &types.TaskPublicState{},
},
},
}, nil
}
func NewFullTextChangeOwnerTaskFromModel(t *ent.Task) queue.Task {
return &FullTextChangeOwnerTask{
DBTask: &queue.DBTask{
Task: t,
},
}
}
func (t *FullTextChangeOwnerTask) Do(ctx context.Context) (task.Status, error) {
dep := dependency.FromContext(ctx)
l := dep.Logger()
fm := NewFileManager(dep, inventory.UserFromContext(ctx)).(*manager)
if !fm.settings.FTSEnabled(ctx) {
l.Debug("FTS disabled, skipping full text change owner task.")
return task.StatusCompleted, nil
}
var state FullTextChangeOwnerTaskState
if err := json.Unmarshal([]byte(t.State()), &state); err != nil {
return task.StatusError, fmt.Errorf("failed to unmarshal state: %s (%w)", err, queue.CriticalErr)
}
// Get fresh file to make sure task is not stale.
file, err := fm.Get(ctx, state.Uri, dbfs.WithFilePublicMetadata())
if err != nil {
return task.StatusError, fmt.Errorf("failed to get latest file: %w", err)
}
if file.PrimaryEntityID() != state.EntityID {
l.Debug("File %d entity changed, skipping owner change.", state.FileID)
return task.StatusCompleted, nil
}
indexer := dep.SearchIndexer(ctx)
if err := indexer.ChangeOwner(ctx, state.FileID, state.OriginalOwnerID, state.NewOwnerID); err != nil {
return task.StatusError, fmt.Errorf("failed to change owner for file %d: %w", state.FileID, err)
}
l.Debug("Successfully changed index owner for file %d from %d to %d.", state.FileID, state.OriginalOwnerID, state.NewOwnerID)
return task.StatusCompleted, nil
}
type (
FullTextDeleteTask struct {
*queue.DBTask
}
FullTextDeleteTaskState struct {
FileIDs []int `json:"file_ids"`
}
)
func NewFullTextDeleteTask(ctx context.Context, fileIDs []int, creator *ent.User) (*FullTextDeleteTask, error) {
state := &FullTextDeleteTaskState{
FileIDs: fileIDs,
}
stateBytes, err := json.Marshal(state)
if err != nil {
return nil, fmt.Errorf("failed to marshal state: %w", err)
}
return &FullTextDeleteTask{
DBTask: &queue.DBTask{
DirectOwner: creator,
Task: &ent.Task{
Type: queue.FullTextDeleteTaskType,
CorrelationID: logging.CorrelationID(ctx),
PrivateState: string(stateBytes),
PublicState: &types.TaskPublicState{},
},
},
}, nil
}
func NewFullTextDeleteTaskFromModel(t *ent.Task) queue.Task {
return &FullTextDeleteTask{
DBTask: &queue.DBTask{
Task: t,
},
}
}
func (t *FullTextDeleteTask) Do(ctx context.Context) (task.Status, error) {
dep := dependency.FromContext(ctx)
l := dep.Logger()
var state FullTextDeleteTaskState
if err := json.Unmarshal([]byte(t.State()), &state); err != nil {
return task.StatusError, fmt.Errorf("failed to unmarshal state: %s (%w)", err, queue.CriticalErr)
}
indexer := dep.SearchIndexer(ctx)
if err := indexer.DeleteByFileIDs(ctx, state.FileIDs...); err != nil {
return task.StatusError, fmt.Errorf("failed to delete index for %d file(s): %w", len(state.FileIDs), err)
}
l.Debug("Successfully deleted index for %d file(s).", len(state.FileIDs))
return task.StatusCompleted, nil
}
func (t *FullTextIndexTask) Do(ctx context.Context) (task.Status, error) {
dep := dependency.FromContext(ctx)
l := dep.Logger()
fm := NewFileManager(dep, inventory.UserFromContext(ctx)).(*manager)
// Check FTS enabled
if !fm.settings.FTSEnabled(ctx) {
l.Debug("FTS disabled, skipping full text index task.")
return task.StatusCompleted, nil
}
// Unmarshal state
var state FullTextIndexTaskState
if err := json.Unmarshal([]byte(t.State()), &state); err != nil {
return task.StatusError, fmt.Errorf("failed to unmarshal state: %s (%w)", err, queue.CriticalErr)
}
// Get fresh file to make sure task is not stale
file, err := fm.Get(ctx, state.Uri, dbfs.WithFilePublicMetadata())
if err != nil {
return task.StatusError, fmt.Errorf("failed to get latest file: %w", err)
}
if file.PrimaryEntityID() != state.EntityID {
l.Debug("File %d is not the latest version, skipping indexing.", state.FileID)
return task.StatusCompleted, nil
}
deleteOldChunks := false
if _, ok := file.Metadata()[dbfs.FullTextIndexKey]; ok {
deleteOldChunks = true
}
return performIndexing(ctx, fm, state.Uri, state.EntityID, state.FileID, state.OwnerID, state.Uri.Name(), deleteOldChunks)
}
// performIndexing extracts text from the entity and indexes it. This is shared between
// the regular index task and the copy task (as a fallback when copy fails).
func performIndexing(ctx context.Context, fm *manager, uri *fs.URI, entityID, fileID, ownerID int, fileName string, deleteOldChunks bool) (task.Status, error) {
dep := dependency.FromContext(ctx)
l := dep.Logger()
// Get entity source
source, err := fm.GetEntitySource(ctx, entityID)
if err != nil {
return task.StatusError, fmt.Errorf("failed to get entity source: %w", err)
}
defer source.Close()
// Extract text
var text string
if source.Entity().Size() > 0 {
extractor := dep.TextExtractor(ctx)
text, err = extractor.Extract(ctx, source)
if err != nil {
l.Warning("Failed to extract text for file %d: %s", fileID, err)
return task.StatusCompleted, nil
}
}
indexer := dep.SearchIndexer(ctx)
// Delete old chunks first so that stale chunks from a previously longer
// version of the file are removed before upserting the new (possibly fewer)
// chunks.
if deleteOldChunks {
if err := indexer.DeleteByFileIDs(ctx, fileID); err != nil {
l.Warning("Failed to delete old index chunks for file %d: %s", fileID, err)
}
}
// Index via SearchIndexer
if err := indexer.IndexFile(ctx, ownerID, fileID, entityID, fileName, text); err != nil {
return task.StatusError, fmt.Errorf("failed to index file %d: %w", fileID, err)
}
// Upsert metadata
if err := fm.fs.PatchMetadata(ctx, []*fs.URI{uri}, fs.MetadataPatch{
Key: dbfs.FullTextIndexKey,
Value: hashid.EncodeEntityID(fm.hasher, entityID),
}); err != nil {
return task.StatusError, fmt.Errorf("failed to patch metadata: %w", err)
}
l.Debug("Successfully indexed file %d for owner %d.", fileID, ownerID)
return task.StatusCompleted, nil
}
// ShouldExtractText checks if a file is eligible for text extraction based on
// the extractor's supported extensions and max file size. This is exported for
// use by the rebuild index workflow.
func ShouldExtractText(extractor searcher.TextExtractor, fileName string, size int64) bool {
return util.IsInExtensionList(extractor.Exts(), fileName) && extractor.MaxFileSize() > size
}
// shouldIndexFullText checks if a file should be indexed for full-text search.
func (m *manager) shouldIndexFullText(ctx context.Context, fileName string, size int64) bool {
if !m.settings.FTSEnabled(ctx) {
return false
}
extractor := m.dep.TextExtractor(ctx)
return ShouldExtractText(extractor, fileName, size)
}
// fullTextIndexForNewEntity creates and queues a full text index task for a newly uploaded entity.
func (m *manager) fullTextIndexForNewEntity(ctx context.Context, session *fs.UploadSession, owner int) {
if session.Props.EntityType != nil && *session.Props.EntityType != types.EntityTypeVersion {
return
}
if !m.shouldIndexFullText(ctx, session.Props.Uri.Name(), session.Props.Size) {
return
}
t, err := NewFullTextIndexTask(ctx, session.Props.Uri, session.EntityID, session.FileID, owner, m.user)
if err != nil {
m.l.Warning("Failed to create full text index task: %s", err)
return
}
if err := m.dep.MediaMetaQueue(ctx).QueueTask(ctx, t); err != nil {
m.l.Warning("Failed to queue full text index task: %s", err)
}
}
func (m *manager) processIndexDiff(ctx context.Context, diff *fs.IndexDiff) {
if diff == nil {
return
}
for _, update := range diff.IndexToUpdate {
t, err := NewFullTextIndexTask(ctx, &update.Uri, update.EntityID, update.FileID, update.OwnerID, m.user)
if err != nil {
m.l.Warning("Failed to create full text update task: %s", err)
continue
}
if err := m.dep.MediaMetaQueue(ctx).QueueTask(ctx, t); err != nil {
m.l.Warning("Failed to queue full text update task: %s", err)
}
}
for _, cp := range diff.IndexToCopy {
t, err := NewFullTextCopyTask(ctx, &cp.Uri, cp.OriginalFileID, cp.FileID, cp.OwnerID, cp.EntityID, m.user)
if err != nil {
m.l.Warning("Failed to create full text copy task: %s", err)
continue
}
if err := m.dep.MediaMetaQueue(ctx).QueueTask(ctx, t); err != nil {
m.l.Warning("Failed to queue full text copy task: %s", err)
}
}
for _, change := range diff.IndexToChangeOwner {
t, err := NewFullTextChangeOwnerTask(ctx, &change.Uri, change.EntityID, change.FileID, change.OriginalOwnerID, change.NewOwnerID, m.user)
if err != nil {
m.l.Warning("Failed to create full text change owner task: %s", err)
continue
}
if err := m.dep.MediaMetaQueue(ctx).QueueTask(ctx, t); err != nil {
m.l.Warning("Failed to queue full text change owner task: %s", err)
}
}
if len(diff.IndexToDelete) > 0 && m.dep.SettingProvider().FTSEnabled(ctx) {
t, err := NewFullTextDeleteTask(ctx, diff.IndexToDelete, m.user)
if err != nil {
m.l.Warning("Failed to create full text delete task: %s", err)
return
}
if err := m.dep.MediaMetaQueue(ctx).QueueTask(ctx, t); err != nil {
m.l.Warning("Failed to queue full text delete task: %s", err)
}
}
ctx = context.WithoutCancel(ctx)
indexer := m.dep.SearchIndexer(ctx)
go func() {
for _, rename := range diff.IndexToRename {
if err := indexer.Rename(ctx, rename.FileID, rename.EntityID, rename.Uri.Name()); err != nil {
m.l.Warning("Failed to rename index for file %d: %s", rename.FileID, err)
}
}
}()
}
pkg/filemanager/manager/manager.go
+196
View File
@@ -0,0 +1,196 @@
package manager
import (
"context"
"io"
"time"
"github.com/cloudreve/Cloudreve/v4/application/dependency"
"github.com/cloudreve/Cloudreve/v4/ent"
"github.com/cloudreve/Cloudreve/v4/inventory"
"github.com/cloudreve/Cloudreve/v4/inventory/types"
"github.com/cloudreve/Cloudreve/v4/pkg/auth"
"github.com/cloudreve/Cloudreve/v4/pkg/cache"
"github.com/cloudreve/Cloudreve/v4/pkg/conf"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/driver"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/fs"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/fs/dbfs"
"github.com/cloudreve/Cloudreve/v4/pkg/hashid"
"github.com/cloudreve/Cloudreve/v4/pkg/logging"
"github.com/cloudreve/Cloudreve/v4/pkg/serializer"
"github.com/cloudreve/Cloudreve/v4/pkg/setting"
)
var (
ErrUnknownPolicyType = serializer.NewError(serializer.CodeInternalSetting, "Unknown policy type", nil)
)
const (
UploadSessionCachePrefix = "callback_"
// Ctx key for upload session
UploadSessionCtx = "uploadSession"
)
type (
FileOperation interface {
// Get gets file object by given path
Get(ctx context.Context, path *fs.URI, opts ...fs.Option) (fs.File, error)
// List lists files under given path
List(ctx context.Context, path *fs.URI, args *ListArgs) (fs.File, *fs.ListFileResult, error)
// Create creates a file or directory
Create(ctx context.Context, path *fs.URI, fileType types.FileType, opt ...fs.Option) (fs.File, error)
// Rename renames a file or directory
Rename(ctx context.Context, path *fs.URI, newName string) (fs.File, error)
// Delete deletes a group of file or directory. UnlinkOnly indicates whether to delete file record in DB only.
Delete(ctx context.Context, path []*fs.URI, opts ...fs.Option) error
// EmptyTrash hard-deletes every top-level item in the current user's trash bin.
EmptyTrash(ctx context.Context) error
// Restore restores a group of files
Restore(ctx context.Context, path ...*fs.URI) error
// MoveOrCopy moves or copies a group of files
MoveOrCopy(ctx context.Context, src []*fs.URI, dst *fs.URI, isCopy bool) error
// Update puts file content. If given file does not exist, it will create a new one.
Update(ctx context.Context, req *fs.UploadRequest, opts ...fs.Option) (fs.File, error)
// Walk walks through given path
Walk(ctx context.Context, path *fs.URI, depth int, f fs.WalkFunc, opts ...fs.Option) error
// UpsertMedata update or insert metadata of given file
PatchMedata(ctx context.Context, path []*fs.URI, data ...fs.MetadataPatch) error
// CreateViewerSession creates a viewer session for given file
CreateViewerSession(ctx context.Context, uri *fs.URI, version string, viewer *types.Viewer) (*ViewerSession, error)
// TraverseFile traverses a file to its root file, return the file with linked root.
TraverseFile(ctx context.Context, fileID int) (fs.File, error)
// SearchFullText searches full text for given query and offset
SearchFullText(ctx context.Context, query string, offset int) (*FullTextSearchResults, error)
}
FsManagement interface {
// SharedAddressTranslation translates shared symbolic address to real address. If path does not exist,
// most recent existing parent directory will be returned.
SharedAddressTranslation(ctx context.Context, path *fs.URI, opts ...fs.Option) (fs.File, *fs.URI, error)
// Capacity gets capacity of current file system
Capacity(ctx context.Context) (*fs.Capacity, error)
// CheckIfCapacityExceeded checks if given user's capacity exceeded, and send notification email
CheckIfCapacityExceeded(ctx context.Context) error
// LocalDriver gets local driver for operating local files.
LocalDriver(policy *ent.StoragePolicy) driver.Handler
// CastStoragePolicyOnSlave check if given storage policy need to be casted to another.
// It is used on slave node, when local policy need to cast to remote policy;
// Remote policy with same node ID can be casted to local policy.
CastStoragePolicyOnSlave(ctx context.Context, policy *ent.StoragePolicy) *ent.StoragePolicy
// GetStorageDriver gets storage driver for given policy
GetStorageDriver(ctx context.Context, policy *ent.StoragePolicy) (driver.Handler, error)
// PatchView patches the view setting of a file
PatchView(ctx context.Context, uri *fs.URI, view *types.ExplorerView) error
}
ShareManagement interface {
// CreateShare creates a share link for given path
CreateOrUpdateShare(ctx context.Context, path *fs.URI, args *CreateShareArgs) (*ent.Share, error)
}
Archiver interface {
// CreateArchive creates an archive
CreateArchive(ctx context.Context, uris []*fs.URI, writer io.Writer, opts ...fs.Option) (int, error)
// ListArchiveFiles lists files in an archive
ListArchiveFiles(ctx context.Context, uri *fs.URI, entity, zipEncoding string) ([]ArchivedFile, error)
}
FileManager interface {
fs.LockSystem
FileOperation
EntityManagement
UploadManagement
FsManagement
ShareManagement
Archiver
// Recycle reset current FileManager object and put back to resource pool
Recycle()
}
// GetEntityUrlArgs single args to get entity url
GetEntityUrlArgs struct {
URI *fs.URI
PreferredEntityID string
}
// CreateShareArgs args to create share link
CreateShareArgs struct {
ExistedShareID int
IsPrivate bool
Password string
RemainDownloads int
Expire *time.Time
ShareView bool
ShowReadMe bool
}
FullTextSearchResults struct {
Hits []FullTextSearchResult
Total int64
}
FullTextSearchResult struct {
File fs.File
Content string
}
)
type manager struct {
user *ent.User
l logging.Logger
fs fs.FileSystem
settings setting.Provider
kv cache.Driver
config conf.ConfigProvider
stateless bool
auth auth.Auth
hasher hashid.Encoder
policyClient inventory.StoragePolicyClient
dep dependency.Dep
}
func NewFileManager(dep dependency.Dep, u *ent.User) FileManager {
config := dep.ConfigProvider()
if config.System().Mode == conf.SlaveMode || u == nil {
return newStatelessFileManager(dep)
}
return &manager{
l: dep.Logger(),
user: u,
settings: dep.SettingProvider(),
fs: dbfs.NewDatabaseFS(u, dep.FileClient(), dep.ShareClient(), dep.Logger(), dep.LockSystem(),
dep.SettingProvider(), dep.StoragePolicyClient(), dep.HashIDEncoder(), dep.UserClient(), dep.KV(), dep.NavigatorStateKV(),
dep.DirectLinkClient(), dep.EncryptorFactory(context.TODO()), dep.EventHub()),
kv: dep.KV(),
config: config,
auth: dep.GeneralAuth(),
hasher: dep.HashIDEncoder(),
policyClient: dep.StoragePolicyClient(),
dep: dep,
}
}
func newStatelessFileManager(dep dependency.Dep) FileManager {
return &manager{
l: dep.Logger(),
settings: dep.SettingProvider(),
kv: dep.KV(),
config: dep.ConfigProvider(),
stateless: true,
auth: dep.GeneralAuth(),
dep: dep,
hasher: dep.HashIDEncoder(),
}
}
func (m *manager) Recycle() {
if m.fs != nil {
m.fs.Recycle()
}
}
func newOption() *fs.FsOption {
return &fs.FsOption{}
}
pkg/filemanager/manager/mediameta.go
+199
View File
@@ -0,0 +1,199 @@
package manager
import (
"context"
"encoding/json"
"fmt"
"github.com/cloudreve/Cloudreve/v4/application/dependency"
"github.com/cloudreve/Cloudreve/v4/ent"
"github.com/cloudreve/Cloudreve/v4/ent/task"
"github.com/cloudreve/Cloudreve/v4/inventory"
"github.com/cloudreve/Cloudreve/v4/inventory/types"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/driver"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/fs"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/fs/dbfs"
"github.com/cloudreve/Cloudreve/v4/pkg/logging"
"github.com/cloudreve/Cloudreve/v4/pkg/mediameta"
"github.com/cloudreve/Cloudreve/v4/pkg/queue"
"github.com/cloudreve/Cloudreve/v4/pkg/util"
"github.com/samber/lo"
)
type (
MediaMetaTask struct {
*queue.DBTask
}
MediaMetaTaskState struct {
Uri *fs.URI `json:"uri"`
EntityID int `json:"entity_id"`
}
)
func init() {
queue.RegisterResumableTaskFactory(queue.MediaMetaTaskType, NewMediaMetaTaskFromModel)
}
// NewMediaMetaTask creates a new MediaMetaTask to
func NewMediaMetaTask(ctx context.Context, uri *fs.URI, entityID int, creator *ent.User) (*MediaMetaTask, error) {
state := &MediaMetaTaskState{
Uri: uri,
EntityID: entityID,
}
stateBytes, err := json.Marshal(state)
if err != nil {
return nil, fmt.Errorf("failed to marshal state: %w", err)
}
return &MediaMetaTask{
DBTask: &queue.DBTask{
DirectOwner: creator,
Task: &ent.Task{
Type: queue.MediaMetaTaskType,
CorrelationID: logging.CorrelationID(ctx),
PrivateState: string(stateBytes),
PublicState: &types.TaskPublicState{},
},
},
}, nil
}
func NewMediaMetaTaskFromModel(task *ent.Task) queue.Task {
return &MediaMetaTask{
DBTask: &queue.DBTask{
Task: task,
},
}
}
func (m *MediaMetaTask) Do(ctx context.Context) (task.Status, error) {
dep := dependency.FromContext(ctx)
fm := NewFileManager(dep, inventory.UserFromContext(ctx)).(*manager)
// unmarshal state
var state MediaMetaTaskState
if err := json.Unmarshal([]byte(m.State()), &state); err != nil {
return task.StatusError, fmt.Errorf("failed to unmarshal state: %s (%w)", err, queue.CriticalErr)
}
err := fm.ExtractAndSaveMediaMeta(ctx, state.Uri, state.EntityID)
if err != nil {
return task.StatusError, err
}
return task.StatusCompleted, nil
}
func (m *manager) ExtractAndSaveMediaMeta(ctx context.Context, uri *fs.URI, entityID int) error {
// 1. retrieve file info
file, err := m.fs.Get(ctx, uri, dbfs.WithFileEntities())
if err != nil {
return fmt.Errorf("failed to get file: %w", err)
}
versions := lo.Filter(file.Entities(), func(i fs.Entity, index int) bool {
return i.Type() == types.EntityTypeVersion
})
targetVersion, versionIndex, found := lo.FindIndexOf(versions, func(i fs.Entity) bool {
return i.ID() == entityID
})
if !found {
return fmt.Errorf("failed to find version: %s (%w)", err, queue.CriticalErr)
}
if versionIndex != 0 {
m.l.Debug("Skip media meta task for non-latest version.")
return nil
}
language := ""
if file.Owner().Settings != nil {
language = file.Owner().Settings.Language
}
var (
metas []driver.MediaMeta
)
// 2. try using native driver
_, d, err := m.getEntityPolicyDriver(ctx, targetVersion, nil)
if err != nil {
return fmt.Errorf("failed to get storage driver: %s (%w)", err, queue.CriticalErr)
}
driverCaps := d.Capabilities()
if util.IsInExtensionList(driverCaps.MediaMetaSupportedExts, file.Name()) {
m.l.Debug("Using native driver to generate media meta.")
metas, err = d.MediaMeta(ctx, targetVersion.Source(), file.Ext(), language)
if err != nil {
return fmt.Errorf("failed to get media meta using native driver: %w", err)
}
} else if driverCaps.MediaMetaProxy && util.IsInExtensionList(m.dep.MediaMetaExtractor(ctx).Exts(), file.Name()) {
m.l.Debug("Using local extractor to generate media meta.")
extractor := m.dep.MediaMetaExtractor(ctx)
source, err := m.GetEntitySource(ctx, targetVersion.ID())
defer source.Close()
if err != nil {
return fmt.Errorf("failed to get entity source: %w", err)
}
metas, err = extractor.Extract(ctx, file.Ext(), source, mediameta.WithLanguage(language))
if err != nil {
return fmt.Errorf("failed to extract media meta using local extractor: %w", err)
}
} else {
m.l.Debug("No available generator for media meta.")
return nil
}
m.l.Debug("%d media meta generated.", len(metas))
m.l.Debug("Media meta: %v", metas)
// 3. save meta
if len(metas) > 0 {
if err := m.fs.PatchMetadata(ctx, []*fs.URI{uri}, lo.Map(metas, func(i driver.MediaMeta, index int) fs.MetadataPatch {
return fs.MetadataPatch{
Key: fmt.Sprintf("%s:%s", i.Type, i.Key),
Value: i.Value,
}
})...); err != nil {
return fmt.Errorf("failed to save media meta: %s (%w)", err, queue.CriticalErr)
}
}
return nil
}
func (m *manager) shouldGenerateMediaMeta(ctx context.Context, d driver.Handler, fileName string) bool {
driverCaps := d.Capabilities()
if util.IsInExtensionList(driverCaps.MediaMetaSupportedExts, fileName) {
// Handler support it natively
return true
}
if driverCaps.MediaMetaProxy && util.IsInExtensionList(m.dep.MediaMetaExtractor(ctx).Exts(), fileName) {
// Handler does not support. but proxy is enabled.
return true
}
return false
}
func (m *manager) mediaMetaForNewEntity(ctx context.Context, session *fs.UploadSession, d driver.Handler) {
if session.Props.EntityType == nil || *session.Props.EntityType == types.EntityTypeVersion {
if !m.shouldGenerateMediaMeta(ctx, d, session.Props.Uri.Name()) {
return
}
mediaMetaTask, err := NewMediaMetaTask(ctx, session.Props.Uri, session.EntityID, m.user)
if err != nil {
m.l.Warning("Failed to create media meta task: %s", err)
return
}
if err := m.dep.MediaMetaQueue(ctx).QueueTask(ctx, mediaMetaTask); err != nil {
m.l.Warning("Failed to queue media meta task: %s", err)
}
return
}
}
pkg/filemanager/manager/metadata.go
+322
View File
@@ -0,0 +1,322 @@
package manager
import (
"context"
"crypto/sha1"
"encoding/json"
"fmt"
"strconv"
"strings"
"github.com/cloudreve/Cloudreve/v4/application/constants"
"github.com/cloudreve/Cloudreve/v4/application/dependency"
"github.com/cloudreve/Cloudreve/v4/inventory/types"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/fs"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/fs/dbfs"
"github.com/cloudreve/Cloudreve/v4/pkg/hashid"
"github.com/cloudreve/Cloudreve/v4/pkg/serializer"
"github.com/go-playground/validator/v10"
"github.com/samber/lo"
)
type (
metadataValidator func(ctx context.Context, m *manager, patch *fs.MetadataPatch) error
)
const (
wildcardMetadataKey = "*"
customizeMetadataSuffix = "customize"
tagMetadataSuffix = "tag"
customPropsMetadataSuffix = "props"
iconColorMetadataKey = customizeMetadataSuffix + ":icon_color"
emojiIconMetadataKey = customizeMetadataSuffix + ":emoji"
shareOwnerMetadataKey = dbfs.MetadataSysPrefix + "shared_owner"
shareRedirectMetadataKey = dbfs.MetadataSysPrefix + "shared_redirect"
)
var (
validate = validator.New()
lastEmojiHash = ""
emojiPresets = map[string]struct{}{}
// validateColor validates a color value
validateColor = func(optional bool) metadataValidator {
return func(ctx context.Context, m *manager, patch *fs.MetadataPatch) error {
patch.UpdateModifiedAt = true
if patch.Remove {
return nil
}
tag := "omitempty,iscolor"
if !optional {
tag = "required,iscolor"
}
res := validate.Var(patch.Value, tag)
if res != nil {
return fmt.Errorf("invalid color: %w", res)
}
return nil
}
}
validators = map[string]map[string]metadataValidator{
"sys": {
wildcardMetadataKey: func(ctx context.Context, m *manager, patch *fs.MetadataPatch) error {
if patch.Remove {
return fmt.Errorf("cannot remove system metadata")
}
patch.UpdateModifiedAt = true
dep := dependency.FromContext(ctx)
// Validate share owner is valid hashid
if patch.Key == shareOwnerMetadataKey {
hasher := dep.HashIDEncoder()
_, err := hasher.Decode(patch.Value, hashid.UserID)
if err != nil {
return fmt.Errorf("invalid share owner: %w", err)
}
return nil
}
// Validate share redirect uri is valid share uri
if patch.Key == shareRedirectMetadataKey {
uri, err := fs.NewUriFromString(patch.Value)
if err != nil || uri.FileSystem() != constants.FileSystemShare {
return fmt.Errorf("invalid redirect uri: %w", err)
}
return nil
}
return fmt.Errorf("unsupported system metadata key: %s", patch.Key)
},
},
"dav": {},
// Allow manipulating thumbnail metadata via public PatchMetadata API
"thumb": {
// Only supported thumb metadata currently is thumb:disabled
dbfs.ThumbDisabledKey: func(ctx context.Context, m *manager, patch *fs.MetadataPatch) error {
// Presence of this key disables thumbnails; value is ignored.
// We allow both setting and removing this key.
return nil
},
},
customizeMetadataSuffix: {
iconColorMetadataKey: validateColor(false),
emojiIconMetadataKey: func(ctx context.Context, m *manager, patch *fs.MetadataPatch) error {
patch.UpdateModifiedAt = true
if patch.Remove {
return nil
}
// Validate if patched emoji is within preset list.
emojis := m.settings.EmojiPresets(ctx)
current := fmt.Sprintf("%x", (sha1.Sum([]byte(emojis))))
if current != lastEmojiHash {
presets := make(map[string][]string)
if err := json.Unmarshal([]byte(emojis), &presets); err != nil {
return fmt.Errorf("failed to read emoji setting: %w", err)
}
emojiPresets = make(map[string]struct{})
for _, v := range presets {
for _, emoji := range v {
emojiPresets[emoji] = struct{}{}
}
}
}
if _, ok := emojiPresets[patch.Value]; !ok {
return fmt.Errorf("unsupported emoji")
}
return nil
},
},
tagMetadataSuffix: {
wildcardMetadataKey: func(ctx context.Context, m *manager, patch *fs.MetadataPatch) error {
patch.UpdateModifiedAt = true
if err := validateColor(true)(ctx, m, patch); err != nil {
return err
}
if patch.Key == tagMetadataSuffix+":" {
return fmt.Errorf("invalid metadata key")
}
return nil
},
},
customPropsMetadataSuffix: {
wildcardMetadataKey: func(ctx context.Context, m *manager, patch *fs.MetadataPatch) error {
patch.UpdateModifiedAt = true
if patch.Remove {
return nil
}
customProps := m.settings.CustomProps(ctx)
propId := strings.TrimPrefix(patch.Key, customPropsMetadataSuffix+":")
for _, prop := range customProps {
if prop.ID == propId {
switch prop.Type {
case types.CustomPropsTypeText:
if prop.Min > 0 && prop.Min > len(patch.Value) {
return fmt.Errorf("value is too short")
}
if prop.Max > 0 && prop.Max < len(patch.Value) {
return fmt.Errorf("value is too long")
}
return nil
case types.CustomPropsTypeRating:
if patch.Value == "" {
return nil
}
// validate the value is a number
rating, err := strconv.Atoi(patch.Value)
if err != nil {
return fmt.Errorf("value is not a number")
}
if prop.Max < rating {
return fmt.Errorf("value is too large")
}
return nil
case types.CustomPropsTypeNumber:
if patch.Value == "" {
return nil
}
value, err := strconv.Atoi(patch.Value)
if err != nil {
return fmt.Errorf("value is not a number")
}
if prop.Min > value {
return fmt.Errorf("value is too small")
}
if prop.Max > 0 && prop.Max < value {
return fmt.Errorf("value is too large")
}
return nil
case types.CustomPropsTypeBoolean:
if patch.Value == "" {
return nil
}
if patch.Value != "true" && patch.Value != "false" {
return fmt.Errorf("value is not a boolean")
}
return nil
case types.CustomPropsTypeSelect:
if patch.Value == "" {
return nil
}
for _, option := range prop.Options {
if option == patch.Value {
return nil
}
}
return fmt.Errorf("invalid option")
case types.CustomPropsTypeMultiSelect:
if patch.Value == "" {
return nil
}
var values []string
if err := json.Unmarshal([]byte(patch.Value), &values); err != nil {
return fmt.Errorf("invalid multi select value: %w", err)
}
// make sure all values are in the options
for _, value := range values {
if !lo.Contains(prop.Options, value) {
return fmt.Errorf("invalid option")
}
}
return nil
case types.CustomPropsTypeLink:
if patch.Value == "" {
return nil
}
if prop.Min > 0 && len(patch.Value) < prop.Min {
return fmt.Errorf("value is too small")
}
if prop.Max > 0 && len(patch.Value) > prop.Max {
return fmt.Errorf("value is too large")
}
return nil
default:
return nil
}
}
}
return fmt.Errorf("unkown custom props")
},
},
}
)
func (m *manager) PatchMedata(ctx context.Context, path []*fs.URI, data ...fs.MetadataPatch) error {
data, err := m.validateMetadata(ctx, data...)
if err != nil {
return err
}
return m.fs.PatchMetadata(ctx, path, data...)
}
func (m *manager) validateMetadata(ctx context.Context, data ...fs.MetadataPatch) ([]fs.MetadataPatch, error) {
validated := make([]fs.MetadataPatch, 0, len(data))
for _, patch := range data {
category := strings.Split(patch.Key, ":")
if len(category) < 2 {
return validated, serializer.NewError(serializer.CodeParamErr, "Invalid metadata key", nil)
}
categoryValidators, ok := validators[category[0]]
if !ok {
return validated, serializer.NewError(serializer.CodeParamErr, "Invalid metadata key",
fmt.Errorf("unknown category: %s", category[0]))
}
// Explicit validators
if v, ok := categoryValidators[patch.Key]; ok {
if err := v(ctx, m, &patch); err != nil {
return validated, serializer.NewError(serializer.CodeParamErr, "Invalid metadata patch", err)
}
}
// Wildcard validators
if v, ok := categoryValidators[wildcardMetadataKey]; ok {
if err := v(ctx, m, &patch); err != nil {
return validated, serializer.NewError(serializer.CodeParamErr, "Invalid metadata patch", err)
}
}
validated = append(validated, patch)
}
return validated, nil
}
pkg/filemanager/manager/operation.go
+400
View File
@@ -0,0 +1,400 @@
package manager
import (
"context"
"encoding/gob"
"fmt"
"strings"
"time"
"github.com/cloudreve/Cloudreve/v4/application/constants"
"github.com/cloudreve/Cloudreve/v4/ent"
"github.com/cloudreve/Cloudreve/v4/inventory"
"github.com/cloudreve/Cloudreve/v4/inventory/types"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/fs"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/fs/dbfs"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/lock"
"github.com/cloudreve/Cloudreve/v4/pkg/serializer"
"github.com/cloudreve/Cloudreve/v4/pkg/setting"
"github.com/cloudreve/Cloudreve/v4/pkg/util"
"github.com/samber/lo"
)
const (
EntityUrlCacheKeyPrefix = "entity_url_"
DownloadSentinelCachePrefix = "download_sentinel_"
)
type (
ListArgs struct {
Page int
PageSize int
PageToken string
Order string
OrderDirection string
// StreamResponseCallback is used for streamed list operation, e.g. searching files.
// Whenever a new item is found, this callback will be called with the current item and the parent item.
StreamResponseCallback func(fs.File, []fs.File)
}
EntityUrlCache struct {
Url string
BrowserDownloadDisplayName string
ExpireAt *time.Time
}
)
func init() {
gob.Register(EntityUrlCache{})
}
func (m *manager) Get(ctx context.Context, path *fs.URI, opts ...fs.Option) (fs.File, error) {
return m.fs.Get(ctx, path, opts...)
}
func (m *manager) List(ctx context.Context, path *fs.URI, args *ListArgs) (fs.File, *fs.ListFileResult, error) {
dbfsSetting := m.settings.DBFS(ctx)
opts := []fs.Option{
fs.WithPageSize(args.PageSize),
fs.WithOrderBy(args.Order),
fs.WithOrderDirection(args.OrderDirection),
dbfs.WithFilePublicMetadata(),
dbfs.WithContextHint(),
dbfs.WithFileShareIfOwned(),
}
searchParams := path.SearchParameters()
if searchParams != nil {
if dbfsSetting.UseSSEForSearch {
opts = append(opts, dbfs.WithStreamListResponseCallback(args.StreamResponseCallback))
}
if searchParams.Category != "" {
// Overwrite search query with predefined category
category := fs.SearchCategoryFromString(searchParams.Category)
if category == setting.CategoryUnknown {
return nil, nil, fmt.Errorf("unknown category: %s", searchParams.Category)
}
path = path.SetQuery(m.settings.SearchCategoryQuery(ctx, category))
searchParams = path.SearchParameters()
}
}
if dbfsSetting.UseCursorPagination || searchParams != nil {
opts = append(opts, dbfs.WithCursorPagination(args.PageToken))
} else {
opts = append(opts, fs.WithPage(args.Page))
}
return m.fs.List(ctx, path, opts...)
}
func (m *manager) SharedAddressTranslation(ctx context.Context, path *fs.URI, opts ...fs.Option) (fs.File, *fs.URI, error) {
o := newOption()
for _, opt := range opts {
opt.Apply(o)
}
return m.fs.SharedAddressTranslation(ctx, path)
}
func (m *manager) Create(ctx context.Context, path *fs.URI, fileType types.FileType, opts ...fs.Option) (fs.File, error) {
o := newOption()
for _, opt := range opts {
opt.Apply(o)
}
if m.stateless {
return nil, o.Node.CreateFile(ctx, &fs.StatelessCreateFileService{
Path: path.String(),
Type: fileType,
UserID: o.StatelessUserID,
})
}
isSymbolic := false
if o.Metadata != nil {
_, err := m.validateMetadata(ctx, lo.MapToSlice(o.Metadata, func(key string, value string) fs.MetadataPatch {
if key == shareRedirectMetadataKey {
isSymbolic = true
}
return fs.MetadataPatch{
Key: key,
Value: value,
}
})...)
if err != nil {
return nil, err
}
}
if isSymbolic {
opts = append(opts, dbfs.WithSymbolicLink())
}
return m.fs.Create(ctx, path, fileType, opts...)
}
func (m *manager) Rename(ctx context.Context, path *fs.URI, newName string) (fs.File, error) {
file, indexDiff, err := m.fs.Rename(ctx, path, newName)
m.processIndexDiff(ctx, indexDiff)
return file, err
}
func (m *manager) MoveOrCopy(ctx context.Context, src []*fs.URI, dst *fs.URI, isCopy bool) error {
indexDiff, err := m.fs.MoveOrCopy(ctx, src, dst, isCopy)
if err != nil {
return err
}
m.processIndexDiff(ctx, indexDiff)
return nil
}
func (m *manager) SoftDelete(ctx context.Context, path ...*fs.URI) error {
return m.fs.SoftDelete(ctx, path...)
}
// EmptyTrash hard-deletes every top-level item in the current user's trash bin.
// It iterates through all pages of the trash listing and performs the deletion in
// batches. Returns nil if the trash is already empty.
func (m *manager) EmptyTrash(ctx context.Context) error {
trashUri, err := fs.NewUriFromString(fmt.Sprintf("%s://%s", constants.CloudreveScheme, constants.FileSystemTrash))
if err != nil {
return fmt.Errorf("failed to build trash uri: %w", err)
}
dbfsSetting := m.settings.DBFS(ctx)
pageSize := dbfsSetting.MaxPageSize
for {
_, listRes, err := m.fs.List(ctx, trashUri, fs.WithPageSize(pageSize))
if err != nil {
return fmt.Errorf("failed to list trash: %w", err)
}
if len(listRes.Files) == 0 {
return nil
}
uris := make([]*fs.URI, 0, len(listRes.Files))
for _, f := range listRes.Files {
uris = append(uris, f.Uri(false))
}
if err := m.Delete(ctx, uris, fs.WithSysSkipSoftDelete(true)); err != nil {
return fmt.Errorf("failed to delete trash files: %w", err)
}
// Stop when there are no more pages.
if listRes.Pagination == nil ||
(listRes.Pagination.IsCursor && listRes.Pagination.NextPageToken == "") ||
(!listRes.Pagination.IsCursor && len(listRes.Files) < pageSize) {
return nil
}
}
}
func (m *manager) Delete(ctx context.Context, path []*fs.URI, opts ...fs.Option) error {
o := newOption()
for _, opt := range opts {
opt.Apply(o)
}
if !o.SkipSoftDelete && !o.SysSkipSoftDelete {
return m.SoftDelete(ctx, path...)
}
staleEntities, indexDiff, err := m.fs.Delete(ctx, path, fs.WithUnlinkOnly(o.UnlinkOnly), fs.WithSysSkipSoftDelete(o.SysSkipSoftDelete))
if err != nil {
return err
}
m.l.Debug("New stale entities: %v", staleEntities)
// Delete stale entities
if len(staleEntities) > 0 {
t, err := newExplicitEntityRecycleTask(ctx, lo.Map(staleEntities, func(entity fs.Entity, index int) int {
return entity.ID()
}))
if err != nil {
return fmt.Errorf("failed to create explicit entity recycle task: %w", err)
}
if err := m.dep.EntityRecycleQueue(ctx).QueueTask(ctx, t); err != nil {
return fmt.Errorf("failed to queue explicit entity recycle task: %w", err)
}
}
// Process index diff
if indexDiff != nil {
m.processIndexDiff(ctx, indexDiff)
}
return nil
}
func (m *manager) Walk(ctx context.Context, path *fs.URI, depth int, f fs.WalkFunc, opts ...fs.Option) error {
return m.fs.Walk(ctx, path, depth, f, opts...)
}
func (m *manager) Capacity(ctx context.Context) (*fs.Capacity, error) {
res, err := m.fs.Capacity(ctx, m.user)
if err != nil {
return nil, err
}
return res, nil
}
func (m *manager) CheckIfCapacityExceeded(ctx context.Context) error {
ctx = context.WithValue(ctx, inventory.LoadUserGroup{}, true)
capacity, err := m.Capacity(ctx)
if err != nil {
return fmt.Errorf("failed to get user capacity: %w", err)
}
if capacity.Used <= capacity.Total {
return nil
}
return nil
}
func (l *manager) ConfirmLock(ctx context.Context, ancestor fs.File, uri *fs.URI, token ...string) (func(), fs.LockSession, error) {
return l.fs.ConfirmLock(ctx, ancestor, uri, token...)
}
func (l *manager) Lock(ctx context.Context, d time.Duration, requester *ent.User, zeroDepth bool,
application lock.Application, uri *fs.URI, token string) (fs.LockSession, error) {
return l.fs.Lock(ctx, d, requester, zeroDepth, application, uri, token)
}
func (l *manager) Unlock(ctx context.Context, tokens ...string) error {
return l.fs.Unlock(ctx, tokens...)
}
func (l *manager) Refresh(ctx context.Context, d time.Duration, token string) (lock.LockDetails, error) {
return l.fs.Refresh(ctx, d, token)
}
func (l *manager) Restore(ctx context.Context, path ...*fs.URI) error {
return l.fs.Restore(ctx, path...)
}
func (l *manager) CreateOrUpdateShare(ctx context.Context, path *fs.URI, args *CreateShareArgs) (*ent.Share, error) {
file, err := l.fs.Get(ctx, path, dbfs.WithRequiredCapabilities(dbfs.NavigatorCapabilityShare), dbfs.WithNotRoot())
if err != nil {
return nil, serializer.NewError(serializer.CodeNotFound, "src file not found", err)
}
// Only file owner can share file
if file.OwnerID() != l.user.ID {
return nil, serializer.NewError(serializer.CodeNoPermissionErr, "permission denied", nil)
}
if file.IsSymbolic() {
return nil, serializer.NewError(serializer.CodeNoPermissionErr, "cannot share symbolic file", nil)
}
var existed *ent.Share
shareClient := l.dep.ShareClient()
if args.ExistedShareID != 0 {
loadShareCtx := context.WithValue(ctx, inventory.LoadShareFile{}, true)
existed, err = shareClient.GetByID(loadShareCtx, args.ExistedShareID)
if err != nil {
return nil, serializer.NewError(serializer.CodeNotFound, "failed to get existed share", err)
}
if existed.Edges.File.ID != file.ID() {
return nil, serializer.NewError(serializer.CodeNotFound, "share link not found", nil)
}
}
password := ""
if args.IsPrivate {
password = args.Password
if strings.TrimSpace(password) == "" {
password = util.RandString(8, util.RandomLowerCases)
}
}
props := &types.ShareProps{
ShareView: args.ShareView,
ShowReadMe: args.ShowReadMe,
}
share, err := shareClient.Upsert(ctx, &inventory.CreateShareParams{
OwnerID: file.OwnerID(),
FileID: file.ID(),
Password: password,
Expires: args.Expire,
RemainDownloads: args.RemainDownloads,
Existed: existed,
Props: props,
})
if err != nil {
return nil, serializer.NewError(serializer.CodeDBError, "failed to create share", err)
}
return share, nil
}
func (m *manager) TraverseFile(ctx context.Context, fileID int) (fs.File, error) {
return m.fs.TraverseFile(ctx, fileID)
}
func (m *manager) PatchView(ctx context.Context, uri *fs.URI, view *types.ExplorerView) error {
if uri.PathTrimmed() == "" && uri.FileSystem() != constants.FileSystemMy && uri.FileSystem() != constants.FileSystemShare {
if m.user.Settings.FsViewMap == nil {
m.user.Settings.FsViewMap = make(map[string]types.ExplorerView)
}
if view == nil {
delete(m.user.Settings.FsViewMap, string(uri.FileSystem()))
} else {
m.user.Settings.FsViewMap[string(uri.FileSystem())] = *view
}
if err := m.dep.UserClient().SaveSettings(ctx, m.user); err != nil {
return serializer.NewError(serializer.CodeDBError, "failed to save user settings", err)
}
return nil
}
patch := &types.FileProps{
View: view,
}
isDelete := view == nil
if isDelete {
patch.View = &types.ExplorerView{}
}
if err := m.fs.PatchProps(ctx, uri, patch, isDelete); err != nil {
return err
}
return nil
}
func getEntityDisplayName(f fs.File, e fs.Entity) string {
switch e.Type() {
case types.EntityTypeThumbnail:
return fmt.Sprintf("%s_thumbnail", f.DisplayName())
case types.EntityTypeLivePhoto:
return fmt.Sprintf("%s_live_photo.mov", f.DisplayName())
default:
return f.Name()
}
}
func expireTimeToTTL(expireAt *time.Time) int {
if expireAt == nil {
return -1
}
return int(time.Until(*expireAt).Seconds())
}
pkg/filemanager/manager/recycle.go
+387
View File
@@ -0,0 +1,387 @@
package manager
import (
"context"
"encoding/json"
"errors"
"fmt"
"strconv"
"time"
"github.com/cloudreve/Cloudreve/v4/application/dependency"
"github.com/cloudreve/Cloudreve/v4/ent"
"github.com/cloudreve/Cloudreve/v4/ent/task"
"github.com/cloudreve/Cloudreve/v4/inventory"
"github.com/cloudreve/Cloudreve/v4/inventory/types"
"github.com/cloudreve/Cloudreve/v4/pkg/cache"
"github.com/cloudreve/Cloudreve/v4/pkg/crontab"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/fs"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/fs/dbfs"
"github.com/cloudreve/Cloudreve/v4/pkg/logging"
"github.com/cloudreve/Cloudreve/v4/pkg/queue"
"github.com/cloudreve/Cloudreve/v4/pkg/serializer"
"github.com/cloudreve/Cloudreve/v4/pkg/setting"
"github.com/samber/lo"
)
type (
ExplicitEntityRecycleTask struct {
*queue.DBTask
}
ExplicitEntityRecycleTaskState struct {
EntityIDs []int `json:"entity_ids,omitempty"`
Errors [][]RecycleError `json:"errors,omitempty"`
}
RecycleError struct {
ID string `json:"id"`
Error string `json:"error"`
}
)
func init() {
queue.RegisterResumableTaskFactory(queue.ExplicitEntityRecycleTaskType, NewExplicitEntityRecycleTaskFromModel)
queue.RegisterResumableTaskFactory(queue.EntityRecycleRoutineTaskType, NewEntityRecycleRoutineTaskFromModel)
crontab.Register(setting.CronTypeEntityCollect, func(ctx context.Context) {
dep := dependency.FromContext(ctx)
l := dep.Logger()
t, err := NewEntityRecycleRoutineTask(ctx)
if err != nil {
l.Error("Failed to create entity recycle routine task: %s", err)
}
if err := dep.EntityRecycleQueue(ctx).QueueTask(ctx, t); err != nil {
l.Error("Failed to queue entity recycle routine task: %s", err)
}
})
crontab.Register(setting.CronTypeTrashBinCollect, CronCollectTrashBin)
}
func NewExplicitEntityRecycleTaskFromModel(task *ent.Task) queue.Task {
return &ExplicitEntityRecycleTask{
DBTask: &queue.DBTask{
Task: task,
},
}
}
func newExplicitEntityRecycleTask(ctx context.Context, entities []int) (*ExplicitEntityRecycleTask, error) {
state := &ExplicitEntityRecycleTaskState{
EntityIDs: entities,
Errors: make([][]RecycleError, 0),
}
stateBytes, err := json.Marshal(state)
if err != nil {
return nil, fmt.Errorf("failed to marshal state: %w", err)
}
t := &ExplicitEntityRecycleTask{
DBTask: &queue.DBTask{
Task: &ent.Task{
Type: queue.ExplicitEntityRecycleTaskType,
CorrelationID: logging.CorrelationID(ctx),
PrivateState: string(stateBytes),
PublicState: &types.TaskPublicState{
ResumeTime: time.Now().Unix() - 1,
},
},
DirectOwner: inventory.UserFromContext(ctx),
},
}
return t, nil
}
func (m *ExplicitEntityRecycleTask) Do(ctx context.Context) (task.Status, error) {
dep := dependency.FromContext(ctx)
fm := NewFileManager(dep, inventory.UserFromContext(ctx)).(*manager)
// unmarshal state
state := &ExplicitEntityRecycleTaskState{}
if err := json.Unmarshal([]byte(m.State()), state); err != nil {
return task.StatusError, fmt.Errorf("failed to unmarshal state: %w", err)
}
// recycle entities
err := fm.RecycleEntities(ctx, false, state.EntityIDs...)
if err != nil {
appendAe(&state.Errors, err)
privateState, err := json.Marshal(state)
if err != nil {
return task.StatusError, fmt.Errorf("failed to marshal state: %w", err)
}
m.Task.PrivateState = string(privateState)
return task.StatusError, err
}
return task.StatusCompleted, nil
}
type (
EntityRecycleRoutineTask struct {
*queue.DBTask
}
EntityRecycleRoutineTaskState struct {
Errors [][]RecycleError `json:"errors,omitempty"`
}
)
func NewEntityRecycleRoutineTaskFromModel(task *ent.Task) queue.Task {
return &EntityRecycleRoutineTask{
DBTask: &queue.DBTask{
Task: task,
},
}
}
func NewEntityRecycleRoutineTask(ctx context.Context) (queue.Task, error) {
state := &EntityRecycleRoutineTaskState{
Errors: make([][]RecycleError, 0),
}
stateBytes, err := json.Marshal(state)
if err != nil {
return nil, fmt.Errorf("failed to marshal state: %w", err)
}
t := &EntityRecycleRoutineTask{
DBTask: &queue.DBTask{
Task: &ent.Task{
Type: queue.EntityRecycleRoutineTaskType,
CorrelationID: logging.CorrelationID(ctx),
PrivateState: string(stateBytes),
PublicState: &types.TaskPublicState{
ResumeTime: time.Now().Unix() - 1,
},
},
DirectOwner: inventory.UserFromContext(ctx),
},
}
return t, nil
}
func (m *EntityRecycleRoutineTask) Do(ctx context.Context) (task.Status, error) {
dep := dependency.FromContext(ctx)
fm := NewFileManager(dep, inventory.UserFromContext(ctx)).(*manager)
// unmarshal state
state := &EntityRecycleRoutineTaskState{}
if err := json.Unmarshal([]byte(m.State()), state); err != nil {
return task.StatusError, fmt.Errorf("failed to unmarshal state: %w", err)
}
// recycle entities
err := fm.RecycleEntities(ctx, false)
if err != nil {
appendAe(&state.Errors, err)
privateState, err := json.Marshal(state)
if err != nil {
return task.StatusError, fmt.Errorf("failed to marshal state: %w", err)
}
m.Task.PrivateState = string(privateState)
return task.StatusError, err
}
return task.StatusCompleted, nil
}
// RecycleEntities delete given entities. If the ID list is empty, it will walk through
// all stale entities in DB.
func (m *manager) RecycleEntities(ctx context.Context, force bool, entityIDs ...int) error {
ae := serializer.NewAggregateError()
entities, err := m.fs.StaleEntities(ctx, entityIDs...)
if err != nil {
return fmt.Errorf("failed to get entities: %w", err)
}
// Group entities by policy ID
entityGroup := lo.GroupBy(entities, func(entity fs.Entity) int {
return entity.PolicyID()
})
// Delete entity in each group in batch
for _, entities := range entityGroup {
entityChunk := lo.Chunk(entities, 100)
m.l.Info("Recycling %d entities in %d batches", len(entities), len(entityChunk))
for batch, chunk := range entityChunk {
m.l.Info("Start to recycle batch #%d, %d entities", batch, len(chunk))
mapSrcToId := make(map[string]int, len(chunk))
_, d, err := m.getEntityPolicyDriver(ctx, chunk[0], nil)
if err != nil {
for _, entity := range chunk {
ae.Add(strconv.Itoa(entity.ID()), err)
}
continue
}
for _, entity := range chunk {
mapSrcToId[entity.Source()] = entity.ID()
}
toBeDeletedSrc := lo.Map(lo.Filter(chunk, func(item fs.Entity, index int) bool {
// Only delete entities that are not marked as "unlink only"
return item.Model().Props == nil || !item.Model().Props.UnlinkOnly
}), func(entity fs.Entity, index int) string {
return entity.Source()
})
if len(toBeDeletedSrc) > 0 {
res, err := d.Delete(ctx, toBeDeletedSrc...)
if err != nil {
for _, src := range res {
ae.Add(strconv.Itoa(mapSrcToId[src]), err)
}
}
}
// Delete upload session if it's still valid
for _, entity := range chunk {
sid := entity.UploadSessionID()
if sid == nil {
continue
}
if session, ok := m.kv.Get(UploadSessionCachePrefix + sid.String()); ok {
session := session.(fs.UploadSession)
if err := d.CancelToken(ctx, &session); err != nil {
m.l.Warning("Failed to cancel upload session for %q: %s, this is expected if it's remote policy.", session.Props.Uri.String(), err)
}
_ = m.kv.Delete(UploadSessionCachePrefix, sid.String())
}
}
// Filtering out entities that are successfully deleted
rawAe := ae.Raw()
successEntities := lo.FilterMap(chunk, func(entity fs.Entity, index int) (int, bool) {
entityIdStr := fmt.Sprintf("%d", entity.ID())
_, ok := rawAe[entityIdStr]
if !ok {
// No error, deleted
return entity.ID(), true
}
if force {
ae.Remove(entityIdStr)
}
return entity.ID(), force
})
// Remove entities from DB
fc, tx, ctx, err := inventory.WithTx(ctx, m.dep.FileClient())
if err != nil {
return fmt.Errorf("failed to start transaction: %w", err)
}
storageReduced, err := fc.RemoveEntitiesByID(ctx, successEntities...)
if err != nil {
_ = inventory.Rollback(tx)
return fmt.Errorf("failed to remove entities from DB: %w", err)
}
tx.AppendStorageDiff(storageReduced)
if err := inventory.CommitWithStorageDiff(ctx, tx, m.l, m.dep.UserClient()); err != nil {
return fmt.Errorf("failed to commit delete change: %w", err)
}
}
}
return ae.Aggregate()
}
const (
MinimumTrashCollectBatch = 1000
)
// CronCollectTrashBin walks through all files in trash bin and delete them if they are expired.
func CronCollectTrashBin(ctx context.Context) {
dep := dependency.FromContext(ctx)
l := dep.Logger()
kv := dep.KV()
if memKv, ok := kv.(*cache.MemoStore); ok {
memKv.GarbageCollect(l)
}
fm := NewFileManager(dep, inventory.UserFromContext(ctx)).(*manager)
pageSize := dep.SettingProvider().DBFS(ctx).MaxPageSize
batch := 0
expiredFiles := make([]fs.File, 0)
for {
res, err := fm.fs.AllFilesInTrashBin(ctx, fs.WithPageSize(pageSize))
if err != nil {
l.Error("Failed to get files in trash bin: %s", err)
return
}
expired := lo.Filter(res.Files, func(file fs.File, index int) bool {
if expire, ok := file.Metadata()[dbfs.MetadataExpectedCollectTime]; ok {
expireUnix, err := strconv.ParseInt(expire, 10, 64)
if err != nil {
l.Warning("Failed to parse expected collect time %q: %s, will treat as expired", expire, err)
}
if expireUnix < time.Now().Unix() {
return true
}
}
return false
})
l.Info("Found %d files in trash bin pending collect, in batch #%d", len(res.Files), batch)
expiredFiles = append(expiredFiles, expired...)
if len(expiredFiles) >= MinimumTrashCollectBatch {
collectTrashBin(ctx, expiredFiles, dep, l)
expiredFiles = expiredFiles[:0]
}
if res.Pagination.NextPageToken == "" {
if len(expiredFiles) > 0 {
collectTrashBin(ctx, expiredFiles, dep, l)
}
break
}
batch++
}
}
func collectTrashBin(ctx context.Context, files []fs.File, dep dependency.Dep, l logging.Logger) {
l.Info("Start to collect %d files in trash bin", len(files))
uc := dep.UserClient()
// Group files by Owners
fileGroup := lo.GroupBy(files, func(file fs.File) int {
return file.OwnerID()
})
for uid, expiredFiles := range fileGroup {
ctx = context.WithValue(ctx, inventory.LoadUserGroup{}, true)
user, err := uc.GetByID(ctx, uid)
if err != nil {
l.Error("Failed to get user %d: %s", uid, err)
continue
}
ctx = context.WithValue(ctx, inventory.UserCtx{}, user)
fm := NewFileManager(dep, user).(*manager)
if err := fm.Delete(ctx, lo.Map(expiredFiles, func(file fs.File, index int) *fs.URI {
return file.Uri(false)
}), fs.WithSkipSoftDelete(true)); err != nil {
l.Error("Failed to delete files for user %d: %s", uid, err)
}
}
}
func appendAe(errs *[][]RecycleError, err error) {
var ae *serializer.AggregateError
*errs = append(*errs, make([]RecycleError, 0))
if errors.As(err, &ae) {
(*errs)[len(*errs)-1] = lo.MapToSlice(ae.Raw(), func(key string, value error) RecycleError {
return RecycleError{
ID: key,
Error: value.Error(),
}
})
}
}
pkg/filemanager/manager/thumbnail.go
+296
View File
@@ -0,0 +1,296 @@
package manager
import (
"context"
"errors"
"fmt"
"os"
"path"
"runtime"
"time"
"github.com/cloudreve/Cloudreve/v4/ent"
"github.com/cloudreve/Cloudreve/v4/ent/task"
"github.com/cloudreve/Cloudreve/v4/inventory/types"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/driver/local"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/fs"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/fs/dbfs"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/manager/entitysource"
"github.com/cloudreve/Cloudreve/v4/pkg/logging"
"github.com/cloudreve/Cloudreve/v4/pkg/queue"
"github.com/cloudreve/Cloudreve/v4/pkg/thumb"
"github.com/cloudreve/Cloudreve/v4/pkg/util"
"github.com/samber/lo"
)
// Thumbnail returns the thumbnail entity of the file.
func (m *manager) Thumbnail(ctx context.Context, uri *fs.URI) (entitysource.EntitySource, error) {
// retrieve file info
file, err := m.fs.Get(ctx, uri, dbfs.WithFileEntities(), dbfs.WithFilePublicMetadata())
if err != nil {
return nil, fmt.Errorf("failed to get file: %w", err)
}
// 0. Check if thumb is disabled in this file.
if _, ok := file.Metadata()[dbfs.ThumbDisabledKey]; ok || file.Type() != types.FileTypeFile {
return nil, fs.ErrEntityNotExist
}
// 1. If thumbnail entity exist, use it.
entities := file.Entities()
thumbEntity, found := lo.Find(entities, func(e fs.Entity) bool {
return e.Type() == types.EntityTypeThumbnail
})
if found {
thumbSource, err := m.GetEntitySource(ctx, 0, fs.WithEntity(thumbEntity))
if err != nil {
return nil, fmt.Errorf("failed to get entity source: %w", err)
}
thumbSource.Apply(entitysource.WithDisplayName(file.DisplayName() + ".jpg"))
return thumbSource, nil
}
latest := file.PrimaryEntity()
// If primary entity not exist, or it's empty
if latest == nil || latest.ID() == 0 {
return nil, fmt.Errorf("failed to get latest version")
}
// 2. Thumb entity not exist, try native policy generator
_, handler, err := m.getEntityPolicyDriver(ctx, latest, nil)
if err != nil {
return nil, fmt.Errorf("failed to get entity policy driver: %w", err)
}
capabilities := handler.Capabilities()
// Check if file extension and size is supported by native policy generator.
if capabilities.ThumbSupportAllExts || util.IsInExtensionList(capabilities.ThumbSupportedExts, file.DisplayName()) &&
(capabilities.ThumbMaxSize == 0 || latest.Size() <= capabilities.ThumbMaxSize) &&
!latest.Encrypted() {
thumbSource, err := m.GetEntitySource(ctx, 0, fs.WithEntity(latest), fs.WithUseThumb(true))
if err != nil {
return nil, fmt.Errorf("failed to get latest entity source: %w", err)
}
thumbSource.Apply(entitysource.WithDisplayName(file.DisplayName()))
return thumbSource, nil
} else if capabilities.ThumbProxy {
if err := m.fs.CheckCapability(ctx, uri,
dbfs.WithRequiredCapabilities(dbfs.NavigatorCapabilityGenerateThumb)); err != nil {
// Current FS does not support generate new thumb.
return nil, fs.ErrEntityNotExist
}
thumbEntity, err := m.SubmitAndAwaitThumbnailTask(ctx, uri, file.Ext(), latest)
if err != nil {
return nil, fmt.Errorf("failed to execute thumb task: %w", err)
}
thumbSource, err := m.GetEntitySource(ctx, 0, fs.WithEntity(thumbEntity))
if err != nil {
return nil, fmt.Errorf("failed to get entity source: %w", err)
}
return thumbSource, nil
} else {
// 4. If proxy generator not support, mark thumb as not available.
_ = disableThumb(ctx, m, uri)
}
return nil, fs.ErrEntityNotExist
}
func (m *manager) SubmitAndAwaitThumbnailTask(ctx context.Context, uri *fs.URI, ext string, entity fs.Entity) (fs.Entity, error) {
es, err := m.GetEntitySource(ctx, 0, fs.WithEntity(entity))
if err != nil {
return nil, fmt.Errorf("failed to get entity source: %w", err)
}
defer es.Close()
t := newGenerateThumbTask(ctx, m, uri, ext, es)
if err := m.dep.ThumbQueue(ctx).QueueTask(ctx, t); err != nil {
return nil, fmt.Errorf("failed to queue task: %w", err)
}
// Wait for task to finish
select {
case <-ctx.Done():
return nil, ctx.Err()
case res := <-t.sig:
if res.err != nil {
return nil, fmt.Errorf("failed to generate thumb: %w", res.err)
}
return res.thumbEntity, nil
}
}
func (m *manager) generateThumb(ctx context.Context, uri *fs.URI, ext string, es entitysource.EntitySource) (fs.Entity, error) {
// Generate thumb
pipeline := m.dep.ThumbPipeline()
res, err := pipeline.Generate(ctx, es, ext, nil)
if err != nil {
if res != nil && res.Path != "" {
_ = os.Remove(res.Path)
}
if !errors.Is(err, context.Canceled) && !m.stateless {
if err := disableThumb(ctx, m, uri); err != nil {
m.l.Warning("Failed to disable thumb: %v", err)
}
}
return nil, fmt.Errorf("failed to generate thumb: %w", err)
}
defer os.Remove(res.Path)
// Upload thumb entity
thumbFile, err := os.Open(res.Path)
if err != nil {
return nil, fmt.Errorf("failed to open temp thumb %q: %w", res.Path, err)
}
defer thumbFile.Close()
fileInfo, err := thumbFile.Stat()
if err != nil {
return nil, fmt.Errorf("failed to stat temp thumb %q: %w", res.Path, err)
}
var (
thumbEntity fs.Entity
)
if m.stateless {
_, d, err := m.getEntityPolicyDriver(ctx, es.Entity(), nil)
if err != nil {
return nil, fmt.Errorf("failed to get storage driver: %w", err)
}
savePath := es.Entity().Source() + m.settings.ThumbSlaveSidecarSuffix(ctx)
if err := d.Put(ctx, &fs.UploadRequest{
File: thumbFile,
Seeker: thumbFile,
Props: &fs.UploadProps{SavePath: savePath},
}); err != nil {
return nil, fmt.Errorf("failed to save thumb sidecar: %w", err)
}
thumbEntity, err = local.NewLocalFileEntity(types.EntityTypeThumbnail, savePath)
if err != nil {
return nil, fmt.Errorf("failed to create local thumb entity: %w", err)
}
} else {
entityType := types.EntityTypeThumbnail
req := &fs.UploadRequest{
Props: &fs.UploadProps{
Uri: uri,
Size: fileInfo.Size(),
SavePath: path.Clean(util.ReplaceMagicVar(m.settings.ThumbEntitySuffix(ctx), fs.Separator, true, true, time.Now(), m.user.ID, uri.Name(), uri.Path(), es.Entity().Source())),
MimeType: m.dep.MimeDetector(ctx).TypeByName("thumb.jpg"),
EntityType: &entityType,
},
File: thumbFile,
Seeker: thumbFile,
}
// Generating thumb can be triggered by users with read-only permission. We can bypass update permission check.
ctx = dbfs.WithBypassOwnerCheck(ctx)
file, err := m.Update(ctx, req, fs.WithEntityType(types.EntityTypeThumbnail))
if err != nil {
return nil, fmt.Errorf("failed to upload thumb entity: %w", err)
}
entities := file.Entities()
found := false
thumbEntity, found = lo.Find(entities, func(e fs.Entity) bool {
return e.Type() == types.EntityTypeThumbnail
})
if !found {
return nil, fmt.Errorf("failed to find thumb entity")
}
}
if m.settings.ThumbGCAfterGen(ctx) {
m.l.Debug("GC after thumb generation")
runtime.GC()
}
return thumbEntity, nil
}
type (
GenerateThumbTask struct {
*queue.InMemoryTask
es entitysource.EntitySource
ext string
m *manager
uri *fs.URI
sig chan *generateRes
}
generateRes struct {
thumbEntity fs.Entity
err error
}
)
func newGenerateThumbTask(ctx context.Context, m *manager, uri *fs.URI, ext string, es entitysource.EntitySource) *GenerateThumbTask {
t := &GenerateThumbTask{
InMemoryTask: &queue.InMemoryTask{
DBTask: &queue.DBTask{
Task: &ent.Task{
CorrelationID: logging.CorrelationID(ctx),
PublicState: &types.TaskPublicState{},
},
},
},
es: es,
ext: ext,
m: m,
uri: uri,
sig: make(chan *generateRes, 2),
}
t.InMemoryTask.DBTask.Task.SetUser(m.user)
return t
}
func (m *GenerateThumbTask) Do(ctx context.Context) (task.Status, error) {
// Make sure user does not cancel request before we start generating thumb.
select {
case <-ctx.Done():
err := ctx.Err()
return task.StatusError, err
default:
}
res, err := m.m.generateThumb(ctx, m.uri, m.ext, m.es)
if err != nil {
if errors.Is(err, thumb.ErrNotAvailable) {
m.sig <- &generateRes{nil, err}
return task.StatusCompleted, nil
}
return task.StatusError, err
}
m.sig <- &generateRes{res, nil}
return task.StatusCompleted, nil
}
func (m *GenerateThumbTask) OnError(err error, d time.Duration) {
m.InMemoryTask.OnError(err, d)
m.sig <- &generateRes{nil, err}
}
func disableThumb(ctx context.Context, m *manager, uri *fs.URI) error {
return m.fs.PatchMetadata(
dbfs.WithBypassOwnerCheck(ctx),
[]*fs.URI{uri}, fs.MetadataPatch{
Key: dbfs.ThumbDisabledKey,
Value: "",
Private: false,
})
}
pkg/filemanager/manager/upload.go
+643
View File
@@ -0,0 +1,643 @@
package manager
import (
"context"
"encoding/json"
"fmt"
"strconv"
"sync"
"time"
"github.com/cloudreve/Cloudreve/v4/application/dependency"
"github.com/cloudreve/Cloudreve/v4/ent"
"github.com/cloudreve/Cloudreve/v4/ent/task"
"github.com/cloudreve/Cloudreve/v4/inventory"
"github.com/cloudreve/Cloudreve/v4/inventory/types"
"github.com/cloudreve/Cloudreve/v4/pkg/cluster"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/driver"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/fs"
"github.com/cloudreve/Cloudreve/v4/pkg/logging"
"github.com/cloudreve/Cloudreve/v4/pkg/queue"
"github.com/cloudreve/Cloudreve/v4/pkg/serializer"
"github.com/gofrs/uuid"
"github.com/samber/lo"
)
type (
UploadManagement interface {
// CreateUploadSession creates a upload session for given upload request
CreateUploadSession(ctx context.Context, req *fs.UploadRequest, opts ...fs.Option) (*fs.UploadCredential, error)
// ConfirmUploadSession confirms whether upload session is valid for upload.
ConfirmUploadSession(ctx context.Context, session *fs.UploadSession, chunkIndex int) (fs.File, error)
// Upload uploads file data to storage
Upload(ctx context.Context, req *fs.UploadRequest, policy *ent.StoragePolicy, session *fs.UploadSession) error
// CompleteUpload completes upload session and returns file object
CompleteUpload(ctx context.Context, session *fs.UploadSession) (fs.File, error)
// CancelUploadSession cancels upload session
CancelUploadSession(ctx context.Context, path *fs.URI, sessionID string) error
// OnUploadFailed should be called when an unmanaged upload failed before complete.
OnUploadFailed(ctx context.Context, session *fs.UploadSession)
// Similar to CompleteUpload, but does not create actual uplaod session in storage.
PrepareUpload(ctx context.Context, req *fs.UploadRequest, opts ...fs.Option) (*fs.UploadSession, error)
// PreValidateUpload pre-validates an upload request.
PreValidateUpload(ctx context.Context, dst *fs.URI, files ...fs.PreValidateFile) error
// MarkChunkUploaded atomically records the given chunk index as received on the
// shared upload session and reports whether every chunk of the file has been
// received. Callers must invoke CompleteUpload only when allReceived is true.
// It is used to avoid triggering CompleteUpload prematurely when the client
// uploads chunks concurrently and the last-indexed chunk arrives before some
// earlier chunks are still in flight.
MarkChunkUploaded(ctx context.Context, session *fs.UploadSession, chunkIndex int) (allReceived bool, err error)
}
)
// uploadSessionLocks holds per-session mutexes used to serialize chunk-progress
// bookkeeping. Entries are removed on every terminal path (CompleteUpload,
// CancelUploadSession, OnUploadFailed) to keep the map bounded to the number of
// in-flight upload sessions on this process.
var uploadSessionLocks sync.Map // map[string]*sync.Mutex
func lockUploadSession(sessionID string) *sync.Mutex {
mu, _ := uploadSessionLocks.LoadOrStore(sessionID, &sync.Mutex{})
return mu.(*sync.Mutex)
}
func releaseUploadSessionLock(sessionID string) {
uploadSessionLocks.Delete(sessionID)
}
// expectedChunkCount returns the total number of chunks the client is expected
// to upload for the given session. An empty file still produces a single
// zero-byte chunk on the client side (see assets/.../utils/helper.ts::getChunks
// which falls back to `file.slice(0)` when no chunks are produced).
func expectedChunkCount(s *fs.UploadSession) int {
if s == nil || s.Props == nil {
return 1
}
if s.ChunkSize <= 0 || s.Props.Size == 0 {
return 1
}
return int((s.Props.Size + s.ChunkSize - 1) / s.ChunkSize)
}
func (m *manager) PreValidateUpload(ctx context.Context, dst *fs.URI, files ...fs.PreValidateFile) error {
return m.fs.PreValidateUpload(ctx, dst, files...)
}
func (m *manager) CreateUploadSession(ctx context.Context, req *fs.UploadRequest, opts ...fs.Option) (*fs.UploadCredential, error) {
o := newOption()
for _, opt := range opts {
opt.Apply(o)
}
// Validate metadata
if req.Props.Metadata != nil {
if _, err := m.validateMetadata(ctx, lo.MapToSlice(req.Props.Metadata, func(key string, value string) fs.MetadataPatch {
return fs.MetadataPatch{
Key: key,
Value: value,
}
})...); err != nil {
return nil, err
}
}
uploadSession := o.UploadSession
var (
err error
)
if uploadSession == nil {
// If upload session not specified, invoke DBFS to create one
sessionID := uuid.Must(uuid.NewV4()).String()
req.Props.UploadSessionID = sessionID
ttl := m.settings.UploadSessionTTL(ctx)
req.Props.ExpireAt = time.Now().Add(ttl)
// Prepare for upload
uploadSession, err = m.fs.PrepareUpload(ctx, req)
if err != nil {
return nil, fmt.Errorf("faield to prepare uplaod: %w", err)
}
}
d, err := m.GetStorageDriver(ctx, m.CastStoragePolicyOnSlave(ctx, uploadSession.Policy))
if err != nil {
m.OnUploadFailed(ctx, uploadSession)
return nil, err
}
uploadSession.ChunkSize = uploadSession.Policy.Settings.ChunkSize
// Create upload credential for underlying storage driver
credential := &fs.UploadCredential{}
unrelayed := !uploadSession.Policy.Settings.Relay || m.stateless
if unrelayed {
credential, err = d.Token(ctx, uploadSession, req)
if err != nil {
m.OnUploadFailed(ctx, uploadSession)
return nil, err
}
} else {
// For relayed upload, we don't need to create credential
uploadSession.ChunkSize = 0
credential.ChunkSize = 0
credential.EncryptMetadata = nil
uploadSession.Props.ClientSideEncrypted = false
}
credential.SessionID = uploadSession.Props.UploadSessionID
credential.Expires = req.Props.ExpireAt.Unix()
credential.StoragePolicy = uploadSession.Policy
credential.CallbackSecret = uploadSession.CallbackSecret
credential.Uri = uploadSession.Props.Uri.String()
credential.EncryptMetadata = uploadSession.EncryptMetadata
if !unrelayed {
credential.EncryptMetadata = nil
}
// If upload sentinel check is required, queue a check task
if d.Capabilities().StaticFeatures.Enabled(int(driver.HandlerCapabilityUploadSentinelRequired)) {
t, err := newUploadSentinelCheckTask(ctx, uploadSession)
if err != nil {
m.OnUploadFailed(ctx, uploadSession)
return nil, fmt.Errorf("failed to create upload sentinel check task: %w", err)
}
if err := m.dep.EntityRecycleQueue(ctx).QueueTask(ctx, t); err != nil {
m.OnUploadFailed(ctx, uploadSession)
return nil, fmt.Errorf("failed to queue upload sentinel check task: %w", err)
}
uploadSession.SentinelTaskID = t.ID()
}
err = m.kv.Set(
UploadSessionCachePrefix+req.Props.UploadSessionID,
*uploadSession,
max(1, int(req.Props.ExpireAt.Sub(time.Now()).Seconds())),
)
if err != nil {
m.OnUploadFailed(ctx, uploadSession)
return nil, err
}
return credential, nil
}
func (m *manager) ConfirmUploadSession(ctx context.Context, session *fs.UploadSession, chunkIndex int) (fs.File, error) {
// Get placeholder file
file, err := m.fs.Get(ctx, session.Props.Uri)
if err != nil {
return nil, fmt.Errorf("failed to get placeholder file: %w", err)
}
// Confirm locks on placeholder file
if session.LockToken != "" {
release, ls, err := m.fs.ConfirmLock(ctx, file, file.Uri(false), session.LockToken)
if err != nil {
return nil, fs.ErrLockExpired.WithError(err)
}
defer release()
ctx = fs.LockSessionToContext(ctx, ls)
}
// Make sure this storage policy is OK to receive data from clients to Cloudreve server.
if session.Policy.Type != types.PolicyTypeLocal && !session.Policy.Settings.Relay {
return nil, serializer.NewError(serializer.CodePolicyNotAllowed, "", nil)
}
actualSizeStart := int64(chunkIndex) * session.ChunkSize
if session.Policy.Settings.ChunkSize == 0 && chunkIndex > 0 {
return nil, serializer.NewError(serializer.CodeInvalidChunkIndex, "Chunk index cannot be greater than 0", nil)
}
if actualSizeStart > 0 && actualSizeStart >= session.Props.Size {
return nil, serializer.NewError(serializer.CodeInvalidChunkIndex, "Chunk offset cannot be greater than file size", nil)
}
return file, nil
}
func (m *manager) PrepareUpload(ctx context.Context, req *fs.UploadRequest, opts ...fs.Option) (*fs.UploadSession, error) {
return m.fs.PrepareUpload(ctx, req, opts...)
}
func (m *manager) Upload(ctx context.Context, req *fs.UploadRequest, policy *ent.StoragePolicy, session *fs.UploadSession) error {
d, err := m.GetStorageDriver(ctx, m.CastStoragePolicyOnSlave(ctx, policy))
if err != nil {
return err
}
if session != nil && session.EncryptMetadata != nil && !req.Props.ClientSideEncrypted {
cryptor, err := m.dep.EncryptorFactory(ctx)(session.EncryptMetadata.Algorithm)
if err != nil {
return fmt.Errorf("failed to create cryptor: %w", err)
}
err = cryptor.LoadMetadata(ctx, session.EncryptMetadata)
if err != nil {
return fmt.Errorf("failed to load encrypt metadata: %w", err)
}
if err := cryptor.SetSource(req.File, req.Seeker, req.Props.Size, 0); err != nil {
return fmt.Errorf("failed to set source: %w", err)
}
req.File = cryptor
if req.Seeker != nil {
req.Seeker = cryptor
}
}
if err := d.Put(ctx, req); err != nil {
return serializer.NewError(serializer.CodeIOFailed, "Failed to upload file", err)
}
return nil
}
func (m *manager) MarkChunkUploaded(ctx context.Context, session *fs.UploadSession, chunkIndex int) (bool, error) {
if session == nil || session.Props == nil {
return false, fmt.Errorf("invalid upload session")
}
sessionID := session.Props.UploadSessionID
mu := lockUploadSession(sessionID)
mu.Lock()
defer mu.Unlock()
// Reload the session from KV so we observe progress recorded by concurrent
// chunk uploads for the same session. On slave nodes m.kv is the in-process
// MemoStore, on master it is either MemoStore or RedisStore; both are read
// through the same interface here.
raw, ok := m.kv.Get(UploadSessionCachePrefix + sessionID)
if !ok {
// Session already completed or cancelled elsewhere. Nothing to do —
// the caller must not trigger CompleteUpload again.
return false, nil
}
fresh, ok := raw.(fs.UploadSession)
if !ok {
return false, fmt.Errorf("unexpected upload session type in KV")
}
if fresh.ChunksReceived == nil {
fresh.ChunksReceived = make(map[int]struct{})
}
total := expectedChunkCount(&fresh)
// If completion has already been claimed by a previous caller (e.g. a
// duplicate chunk retry after the server-side write succeeded), do not
// re-trigger CompleteUpload.
alreadyComplete := len(fresh.ChunksReceived) >= total
// Idempotent add: recording the same chunk index twice is a no-op.
fresh.ChunksReceived[chunkIndex] = struct{}{}
// Persist the updated tracker unconditionally so retries, subsequent
// chunks, and completion state survive across requests.
ttl := max(1, int(time.Until(fresh.Props.ExpireAt).Seconds()))
if err := m.kv.Set(UploadSessionCachePrefix+sessionID, fresh, ttl); err != nil {
return false, fmt.Errorf("failed to persist upload session progress: %w", err)
}
// Only the caller that transitioned the tracker from incomplete to
// complete is allowed to trigger CompleteUpload.
if !alreadyComplete && len(fresh.ChunksReceived) >= total {
session.ChunksReceived = fresh.ChunksReceived
return true, nil
}
return false, nil
}
func (m *manager) CancelUploadSession(ctx context.Context, path *fs.URI, sessionID string) error {
// Get upload session
var session *fs.UploadSession
sessionRaw, ok := m.kv.Get(UploadSessionCachePrefix + sessionID)
if ok {
sessionTyped := sessionRaw.(fs.UploadSession)
session = &sessionTyped
}
var (
staleEntities []fs.Entity
indexDiff *fs.IndexDiff
err error
)
if !m.stateless {
staleEntities, indexDiff, err = m.fs.CancelUploadSession(ctx, path, sessionID, session)
if err != nil {
return err
}
m.l.Debug("New stale entities: %v", staleEntities)
}
if session != nil {
ctx = context.WithValue(ctx, cluster.SlaveNodeIDCtx{}, strconv.Itoa(session.Policy.NodeID))
d, err := m.GetStorageDriver(ctx, m.CastStoragePolicyOnSlave(ctx, session.Policy))
if err != nil {
return fmt.Errorf("failed to get storage driver: %w", err)
}
if m.stateless {
if _, err = d.Delete(ctx, session.Props.SavePath); err != nil {
return fmt.Errorf("failed to delete file: %w", err)
}
} else {
if err = d.CancelToken(ctx, session); err != nil {
return fmt.Errorf("failed to cancel upload session: %w", err)
}
}
m.kv.Delete(UploadSessionCachePrefix, session.Props.UploadSessionID)
releaseUploadSessionLock(session.Props.UploadSessionID)
}
// Delete stale entities
if len(staleEntities) > 0 {
t, err := newExplicitEntityRecycleTask(ctx, lo.Map(staleEntities, func(entity fs.Entity, index int) int {
return entity.ID()
}))
if err != nil {
return fmt.Errorf("failed to create explicit entity recycle task: %w", err)
}
if err := m.dep.EntityRecycleQueue(ctx).QueueTask(ctx, t); err != nil {
return fmt.Errorf("failed to queue explicit entity recycle task: %w", err)
}
}
// Process index diff
if indexDiff != nil {
m.processIndexDiff(ctx, indexDiff)
}
return nil
}
func (m *manager) CompleteUpload(ctx context.Context, session *fs.UploadSession) (fs.File, error) {
d, err := m.GetStorageDriver(ctx, m.CastStoragePolicyOnSlave(ctx, session.Policy))
if err != nil {
return nil, err
}
if err := d.CompleteUpload(ctx, session); err != nil {
return nil, err
}
var (
file fs.File
ownerId int
)
if m.fs != nil {
file, err = m.fs.CompleteUpload(ctx, session)
if err != nil {
return nil, fmt.Errorf("failed to complete upload: %w", err)
}
ownerId = file.OwnerID()
}
if session.SentinelTaskID > 0 {
// Cancel sentinel check task
m.l.Debug("Cancel upload sentinel check task [%d].", session.SentinelTaskID)
if err := m.dep.TaskClient().SetCompleteByID(ctx, session.SentinelTaskID); err != nil {
m.l.Warning("Failed to set upload sentinel check task [%d] to complete: %s", session.SentinelTaskID, err)
}
}
m.onNewEntityUploaded(ctx, session, d, ownerId)
// Remove upload session
_ = m.kv.Delete(UploadSessionCachePrefix, session.Props.UploadSessionID)
releaseUploadSessionLock(session.Props.UploadSessionID)
return file, nil
}
func (m *manager) Update(ctx context.Context, req *fs.UploadRequest, opts ...fs.Option) (fs.File, error) {
o := newOption()
for _, opt := range opts {
opt.Apply(o)
}
entityType := types.EntityTypeVersion
if o.EntityType != nil {
entityType = *o.EntityType
}
req.Props.EntityType = &entityType
if o.EntityTypeNil {
req.Props.EntityType = nil
}
req.Props.UploadSessionID = uuid.Must(uuid.NewV4()).String()
// Sever side supported encryption algorithms
req.Props.EncryptionSupported = []types.Cipher{types.CipherAES256CTR}
if m.stateless {
return m.updateStateless(ctx, req, o)
}
// Prepare for upload
uploadSession, err := m.fs.PrepareUpload(ctx, req)
if err != nil {
return nil, fmt.Errorf("faield to prepare uplaod: %w", err)
}
if err := m.Upload(ctx, req, uploadSession.Policy, uploadSession); err != nil {
m.OnUploadFailed(ctx, uploadSession)
return nil, fmt.Errorf("failed to upload new entity: %w", err)
}
file, err := m.CompleteUpload(ctx, uploadSession)
if err != nil {
m.OnUploadFailed(ctx, uploadSession)
return nil, fmt.Errorf("failed to complete update: %w", err)
}
return file, nil
}
func (m *manager) OnUploadFailed(ctx context.Context, session *fs.UploadSession) {
ctx = context.WithoutCancel(ctx)
if session != nil && session.Props != nil {
defer releaseUploadSessionLock(session.Props.UploadSessionID)
}
if !m.stateless {
if session.LockToken != "" {
if err := m.Unlock(ctx, session.LockToken); err != nil {
m.l.Warning("OnUploadFailed hook failed to unlock: %s", err)
}
}
if session.NewFileCreated {
if err := m.Delete(ctx, []*fs.URI{session.Props.Uri}, fs.WithSysSkipSoftDelete(true)); err != nil {
m.l.Warning("OnUploadFailed hook failed to delete file: %s", err)
}
} else if !session.Importing {
if _, err := m.fs.VersionControl(ctx, session.Props.Uri, session.EntityID, true); err != nil {
m.l.Warning("OnUploadFailed hook failed to version control: %s", err)
}
}
} else {
d, err := m.GetStorageDriver(ctx, m.CastStoragePolicyOnSlave(ctx, session.Policy))
if err != nil {
m.l.Warning("OnUploadFailed hook failed: %s", err)
}
if failed, err := d.Delete(ctx, session.Props.SavePath); err != nil {
m.l.Warning("OnUploadFailed hook failed to remove uploaded file: %s, failed file: %v", err, failed)
}
}
}
// similar to Update, but expected to be executed on slave node.
func (m *manager) updateStateless(ctx context.Context, req *fs.UploadRequest, o *fs.FsOption) (fs.File, error) {
// Prepare for upload
res, err := o.Node.PrepareUpload(ctx, &fs.StatelessPrepareUploadService{
UploadRequest: req,
UserID: o.StatelessUserID,
})
if err != nil {
return nil, fmt.Errorf("faield to prepare uplaod: %w", err)
}
req.Props = res.Req.Props
if err := m.Upload(ctx, req, res.Session.Policy, res.Session); err != nil {
if err := o.Node.OnUploadFailed(ctx, &fs.StatelessOnUploadFailedService{
UploadSession: res.Session,
UserID: o.StatelessUserID,
}); err != nil {
m.l.Warning("Failed to call stateless OnUploadFailed: %s", err)
}
return nil, fmt.Errorf("failed to upload new entity: %w", err)
}
err = o.Node.CompleteUpload(ctx, &fs.StatelessCompleteUploadService{
UploadSession: res.Session,
UserID: o.StatelessUserID,
})
if err != nil {
if err := o.Node.OnUploadFailed(ctx, &fs.StatelessOnUploadFailedService{
UploadSession: res.Session,
UserID: o.StatelessUserID,
}); err != nil {
m.l.Warning("Failed to call stateless OnUploadFailed: %s", err)
}
return nil, fmt.Errorf("failed to complete update: %w", err)
}
return nil, nil
}
func (m *manager) onNewEntityUploaded(ctx context.Context, session *fs.UploadSession, d driver.Handler, owner int) {
if !m.stateless {
// Submit media meta task for new entity
m.mediaMetaForNewEntity(ctx, session, d)
// Submit full text index task for new entity
m.fullTextIndexForNewEntity(ctx, session, owner)
}
}
// Upload sentinel check task is used for compliant storage policy (COS, S3...), it will delete the marked entity.
// It is expected to be queued after upload session is created, and canceled after upload callback is completed.
// If this task is executed, it means the upload callback does not complete in time.
type (
UploadSentinelCheckTask struct {
*queue.DBTask
}
UploadSentinelCheckTaskState struct {
Session *fs.UploadSession `json:"session"`
}
)
const (
uploadSentinelCheckMargin = 5 * time.Minute
)
func init() {
queue.RegisterResumableTaskFactory(queue.UploadSentinelCheckTaskType, NewUploadSentinelCheckTaskFromModel)
}
func NewUploadSentinelCheckTaskFromModel(task *ent.Task) queue.Task {
return &UploadSentinelCheckTask{
DBTask: &queue.DBTask{
Task: task,
},
}
}
func newUploadSentinelCheckTask(ctx context.Context, uploadSession *fs.UploadSession) (*ExplicitEntityRecycleTask, error) {
state := &UploadSentinelCheckTaskState{
Session: uploadSession,
}
stateBytes, err := json.Marshal(state)
if err != nil {
return nil, fmt.Errorf("failed to marshal state: %w", err)
}
resumeAfter := uploadSession.Props.ExpireAt.Add(uploadSentinelCheckMargin)
t := &ExplicitEntityRecycleTask{
DBTask: &queue.DBTask{
Task: &ent.Task{
Type: queue.UploadSentinelCheckTaskType,
CorrelationID: logging.CorrelationID(ctx),
PrivateState: string(stateBytes),
PublicState: &types.TaskPublicState{
ResumeTime: resumeAfter.Unix(),
},
},
DirectOwner: inventory.UserFromContext(ctx),
},
}
return t, nil
}
func (m *UploadSentinelCheckTask) Do(ctx context.Context) (task.Status, error) {
dep := dependency.FromContext(ctx)
taskClient := dep.TaskClient()
l := dep.Logger()
fm := NewFileManager(dep, inventory.UserFromContext(ctx)).(*manager)
// Check if sentinel is canceled due to callback complete
t, err := taskClient.GetTaskByID(ctx, m.ID())
if err != nil {
return task.StatusError, fmt.Errorf("failed to get task by ID: %w", err)
}
if t.Status == task.StatusCompleted {
l.Info("Upload sentinel check task [%d] is canceled due to callback complete.", m.ID())
return task.StatusCompleted, nil
}
// unmarshal state
state := &UploadSentinelCheckTaskState{}
if err := json.Unmarshal([]byte(m.State()), state); err != nil {
return task.StatusError, fmt.Errorf("failed to unmarshal state: %w", err)
}
l.Info("Upload sentinel check triggered, clean up stale place holder entity [%d].", state.Session.EntityID)
entity, err := fm.fs.GetEntity(ctx, state.Session.EntityID)
if err != nil {
l.Debug("Failed to get entity [%d]: %s, skip sentinel check.", state.Session.EntityID, err)
return task.StatusCompleted, nil
}
_, d, err := fm.getEntityPolicyDriver(ctx, entity, nil)
if err != nil {
l.Debug("Failed to get storage driver for entity [%d]: %s", state.Session.EntityID, err)
return task.StatusError, err
}
_, err = d.Delete(ctx, entity.Source())
if err != nil {
l.Debug("Failed to delete entity source [%d]: %s", state.Session.EntityID, err)
return task.StatusError, err
}
if err := d.CancelToken(ctx, state.Session); err != nil {
l.Debug("Failed to cancel token [%d]: %s", state.Session.EntityID, err)
}
return task.StatusCompleted, nil
}
pkg/filemanager/manager/viewer.go
+92
View File
@@ -0,0 +1,92 @@
package manager
import (
"context"
"encoding/gob"
"fmt"
"time"
"github.com/cloudreve/Cloudreve/v4/inventory/types"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/fs"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/fs/dbfs"
"github.com/cloudreve/Cloudreve/v4/pkg/util"
"github.com/gofrs/uuid"
)
type (
ViewerSession struct {
ID string `json:"id"`
AccessToken string `json:"access_token"`
Expires int64 `json:"expires"`
File fs.File `json:"-"`
}
ViewerSessionCache struct {
ID string
Uri string
UserID int
FileID int
ViewerID string
Version string
Token string
}
ViewerSessionCacheCtx struct{}
ViewerCtx struct{}
)
const (
ViewerSessionCachePrefix = "viewer_session_"
sessionExpiresPadding = 10
)
func init() {
gob.Register(ViewerSessionCache{})
}
func (m *manager) CreateViewerSession(ctx context.Context, uri *fs.URI, version string, viewer *types.Viewer) (*ViewerSession, error) {
file, err := m.fs.Get(ctx, uri, dbfs.WithFileEntities(), dbfs.WithNotRoot())
if err != nil {
return nil, err
}
versionType := types.EntityTypeVersion
found, desired := fs.FindDesiredEntity(file, version, m.hasher, &versionType)
if !found {
return nil, fs.ErrEntityNotExist
}
if desired.Size() > m.settings.MaxOnlineEditSize(ctx) {
return nil, fs.ErrFileSizeTooBig
}
sessionID := uuid.Must(uuid.NewV4()).String()
token := util.RandStringRunesCrypto(128)
sessionCache := &ViewerSessionCache{
ID: sessionID,
Uri: file.Uri(false).String(),
UserID: m.user.ID,
ViewerID: viewer.ID,
FileID: file.ID(),
Version: version,
Token: fmt.Sprintf("%s.%s", sessionID, token),
}
ttl := m.settings.ViewerSessionTTL(ctx)
if err := m.kv.Set(ViewerSessionCachePrefix+sessionID, *sessionCache, ttl); err != nil {
return nil, err
}
return &ViewerSession{
File: file,
ID: sessionID,
AccessToken: sessionCache.Token,
Expires: time.Now().Add(time.Duration(ttl-sessionExpiresPadding) * time.Second).UnixMilli(),
}, nil
}
func ViewerSessionFromContext(ctx context.Context) *ViewerSessionCache {
return ctx.Value(ViewerSessionCacheCtx{}).(*ViewerSessionCache)
}
func ViewerFromContext(ctx context.Context) *types.Viewer {
return ctx.Value(ViewerCtx{}).(*types.Viewer)
}