chore: 初始化 Cloudreve 源码仓库

pkg/searcher/indexer/chunker.go

@@ -0,0 +1,101 @@
+package indexer
+
+import "strings"
+
+const defaultMaxBytes = 2000
+
+// ChunkText splits text into chunks of approximately maxBytes bytes each.
+// It splits on paragraph breaks (\n\n), combines small paragraphs until the
+// byte limit is reached, and splits large paragraphs at word boundaries.
+func ChunkText(text string, maxBytes int) []string {
+	if maxBytes <= 0 {
+		maxBytes = defaultMaxBytes
+	}
+
+	text = strings.TrimSpace(text)
+	if text == "" {
+		return nil
+	}
+
+	paragraphs := strings.Split(text, "\n\n")
+	var chunks []string
+	var current []string
+	currentBytes := 0
+
+	for _, para := range paragraphs {
+		para = strings.TrimSpace(para)
+		if para == "" {
+			continue
+		}
+
+		paraBytes := len(para)
+
+		// If a single paragraph exceeds maxBytes, split it at word boundaries
+		if paraBytes > maxBytes {
+			// Flush accumulated content first
+			if currentBytes > 0 {
+				chunks = append(chunks, strings.Join(current, "\n\n"))
+				current = nil
+				currentBytes = 0
+			}
+			chunks = append(chunks, splitByBytes(para, maxBytes)...)
+			continue
+		}
+
+		// If adding this paragraph (plus separator) would exceed the limit, flush
+		joinerLen := 0
+		if currentBytes > 0 {
+			joinerLen = 2 // "\n\n"
+		}
+		if currentBytes+joinerLen+paraBytes > maxBytes && currentBytes > 0 {
+			chunks = append(chunks, strings.Join(current, "\n\n"))
+			current = nil
+			currentBytes = 0
+		}
+
+		if currentBytes > 0 {
+			currentBytes += 2 // account for "\n\n" joiner
+		}
+		current = append(current, para)
+		currentBytes += paraBytes
+	}
+
+	// Flush remaining
+	if currentBytes > 0 {
+		chunks = append(chunks, strings.Join(current, "\n\n"))
+	}
+
+	return chunks
+}
+
+// splitByBytes splits text into chunks at word boundaries, each at most maxBytes bytes.
+func splitByBytes(text string, maxBytes int) []string {
+	words := strings.Fields(text)
+	var chunks []string
+	var current []string
+	currentBytes := 0
+
+	for _, w := range words {
+		wLen := len(w)
+		spaceLen := 0
+		if currentBytes > 0 {
+			spaceLen = 1
+		}
+
+		if currentBytes+spaceLen+wLen > maxBytes && currentBytes > 0 {
+			chunks = append(chunks, strings.Join(current, " "))
+			current = nil
+			currentBytes = 0
+			spaceLen = 0
+		}
+
+		current = append(current, w)
+		currentBytes += spaceLen + wLen
+	}
+
+	if len(current) > 0 {
+		chunks = append(chunks, strings.Join(current, " "))
+	}
+
+	return chunks
+}

pkg/searcher/indexer/chunker_test.go

@@ -0,0 +1,84 @@
+package indexer
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestChunkText_Empty(t *testing.T) {
+	assert.Nil(t, ChunkText("", 500))
+	assert.Nil(t, ChunkText("   ", 500))
+}
+
+func TestChunkText_SingleSmallParagraph(t *testing.T) {
+	chunks := ChunkText("Hello world", 500)
+	assert.Equal(t, []string{"Hello world"}, chunks)
+}
+
+func TestChunkText_MultipleParagraphsCombined(t *testing.T) {
+	text := "First paragraph.\n\nSecond paragraph.\n\nThird paragraph."
+	chunks := ChunkText(text, 500)
+	assert.Len(t, chunks, 1)
+	assert.Contains(t, chunks[0], "First paragraph.")
+	assert.Contains(t, chunks[0], "Third paragraph.")
+}
+
+func TestChunkText_SplitOnParagraphBoundary(t *testing.T) {
+	// Create two paragraphs each ~300 bytes
+	para := strings.Repeat("abcde ", 50) // 300 bytes
+	para = strings.TrimSpace(para)
+	text := para + "\n\n" + para
+
+	chunks := ChunkText(text, 500)
+	assert.Len(t, chunks, 2)
+}
+
+func TestChunkText_LargeParagraphSplit(t *testing.T) {
+	// Create a single paragraph of 1200 bytes (240 words * 5 bytes each)
+	words := make([]string, 240)
+	for i := range words {
+		words[i] = "word" // 4 bytes + 1 space = 5 per word
+	}
+	text := strings.Join(words, " ") // 240*4 + 239 = 1199 bytes
+
+	chunks := ChunkText(text, 500)
+	assert.Len(t, chunks, 3)
+	assert.LessOrEqual(t, len(chunks[0]), 500)
+	assert.LessOrEqual(t, len(chunks[1]), 500)
+	assert.Greater(t, len(chunks[2]), 0)
+}
+
+func TestChunkText_DefaultMaxBytes(t *testing.T) {
+	chunks := ChunkText("hello", 0)
+	assert.Equal(t, []string{"hello"}, chunks)
+}
+
+func TestChunkText_EmptyParagraphsIgnored(t *testing.T) {
+	text := "First.\n\n\n\n\n\nSecond."
+	chunks := ChunkText(text, 500)
+	assert.Len(t, chunks, 1)
+	assert.Equal(t, "First.\n\nSecond.", chunks[0])
+}
+
+func TestChunkText_ParagraphKeptWhole(t *testing.T) {
+	// A paragraph under the limit should not be split
+	para := strings.Repeat("x", 400)
+	chunks := ChunkText(para, 500)
+	assert.Len(t, chunks, 1)
+	assert.Equal(t, para, chunks[0])
+}
+
+func TestChunkText_JoinerAccountedInLimit(t *testing.T) {
+	// Two paragraphs that fit individually but exceed the limit when joined with "\n\n"
+	p1 := strings.Repeat("a", 250)
+	p2 := strings.Repeat("b", 250)
+	text := p1 + "\n\n" + p2
+
+	chunks := ChunkText(text, 500)
+	// 250 + 2 + 250 = 502 > 500, so they should be split
+	assert.Len(t, chunks, 2)
+	assert.Equal(t, p1, chunks[0])
+	assert.Equal(t, p2, chunks[1])
+}

pkg/searcher/indexer/meilisearch.go

@@ -0,0 +1,394 @@
+package indexer
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"slices"
+	"strings"
+
+	"github.com/cloudreve/Cloudreve/v4/pkg/logging"
+	"github.com/cloudreve/Cloudreve/v4/pkg/searcher"
+	"github.com/cloudreve/Cloudreve/v4/pkg/setting"
+	"github.com/meilisearch/meilisearch-go"
+)
+
+const (
+	indexName         = "cloudreve_files"
+	embedderName      = "cr-text"
+	embeddingTemplate = "Chunk #{{doc.chunk_idx}} in a file named '{{doc.file_name}}': {{ doc.text }}"
+)
+
+// MeilisearchIndexer implements SearchIndexer using Meilisearch.
+type MeilisearchIndexer struct {
+	client    meilisearch.ServiceManager
+	l         logging.Logger
+	pageSize  int
+	chunkSize int
+	cfg       *setting.FTSIndexMeilisearchSetting
+}
+
+// NewMeilisearchIndexer creates a new MeilisearchIndexer.
+func NewMeilisearchIndexer(msCfg *setting.FTSIndexMeilisearchSetting, chunkSize int, l logging.Logger) *MeilisearchIndexer {
+	client := meilisearch.New(msCfg.Endpoint, meilisearch.WithAPIKey(msCfg.APIKey))
+	return &MeilisearchIndexer{
+		client:    client,
+		l:         l,
+		pageSize:  msCfg.PageSize,
+		chunkSize: chunkSize,
+		cfg:       msCfg,
+	}
+}
+
+var (
+	requiredFilterable = []string{"owner_id", "file_id", "entity_id"}
+	requiredSearchable = []string{"text", "file_name"}
+	requiredDistinct   = "file_id"
+)
+
+func (m *MeilisearchIndexer) IndexReady(ctx context.Context) (bool, error) {
+	index := m.client.Index(indexName)
+
+	settings, err := index.GetSettingsWithContext(ctx)
+	if err != nil {
+		// If the index doesn't exist, Meilisearch returns an error.
+		return false, nil
+	}
+
+	// Check filterable attributes.
+	for _, attr := range requiredFilterable {
+		if !slices.Contains(settings.FilterableAttributes, attr) {
+			return false, nil
+		}
+	}
+
+	// Check searchable attributes.
+	for _, attr := range requiredSearchable {
+		if !slices.Contains(settings.SearchableAttributes, attr) {
+			return false, nil
+		}
+	}
+
+	// Check distinct attribute.
+	if settings.DistinctAttribute == nil || *settings.DistinctAttribute != requiredDistinct {
+		return false, nil
+	}
+
+	// Check embedder if embedding is enabled.
+	if m.cfg.EmbeddingEnbaled {
+		if settings.Embedders == nil {
+			return false, nil
+		}
+		if _, ok := settings.Embedders[embedderName]; !ok {
+			return false, nil
+		}
+	}
+
+	return true, nil
+}
+
+func (m *MeilisearchIndexer) EnsureIndex(ctx context.Context) error {
+	_, err := m.client.CreateIndexWithContext(ctx, &meilisearch.IndexConfig{
+		Uid:        indexName,
+		PrimaryKey: "id",
+	})
+	if err != nil {
+		m.l.Debug("Create index returned (may already exist): %s", err)
+	}
+
+	index := m.client.Index(indexName)
+
+	filterableAttrs := []any{"owner_id", "file_id", "entity_id"}
+	if _, err := index.UpdateFilterableAttributesWithContext(ctx, &filterableAttrs); err != nil {
+		return fmt.Errorf("failed to set filterable attributes: %w", err)
+	}
+
+	searchableAttrs := []string{"text", "file_name"}
+	if _, err := index.UpdateSearchableAttributesWithContext(ctx, &searchableAttrs); err != nil {
+		return fmt.Errorf("failed to set searchable attributes: %w", err)
+	}
+
+	_, err = index.UpdateDistinctAttributeWithContext(ctx, "file_id")
+	if err != nil {
+		return fmt.Errorf("failed to set distinct attribute: %w", err)
+	}
+
+	if m.cfg.EmbeddingEnbaled {
+		var embedder meilisearch.Embedder
+		if err := json.Unmarshal([]byte(m.cfg.EmbeddingSetting), &embedder); err != nil {
+			m.cfg.EmbeddingEnbaled = false
+			m.l.Warning("Failed to unmarshal embedding setting: %s, fallback to disable embedding", err)
+			return nil
+		}
+
+		embedder.DocumentTemplate = embeddingTemplate
+		_, err := index.UpdateEmbeddersWithContext(ctx, map[string]meilisearch.Embedder{
+			embedderName: embedder,
+		})
+		if err != nil {
+			return fmt.Errorf("failed to set embedders: %w", err)
+		}
+	} else {
+		_, err := index.ResetEmbeddersWithContext(ctx)
+		if err != nil {
+			m.l.Warning("Failed to reset embedder: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func (m *MeilisearchIndexer) IndexFile(ctx context.Context, ownerID, fileID, entityID int, fileName, text string) error {
+	chunks := ChunkText(text, m.chunkSize)
+	if len(chunks) == 0 {
+		return nil
+	}
+
+	docs := make([]searcher.SearchDocument, 0, len(chunks))
+	for i, chunk := range chunks {
+		docs = append(docs, searcher.SearchDocument{
+			ID:       fmt.Sprintf("%d_%d", fileID, i),
+			FileID:   fileID,
+			OwnerID:  ownerID,
+			EntityID: entityID,
+			ChunkIdx: i,
+			FileName: fileName,
+			Text:     chunk,
+		})
+	}
+
+	index := m.client.Index(indexName)
+	pk := "id"
+	if _, err := index.AddDocumentsWithContext(ctx, docs, &meilisearch.DocumentOptions{PrimaryKey: &pk}); err != nil {
+		return fmt.Errorf("failed to add documents: %w", err)
+	}
+
+	return nil
+}
+
+func (m *MeilisearchIndexer) DeleteByFileIDs(ctx context.Context, fileID ...int) error {
+	if len(fileID) == 0 {
+		return nil
+	}
+
+	index := m.client.Index(indexName)
+	strs := make([]string, len(fileID))
+	for i, id := range fileID {
+		strs[i] = fmt.Sprintf("%d", id)
+	}
+	filter := fmt.Sprintf("file_id IN [%s]", strings.Join(strs, ", "))
+	if _, err := index.DeleteDocumentsByFilterWithContext(ctx, filter, nil); err != nil {
+		return fmt.Errorf("failed to delete documents by file_ids: %w", err)
+	}
+	return nil
+}
+
+func (m *MeilisearchIndexer) ChangeOwner(ctx context.Context, fileID, oldOwnerID, newOwnerID int) error {
+	index := m.client.Index(indexName)
+	filter := fmt.Sprintf("file_id = %d AND owner_id = %d", fileID, oldOwnerID)
+
+	// Fetch all existing document chunks in batches.
+	const batchSize int64 = 100
+	var allDocs []searcher.SearchDocument
+	for offset := int64(0); ; offset += batchSize {
+		var result meilisearch.DocumentsResult
+		if err := index.GetDocumentsWithContext(ctx, &meilisearch.DocumentsQuery{
+			Filter: filter,
+			Limit:  batchSize,
+			Offset: offset,
+		}, &result); err != nil {
+			return fmt.Errorf("failed to get documents: %w", err)
+		}
+
+		for _, hit := range result.Results {
+			var doc searcher.SearchDocument
+			if err := hit.DecodeInto(&doc); err != nil {
+				m.l.Warning("Failed to decode document during owner change: %s", err)
+				continue
+			}
+			allDocs = append(allDocs, doc)
+		}
+
+		if int64(len(result.Results)) < batchSize {
+			break
+		}
+	}
+
+	if len(allDocs) == 0 {
+		return nil
+	}
+
+	// Update owner_id in place — primary key is {fileID}_{chunkIdx} so it stays the same.
+	for i := range allDocs {
+		allDocs[i].OwnerID = newOwnerID
+	}
+
+	if _, err := index.UpdateDocumentsInBatchesWithContext(ctx, allDocs, 100, nil); err != nil {
+		return fmt.Errorf("failed to update documents with new owner: %w", err)
+	}
+
+	return nil
+}
+
+func (m *MeilisearchIndexer) CopyByFileID(ctx context.Context, srcFileID, dstFileID, dstOwnerID, dstEntityID int) error {
+	index := m.client.Index(indexName)
+	filter := fmt.Sprintf("file_id = %d", srcFileID)
+
+	const batchSize int64 = 100
+	var allDocs []searcher.SearchDocument
+	for offset := int64(0); ; offset += batchSize {
+		var result meilisearch.DocumentsResult
+		if err := index.GetDocumentsWithContext(ctx, &meilisearch.DocumentsQuery{
+			Filter: filter,
+			Limit:  batchSize,
+			Offset: offset,
+		}, &result); err != nil {
+			return fmt.Errorf("failed to get source documents: %w", err)
+		}
+
+		for _, hit := range result.Results {
+			var doc searcher.SearchDocument
+			if err := hit.DecodeInto(&doc); err != nil {
+				m.l.Warning("Failed to decode document during copy: %s", err)
+				continue
+			}
+			allDocs = append(allDocs, doc)
+		}
+
+		if int64(len(result.Results)) < batchSize {
+			break
+		}
+	}
+
+	if len(allDocs) == 0 {
+		return fmt.Errorf("no source documents found for file %d", srcFileID)
+	}
+
+	for i := range allDocs {
+		if allDocs[i].EntityID != dstEntityID {
+			m.l.Warning("Entity id mismatch for file %d, original: %d, destination: %d", srcFileID, allDocs[i].EntityID, dstEntityID)
+			continue
+		}
+
+		allDocs[i].ID = fmt.Sprintf("%d_%d", dstFileID, allDocs[i].ChunkIdx)
+		allDocs[i].FileID = dstFileID
+		allDocs[i].OwnerID = dstOwnerID
+		allDocs[i].EntityID = dstEntityID
+	}
+
+	if len(allDocs) == 0 {
+		return fmt.Errorf("no source documents found for file %d", srcFileID)
+	}
+
+	pk := "id"
+	if _, err := index.AddDocumentsWithContext(ctx, allDocs, &meilisearch.DocumentOptions{PrimaryKey: &pk}); err != nil {
+		return fmt.Errorf("failed to add copied documents: %w", err)
+	}
+
+	return nil
+}
+
+func (m *MeilisearchIndexer) Rename(ctx context.Context, fileID, entityID int, newFileName string) error {
+	index := m.client.Index(indexName)
+	filter := fmt.Sprintf("file_id = %d AND entity_id = %d", fileID, entityID)
+
+	const batchSize int64 = 100
+	var allDocs []searcher.SearchDocument
+	for offset := int64(0); ; offset += batchSize {
+		var result meilisearch.DocumentsResult
+		if err := index.GetDocumentsWithContext(ctx, &meilisearch.DocumentsQuery{
+			Filter: filter,
+			Limit:  batchSize,
+			Offset: offset,
+		}, &result); err != nil {
+			return fmt.Errorf("failed to get documents for rename: %w", err)
+		}
+
+		for _, hit := range result.Results {
+			var doc searcher.SearchDocument
+			if err := hit.DecodeInto(&doc); err != nil {
+				m.l.Warning("Failed to decode document during rename: %s", err)
+				continue
+			}
+			doc.FileName = newFileName
+			allDocs = append(allDocs, doc)
+		}
+
+		if int64(len(result.Results)) < batchSize {
+			break
+		}
+	}
+
+	if len(allDocs) == 0 {
+		return nil
+	}
+
+	if _, err := index.UpdateDocumentsInBatchesWithContext(ctx, allDocs, 100, nil); err != nil {
+		return fmt.Errorf("failed to update documents with new file name: %w", err)
+	}
+
+	return nil
+}
+
+func (m *MeilisearchIndexer) Search(ctx context.Context, ownerID int, query string, offset int) ([]searcher.SearchResult, int64, error) {
+	index := m.client.Index(indexName)
+
+	searchReq := &meilisearch.SearchRequest{
+		Filter:                fmt.Sprintf("owner_id = %d", ownerID),
+		Limit:                 int64(m.pageSize),
+		Offset:                int64(offset),
+		AttributesToHighlight: []string{"text"},
+	}
+
+	if m.cfg.EmbeddingEnbaled {
+		searchReq.Hybrid = &meilisearch.SearchRequestHybrid{
+			Embedder: embedderName,
+		}
+	}
+
+	resp, err := index.SearchWithContext(ctx, query, searchReq)
+	if err != nil {
+		return nil, 0, fmt.Errorf("search failed: %w", err)
+	}
+
+	results := make([]searcher.SearchResult, 0, len(resp.Hits))
+	seen := make(map[int]struct{})
+	for _, hit := range resp.Hits {
+		var doc searcher.SearchDocument
+		if err := hit.DecodeInto(&doc); err != nil {
+			continue
+		}
+
+		if _, exists := seen[doc.FileID]; exists {
+			continue
+		}
+		seen[doc.FileID] = struct{}{}
+
+		// Extract text from raw JSON for display
+		textStr := doc.Text
+		if doc.Formated != nil {
+			textStr = doc.Formated.Text
+		}
+
+		results = append(results, searcher.SearchResult{
+			FileID:   doc.FileID,
+			OwnerID:  doc.OwnerID,
+			FileName: doc.FileName,
+			Text:     textStr,
+		})
+	}
+
+	return results, resp.EstimatedTotalHits, nil
+}
+
+func (m *MeilisearchIndexer) DeleteAll(ctx context.Context) error {
+	index := m.client.Index(indexName)
+	if _, err := index.DeleteAllDocumentsWithContext(ctx, nil); err != nil {
+		return fmt.Errorf("failed to delete all documents: %w", err)
+	}
+	return nil
+}
+
+func (m *MeilisearchIndexer) Close() error {
+	return nil
+}

pkg/searcher/indexer/noop.go

@@ -0,0 +1,50 @@
+package indexer
+
+import (
+	"context"
+
+	"github.com/cloudreve/Cloudreve/v4/pkg/searcher"
+)
+
+// NoopIndexer is a no-op implementation of SearchIndexer, used when FTS is disabled.
+type NoopIndexer struct{}
+
+func (n *NoopIndexer) IndexFile(ctx context.Context, ownerID, fileID, entityID int, fileName, text string) error {
+	return nil
+}
+
+func (n *NoopIndexer) DeleteByFileIDs(ctx context.Context, fileID ...int) error {
+	return nil
+}
+
+func (n *NoopIndexer) ChangeOwner(ctx context.Context, fileID, oldOwnerID, newOwnerID int) error {
+	return nil
+}
+
+func (n *NoopIndexer) CopyByFileID(ctx context.Context, srcFileID, dstFileID, dstOwnerID, dstEntityID int) error {
+	return nil
+}
+
+func (n *NoopIndexer) Rename(ctx context.Context, fileID, entityID int, newFileName string) error {
+	return nil
+}
+
+func (n *NoopIndexer) Search(ctx context.Context, ownerID int, query string, offset int) ([]searcher.SearchResult, int64, error) {
+	return nil, 0, nil
+}
+
+func (n *NoopIndexer) IndexReady(ctx context.Context) (bool, error) {
+	return true, nil
+}
+
+func (n *NoopIndexer) EnsureIndex(ctx context.Context) error {
+	return nil
+}
+
+func (n *NoopIndexer) DeleteAll(ctx context.Context) error {
+	return nil
+}
+
+func (n *NoopIndexer) Close() error {
+	return nil
+}