chore: 初始化 Cloudreve 源码仓库
This commit is contained in:
@@ -0,0 +1,15 @@
|
||||
package extractor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
)
|
||||
|
||||
// NoopExtractor is a no-op implementation of TextExtractor, used when text extraction is disabled.
|
||||
type NoopExtractor struct{}
|
||||
|
||||
func (n *NoopExtractor) Exts() []string { return nil }
|
||||
func (n *NoopExtractor) MaxFileSize() int64 { return 0 }
|
||||
func (n *NoopExtractor) Extract(ctx context.Context, reader io.Reader) (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package extractor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/logging"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/request"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/setting"
|
||||
)
|
||||
|
||||
// TikaExtractor extracts text from documents using Apache Tika.
|
||||
type TikaExtractor struct {
|
||||
client request.Client
|
||||
settings setting.Provider
|
||||
l logging.Logger
|
||||
exts []string
|
||||
maxFileSize int64
|
||||
endpoint string
|
||||
}
|
||||
|
||||
// NewTikaExtractor creates a new TikaExtractor.
|
||||
func NewTikaExtractor(client request.Client, settings setting.Provider, l logging.Logger, cfg *setting.FTSTikaExtractorSetting) *TikaExtractor {
|
||||
exts := cfg.Exts
|
||||
return &TikaExtractor{
|
||||
client: client,
|
||||
settings: settings,
|
||||
l: l,
|
||||
exts: exts,
|
||||
maxFileSize: cfg.MaxFileSize,
|
||||
endpoint: cfg.Endpoint,
|
||||
}
|
||||
}
|
||||
|
||||
// Exts returns the list of supported file extensions.
|
||||
func (t *TikaExtractor) Exts() []string {
|
||||
return t.exts
|
||||
}
|
||||
|
||||
// MaxFileSize returns the maximum file size for text extraction.
|
||||
func (t *TikaExtractor) MaxFileSize() int64 {
|
||||
return t.maxFileSize
|
||||
}
|
||||
|
||||
// Extract sends the document to Tika and returns the extracted plain text.
|
||||
func (t *TikaExtractor) Extract(ctx context.Context, reader io.Reader) (string, error) {
|
||||
if t.endpoint == "" {
|
||||
return "", fmt.Errorf("tika endpoint not configured")
|
||||
}
|
||||
|
||||
endpoint := strings.TrimRight(t.endpoint, "/") + "/tika"
|
||||
resp := t.client.Request(
|
||||
"PUT",
|
||||
endpoint,
|
||||
reader,
|
||||
request.WithHeader(map[string][]string{
|
||||
"Accept": {"text/plain"},
|
||||
}),
|
||||
)
|
||||
if resp.Err != nil {
|
||||
return "", fmt.Errorf("tika request failed: %w", resp.Err)
|
||||
}
|
||||
defer resp.Response.Body.Close()
|
||||
|
||||
if resp.Response.StatusCode != 200 {
|
||||
return "", fmt.Errorf("tika returned status %d", resp.Response.StatusCode)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(resp.Response.Body)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to read tika response: %w", err)
|
||||
}
|
||||
|
||||
return strings.TrimSpace(string(body)), nil
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package searcher
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
)
|
||||
|
||||
type SearchDocument struct {
|
||||
ID string `json:"id"`
|
||||
FileID int `json:"file_id"`
|
||||
OwnerID int `json:"owner_id"`
|
||||
EntityID int `json:"entity_id"`
|
||||
ChunkIdx int `json:"chunk_idx"`
|
||||
FileName string `json:"file_name"`
|
||||
Text string `json:"text"`
|
||||
Formated *FormatedHit `json:"_formatted,omitempty"`
|
||||
}
|
||||
|
||||
type FormatedHit struct {
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
type SearchResult struct {
|
||||
FileID int `json:"file_id"`
|
||||
OwnerID int `json:"owner_id"`
|
||||
EntityID int `json:"entity_id"`
|
||||
FileName string `json:"file_name"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
type SearchIndexer interface {
|
||||
IndexFile(ctx context.Context, ownerID, fileID, entityID int, fileName, text string) error
|
||||
DeleteByFileIDs(ctx context.Context, fileID ...int) error
|
||||
ChangeOwner(ctx context.Context, fileID, oldOwnerID, newOwnerID int) error
|
||||
CopyByFileID(ctx context.Context, srcFileID, dstFileID, dstOwnerID, dstEntityID int) error
|
||||
Rename(ctx context.Context, fileID, entityID int, newFileName string) error
|
||||
Search(ctx context.Context, ownerID int, query string, offset int) ([]SearchResult, int64, error)
|
||||
// IndexReady reports whether the search index exists and has the required
|
||||
// configuration (filterable/searchable attributes, etc.).
|
||||
IndexReady(ctx context.Context) (bool, error)
|
||||
EnsureIndex(ctx context.Context) error
|
||||
// DeleteAll removes all documents from the index.
|
||||
DeleteAll(ctx context.Context) error
|
||||
Close() error
|
||||
}
|
||||
|
||||
type TextExtractor interface {
|
||||
Exts() []string
|
||||
MaxFileSize() int64
|
||||
Extract(ctx context.Context, reader io.Reader) (string, error)
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
package indexer
|
||||
|
||||
import "strings"
|
||||
|
||||
const defaultMaxBytes = 2000
|
||||
|
||||
// ChunkText splits text into chunks of approximately maxBytes bytes each.
|
||||
// It splits on paragraph breaks (\n\n), combines small paragraphs until the
|
||||
// byte limit is reached, and splits large paragraphs at word boundaries.
|
||||
func ChunkText(text string, maxBytes int) []string {
|
||||
if maxBytes <= 0 {
|
||||
maxBytes = defaultMaxBytes
|
||||
}
|
||||
|
||||
text = strings.TrimSpace(text)
|
||||
if text == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
paragraphs := strings.Split(text, "\n\n")
|
||||
var chunks []string
|
||||
var current []string
|
||||
currentBytes := 0
|
||||
|
||||
for _, para := range paragraphs {
|
||||
para = strings.TrimSpace(para)
|
||||
if para == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
paraBytes := len(para)
|
||||
|
||||
// If a single paragraph exceeds maxBytes, split it at word boundaries
|
||||
if paraBytes > maxBytes {
|
||||
// Flush accumulated content first
|
||||
if currentBytes > 0 {
|
||||
chunks = append(chunks, strings.Join(current, "\n\n"))
|
||||
current = nil
|
||||
currentBytes = 0
|
||||
}
|
||||
chunks = append(chunks, splitByBytes(para, maxBytes)...)
|
||||
continue
|
||||
}
|
||||
|
||||
// If adding this paragraph (plus separator) would exceed the limit, flush
|
||||
joinerLen := 0
|
||||
if currentBytes > 0 {
|
||||
joinerLen = 2 // "\n\n"
|
||||
}
|
||||
if currentBytes+joinerLen+paraBytes > maxBytes && currentBytes > 0 {
|
||||
chunks = append(chunks, strings.Join(current, "\n\n"))
|
||||
current = nil
|
||||
currentBytes = 0
|
||||
}
|
||||
|
||||
if currentBytes > 0 {
|
||||
currentBytes += 2 // account for "\n\n" joiner
|
||||
}
|
||||
current = append(current, para)
|
||||
currentBytes += paraBytes
|
||||
}
|
||||
|
||||
// Flush remaining
|
||||
if currentBytes > 0 {
|
||||
chunks = append(chunks, strings.Join(current, "\n\n"))
|
||||
}
|
||||
|
||||
return chunks
|
||||
}
|
||||
|
||||
// splitByBytes splits text into chunks at word boundaries, each at most maxBytes bytes.
|
||||
func splitByBytes(text string, maxBytes int) []string {
|
||||
words := strings.Fields(text)
|
||||
var chunks []string
|
||||
var current []string
|
||||
currentBytes := 0
|
||||
|
||||
for _, w := range words {
|
||||
wLen := len(w)
|
||||
spaceLen := 0
|
||||
if currentBytes > 0 {
|
||||
spaceLen = 1
|
||||
}
|
||||
|
||||
if currentBytes+spaceLen+wLen > maxBytes && currentBytes > 0 {
|
||||
chunks = append(chunks, strings.Join(current, " "))
|
||||
current = nil
|
||||
currentBytes = 0
|
||||
spaceLen = 0
|
||||
}
|
||||
|
||||
current = append(current, w)
|
||||
currentBytes += spaceLen + wLen
|
||||
}
|
||||
|
||||
if len(current) > 0 {
|
||||
chunks = append(chunks, strings.Join(current, " "))
|
||||
}
|
||||
|
||||
return chunks
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package indexer
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestChunkText_Empty(t *testing.T) {
|
||||
assert.Nil(t, ChunkText("", 500))
|
||||
assert.Nil(t, ChunkText(" ", 500))
|
||||
}
|
||||
|
||||
func TestChunkText_SingleSmallParagraph(t *testing.T) {
|
||||
chunks := ChunkText("Hello world", 500)
|
||||
assert.Equal(t, []string{"Hello world"}, chunks)
|
||||
}
|
||||
|
||||
func TestChunkText_MultipleParagraphsCombined(t *testing.T) {
|
||||
text := "First paragraph.\n\nSecond paragraph.\n\nThird paragraph."
|
||||
chunks := ChunkText(text, 500)
|
||||
assert.Len(t, chunks, 1)
|
||||
assert.Contains(t, chunks[0], "First paragraph.")
|
||||
assert.Contains(t, chunks[0], "Third paragraph.")
|
||||
}
|
||||
|
||||
func TestChunkText_SplitOnParagraphBoundary(t *testing.T) {
|
||||
// Create two paragraphs each ~300 bytes
|
||||
para := strings.Repeat("abcde ", 50) // 300 bytes
|
||||
para = strings.TrimSpace(para)
|
||||
text := para + "\n\n" + para
|
||||
|
||||
chunks := ChunkText(text, 500)
|
||||
assert.Len(t, chunks, 2)
|
||||
}
|
||||
|
||||
func TestChunkText_LargeParagraphSplit(t *testing.T) {
|
||||
// Create a single paragraph of 1200 bytes (240 words * 5 bytes each)
|
||||
words := make([]string, 240)
|
||||
for i := range words {
|
||||
words[i] = "word" // 4 bytes + 1 space = 5 per word
|
||||
}
|
||||
text := strings.Join(words, " ") // 240*4 + 239 = 1199 bytes
|
||||
|
||||
chunks := ChunkText(text, 500)
|
||||
assert.Len(t, chunks, 3)
|
||||
assert.LessOrEqual(t, len(chunks[0]), 500)
|
||||
assert.LessOrEqual(t, len(chunks[1]), 500)
|
||||
assert.Greater(t, len(chunks[2]), 0)
|
||||
}
|
||||
|
||||
func TestChunkText_DefaultMaxBytes(t *testing.T) {
|
||||
chunks := ChunkText("hello", 0)
|
||||
assert.Equal(t, []string{"hello"}, chunks)
|
||||
}
|
||||
|
||||
func TestChunkText_EmptyParagraphsIgnored(t *testing.T) {
|
||||
text := "First.\n\n\n\n\n\nSecond."
|
||||
chunks := ChunkText(text, 500)
|
||||
assert.Len(t, chunks, 1)
|
||||
assert.Equal(t, "First.\n\nSecond.", chunks[0])
|
||||
}
|
||||
|
||||
func TestChunkText_ParagraphKeptWhole(t *testing.T) {
|
||||
// A paragraph under the limit should not be split
|
||||
para := strings.Repeat("x", 400)
|
||||
chunks := ChunkText(para, 500)
|
||||
assert.Len(t, chunks, 1)
|
||||
assert.Equal(t, para, chunks[0])
|
||||
}
|
||||
|
||||
func TestChunkText_JoinerAccountedInLimit(t *testing.T) {
|
||||
// Two paragraphs that fit individually but exceed the limit when joined with "\n\n"
|
||||
p1 := strings.Repeat("a", 250)
|
||||
p2 := strings.Repeat("b", 250)
|
||||
text := p1 + "\n\n" + p2
|
||||
|
||||
chunks := ChunkText(text, 500)
|
||||
// 250 + 2 + 250 = 502 > 500, so they should be split
|
||||
assert.Len(t, chunks, 2)
|
||||
assert.Equal(t, p1, chunks[0])
|
||||
assert.Equal(t, p2, chunks[1])
|
||||
}
|
||||
@@ -0,0 +1,394 @@
|
||||
package indexer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/logging"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/searcher"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/setting"
|
||||
"github.com/meilisearch/meilisearch-go"
|
||||
)
|
||||
|
||||
const (
|
||||
indexName = "cloudreve_files"
|
||||
embedderName = "cr-text"
|
||||
embeddingTemplate = "Chunk #{{doc.chunk_idx}} in a file named '{{doc.file_name}}': {{ doc.text }}"
|
||||
)
|
||||
|
||||
// MeilisearchIndexer implements SearchIndexer using Meilisearch.
|
||||
type MeilisearchIndexer struct {
|
||||
client meilisearch.ServiceManager
|
||||
l logging.Logger
|
||||
pageSize int
|
||||
chunkSize int
|
||||
cfg *setting.FTSIndexMeilisearchSetting
|
||||
}
|
||||
|
||||
// NewMeilisearchIndexer creates a new MeilisearchIndexer.
|
||||
func NewMeilisearchIndexer(msCfg *setting.FTSIndexMeilisearchSetting, chunkSize int, l logging.Logger) *MeilisearchIndexer {
|
||||
client := meilisearch.New(msCfg.Endpoint, meilisearch.WithAPIKey(msCfg.APIKey))
|
||||
return &MeilisearchIndexer{
|
||||
client: client,
|
||||
l: l,
|
||||
pageSize: msCfg.PageSize,
|
||||
chunkSize: chunkSize,
|
||||
cfg: msCfg,
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
requiredFilterable = []string{"owner_id", "file_id", "entity_id"}
|
||||
requiredSearchable = []string{"text", "file_name"}
|
||||
requiredDistinct = "file_id"
|
||||
)
|
||||
|
||||
func (m *MeilisearchIndexer) IndexReady(ctx context.Context) (bool, error) {
|
||||
index := m.client.Index(indexName)
|
||||
|
||||
settings, err := index.GetSettingsWithContext(ctx)
|
||||
if err != nil {
|
||||
// If the index doesn't exist, Meilisearch returns an error.
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// Check filterable attributes.
|
||||
for _, attr := range requiredFilterable {
|
||||
if !slices.Contains(settings.FilterableAttributes, attr) {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Check searchable attributes.
|
||||
for _, attr := range requiredSearchable {
|
||||
if !slices.Contains(settings.SearchableAttributes, attr) {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Check distinct attribute.
|
||||
if settings.DistinctAttribute == nil || *settings.DistinctAttribute != requiredDistinct {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// Check embedder if embedding is enabled.
|
||||
if m.cfg.EmbeddingEnbaled {
|
||||
if settings.Embedders == nil {
|
||||
return false, nil
|
||||
}
|
||||
if _, ok := settings.Embedders[embedderName]; !ok {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (m *MeilisearchIndexer) EnsureIndex(ctx context.Context) error {
|
||||
_, err := m.client.CreateIndexWithContext(ctx, &meilisearch.IndexConfig{
|
||||
Uid: indexName,
|
||||
PrimaryKey: "id",
|
||||
})
|
||||
if err != nil {
|
||||
m.l.Debug("Create index returned (may already exist): %s", err)
|
||||
}
|
||||
|
||||
index := m.client.Index(indexName)
|
||||
|
||||
filterableAttrs := []any{"owner_id", "file_id", "entity_id"}
|
||||
if _, err := index.UpdateFilterableAttributesWithContext(ctx, &filterableAttrs); err != nil {
|
||||
return fmt.Errorf("failed to set filterable attributes: %w", err)
|
||||
}
|
||||
|
||||
searchableAttrs := []string{"text", "file_name"}
|
||||
if _, err := index.UpdateSearchableAttributesWithContext(ctx, &searchableAttrs); err != nil {
|
||||
return fmt.Errorf("failed to set searchable attributes: %w", err)
|
||||
}
|
||||
|
||||
_, err = index.UpdateDistinctAttributeWithContext(ctx, "file_id")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to set distinct attribute: %w", err)
|
||||
}
|
||||
|
||||
if m.cfg.EmbeddingEnbaled {
|
||||
var embedder meilisearch.Embedder
|
||||
if err := json.Unmarshal([]byte(m.cfg.EmbeddingSetting), &embedder); err != nil {
|
||||
m.cfg.EmbeddingEnbaled = false
|
||||
m.l.Warning("Failed to unmarshal embedding setting: %s, fallback to disable embedding", err)
|
||||
return nil
|
||||
}
|
||||
|
||||
embedder.DocumentTemplate = embeddingTemplate
|
||||
_, err := index.UpdateEmbeddersWithContext(ctx, map[string]meilisearch.Embedder{
|
||||
embedderName: embedder,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to set embedders: %w", err)
|
||||
}
|
||||
} else {
|
||||
_, err := index.ResetEmbeddersWithContext(ctx)
|
||||
if err != nil {
|
||||
m.l.Warning("Failed to reset embedder: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MeilisearchIndexer) IndexFile(ctx context.Context, ownerID, fileID, entityID int, fileName, text string) error {
|
||||
chunks := ChunkText(text, m.chunkSize)
|
||||
if len(chunks) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
docs := make([]searcher.SearchDocument, 0, len(chunks))
|
||||
for i, chunk := range chunks {
|
||||
docs = append(docs, searcher.SearchDocument{
|
||||
ID: fmt.Sprintf("%d_%d", fileID, i),
|
||||
FileID: fileID,
|
||||
OwnerID: ownerID,
|
||||
EntityID: entityID,
|
||||
ChunkIdx: i,
|
||||
FileName: fileName,
|
||||
Text: chunk,
|
||||
})
|
||||
}
|
||||
|
||||
index := m.client.Index(indexName)
|
||||
pk := "id"
|
||||
if _, err := index.AddDocumentsWithContext(ctx, docs, &meilisearch.DocumentOptions{PrimaryKey: &pk}); err != nil {
|
||||
return fmt.Errorf("failed to add documents: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MeilisearchIndexer) DeleteByFileIDs(ctx context.Context, fileID ...int) error {
|
||||
if len(fileID) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
index := m.client.Index(indexName)
|
||||
strs := make([]string, len(fileID))
|
||||
for i, id := range fileID {
|
||||
strs[i] = fmt.Sprintf("%d", id)
|
||||
}
|
||||
filter := fmt.Sprintf("file_id IN [%s]", strings.Join(strs, ", "))
|
||||
if _, err := index.DeleteDocumentsByFilterWithContext(ctx, filter, nil); err != nil {
|
||||
return fmt.Errorf("failed to delete documents by file_ids: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MeilisearchIndexer) ChangeOwner(ctx context.Context, fileID, oldOwnerID, newOwnerID int) error {
|
||||
index := m.client.Index(indexName)
|
||||
filter := fmt.Sprintf("file_id = %d AND owner_id = %d", fileID, oldOwnerID)
|
||||
|
||||
// Fetch all existing document chunks in batches.
|
||||
const batchSize int64 = 100
|
||||
var allDocs []searcher.SearchDocument
|
||||
for offset := int64(0); ; offset += batchSize {
|
||||
var result meilisearch.DocumentsResult
|
||||
if err := index.GetDocumentsWithContext(ctx, &meilisearch.DocumentsQuery{
|
||||
Filter: filter,
|
||||
Limit: batchSize,
|
||||
Offset: offset,
|
||||
}, &result); err != nil {
|
||||
return fmt.Errorf("failed to get documents: %w", err)
|
||||
}
|
||||
|
||||
for _, hit := range result.Results {
|
||||
var doc searcher.SearchDocument
|
||||
if err := hit.DecodeInto(&doc); err != nil {
|
||||
m.l.Warning("Failed to decode document during owner change: %s", err)
|
||||
continue
|
||||
}
|
||||
allDocs = append(allDocs, doc)
|
||||
}
|
||||
|
||||
if int64(len(result.Results)) < batchSize {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if len(allDocs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Update owner_id in place — primary key is {fileID}_{chunkIdx} so it stays the same.
|
||||
for i := range allDocs {
|
||||
allDocs[i].OwnerID = newOwnerID
|
||||
}
|
||||
|
||||
if _, err := index.UpdateDocumentsInBatchesWithContext(ctx, allDocs, 100, nil); err != nil {
|
||||
return fmt.Errorf("failed to update documents with new owner: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MeilisearchIndexer) CopyByFileID(ctx context.Context, srcFileID, dstFileID, dstOwnerID, dstEntityID int) error {
|
||||
index := m.client.Index(indexName)
|
||||
filter := fmt.Sprintf("file_id = %d", srcFileID)
|
||||
|
||||
const batchSize int64 = 100
|
||||
var allDocs []searcher.SearchDocument
|
||||
for offset := int64(0); ; offset += batchSize {
|
||||
var result meilisearch.DocumentsResult
|
||||
if err := index.GetDocumentsWithContext(ctx, &meilisearch.DocumentsQuery{
|
||||
Filter: filter,
|
||||
Limit: batchSize,
|
||||
Offset: offset,
|
||||
}, &result); err != nil {
|
||||
return fmt.Errorf("failed to get source documents: %w", err)
|
||||
}
|
||||
|
||||
for _, hit := range result.Results {
|
||||
var doc searcher.SearchDocument
|
||||
if err := hit.DecodeInto(&doc); err != nil {
|
||||
m.l.Warning("Failed to decode document during copy: %s", err)
|
||||
continue
|
||||
}
|
||||
allDocs = append(allDocs, doc)
|
||||
}
|
||||
|
||||
if int64(len(result.Results)) < batchSize {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if len(allDocs) == 0 {
|
||||
return fmt.Errorf("no source documents found for file %d", srcFileID)
|
||||
}
|
||||
|
||||
for i := range allDocs {
|
||||
if allDocs[i].EntityID != dstEntityID {
|
||||
m.l.Warning("Entity id mismatch for file %d, original: %d, destination: %d", srcFileID, allDocs[i].EntityID, dstEntityID)
|
||||
continue
|
||||
}
|
||||
|
||||
allDocs[i].ID = fmt.Sprintf("%d_%d", dstFileID, allDocs[i].ChunkIdx)
|
||||
allDocs[i].FileID = dstFileID
|
||||
allDocs[i].OwnerID = dstOwnerID
|
||||
allDocs[i].EntityID = dstEntityID
|
||||
}
|
||||
|
||||
if len(allDocs) == 0 {
|
||||
return fmt.Errorf("no source documents found for file %d", srcFileID)
|
||||
}
|
||||
|
||||
pk := "id"
|
||||
if _, err := index.AddDocumentsWithContext(ctx, allDocs, &meilisearch.DocumentOptions{PrimaryKey: &pk}); err != nil {
|
||||
return fmt.Errorf("failed to add copied documents: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MeilisearchIndexer) Rename(ctx context.Context, fileID, entityID int, newFileName string) error {
|
||||
index := m.client.Index(indexName)
|
||||
filter := fmt.Sprintf("file_id = %d AND entity_id = %d", fileID, entityID)
|
||||
|
||||
const batchSize int64 = 100
|
||||
var allDocs []searcher.SearchDocument
|
||||
for offset := int64(0); ; offset += batchSize {
|
||||
var result meilisearch.DocumentsResult
|
||||
if err := index.GetDocumentsWithContext(ctx, &meilisearch.DocumentsQuery{
|
||||
Filter: filter,
|
||||
Limit: batchSize,
|
||||
Offset: offset,
|
||||
}, &result); err != nil {
|
||||
return fmt.Errorf("failed to get documents for rename: %w", err)
|
||||
}
|
||||
|
||||
for _, hit := range result.Results {
|
||||
var doc searcher.SearchDocument
|
||||
if err := hit.DecodeInto(&doc); err != nil {
|
||||
m.l.Warning("Failed to decode document during rename: %s", err)
|
||||
continue
|
||||
}
|
||||
doc.FileName = newFileName
|
||||
allDocs = append(allDocs, doc)
|
||||
}
|
||||
|
||||
if int64(len(result.Results)) < batchSize {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if len(allDocs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
if _, err := index.UpdateDocumentsInBatchesWithContext(ctx, allDocs, 100, nil); err != nil {
|
||||
return fmt.Errorf("failed to update documents with new file name: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MeilisearchIndexer) Search(ctx context.Context, ownerID int, query string, offset int) ([]searcher.SearchResult, int64, error) {
|
||||
index := m.client.Index(indexName)
|
||||
|
||||
searchReq := &meilisearch.SearchRequest{
|
||||
Filter: fmt.Sprintf("owner_id = %d", ownerID),
|
||||
Limit: int64(m.pageSize),
|
||||
Offset: int64(offset),
|
||||
AttributesToHighlight: []string{"text"},
|
||||
}
|
||||
|
||||
if m.cfg.EmbeddingEnbaled {
|
||||
searchReq.Hybrid = &meilisearch.SearchRequestHybrid{
|
||||
Embedder: embedderName,
|
||||
}
|
||||
}
|
||||
|
||||
resp, err := index.SearchWithContext(ctx, query, searchReq)
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("search failed: %w", err)
|
||||
}
|
||||
|
||||
results := make([]searcher.SearchResult, 0, len(resp.Hits))
|
||||
seen := make(map[int]struct{})
|
||||
for _, hit := range resp.Hits {
|
||||
var doc searcher.SearchDocument
|
||||
if err := hit.DecodeInto(&doc); err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if _, exists := seen[doc.FileID]; exists {
|
||||
continue
|
||||
}
|
||||
seen[doc.FileID] = struct{}{}
|
||||
|
||||
// Extract text from raw JSON for display
|
||||
textStr := doc.Text
|
||||
if doc.Formated != nil {
|
||||
textStr = doc.Formated.Text
|
||||
}
|
||||
|
||||
results = append(results, searcher.SearchResult{
|
||||
FileID: doc.FileID,
|
||||
OwnerID: doc.OwnerID,
|
||||
FileName: doc.FileName,
|
||||
Text: textStr,
|
||||
})
|
||||
}
|
||||
|
||||
return results, resp.EstimatedTotalHits, nil
|
||||
}
|
||||
|
||||
func (m *MeilisearchIndexer) DeleteAll(ctx context.Context) error {
|
||||
index := m.client.Index(indexName)
|
||||
if _, err := index.DeleteAllDocumentsWithContext(ctx, nil); err != nil {
|
||||
return fmt.Errorf("failed to delete all documents: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MeilisearchIndexer) Close() error {
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package indexer
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/searcher"
|
||||
)
|
||||
|
||||
// NoopIndexer is a no-op implementation of SearchIndexer, used when FTS is disabled.
|
||||
type NoopIndexer struct{}
|
||||
|
||||
func (n *NoopIndexer) IndexFile(ctx context.Context, ownerID, fileID, entityID int, fileName, text string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (n *NoopIndexer) DeleteByFileIDs(ctx context.Context, fileID ...int) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (n *NoopIndexer) ChangeOwner(ctx context.Context, fileID, oldOwnerID, newOwnerID int) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (n *NoopIndexer) CopyByFileID(ctx context.Context, srcFileID, dstFileID, dstOwnerID, dstEntityID int) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (n *NoopIndexer) Rename(ctx context.Context, fileID, entityID int, newFileName string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (n *NoopIndexer) Search(ctx context.Context, ownerID int, query string, offset int) ([]searcher.SearchResult, int64, error) {
|
||||
return nil, 0, nil
|
||||
}
|
||||
|
||||
func (n *NoopIndexer) IndexReady(ctx context.Context) (bool, error) {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (n *NoopIndexer) EnsureIndex(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (n *NoopIndexer) DeleteAll(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (n *NoopIndexer) Close() error {
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user