weaviate: add deduplication functionality to weaviate

The changes introduce a deduplication function to the vectorstores
options. This function is used when adding documents to the store,
allowing the system to filter out duplicate documents before they
are added. This is particularly useful to prevent wasting time on
creating an embedding when one already exists.

The changes also include a test case for the new deduplication
functionality, ensuring that it works as expected. The test case
adds two documents to the store, one of which is a duplicate. The
test verifies that only the unique document is added to the store.

vectorstores/options.go

@@ -1,6 +1,11 @@
 package vectorstores
 
-import "github.com/tmc/langchaingo/embeddings"
+import (
+	"context"
+
+	"github.com/tmc/langchaingo/embeddings"
+	"github.com/tmc/langchaingo/schema"
+)
 
 // Option is a function that configures an Options.
 type Option func(*Options)
@@ -11,6 +16,7 @@ type Options struct {
 	ScoreThreshold float32
 	Filters        any
 	Embedder       embeddings.Embedder
+	Deduplicater   func(context.Context, schema.Document) bool
 }
 
 // WithNameSpace returns an Option for setting the name space.
@@ -44,3 +50,12 @@ func WithEmbedder(embedder embeddings.Embedder) Option {
 		o.Embedder = embedder
 	}
 }
+
+// WithDeduplicater returns an Option for setting the deduplicater that could be used
+// when adding documents. This is useful to prevent wasting time on creating an embedding
+// when one already exists.
+func WithDeduplicater(fn func(ctx context.Context, doc schema.Document) bool) Option {
+	return func(o *Options) {
+		o.Deduplicater = fn
+	}
+}

vectorstores/weaviate/weaviate.go

@@ -95,6 +95,22 @@ func (s Store) AddDocuments(ctx context.Context,
 	opts := s.getOptions(options...)
 	nameSpace := s.getNameSpace(opts)
 
+	if opts.Deduplicater != nil {
+		filtered := make([]schema.Document, 0, len(docs))
+		for _, doc := range docs {
+			if !opts.Deduplicater(ctx, doc) {
+				filtered = append(filtered, doc)
+			}
+		}
+		docs = filtered
+	}
+
+	if len(docs) == 0 {
+		// nothing to add (perhaps all documents were duplicates). This is not
+		// an error.
+		return nil, nil
+	}
+
 	texts := make([]string, 0, len(docs))
 	for _, doc := range docs {
 		texts = append(texts, doc.PageContent)

vectorstores/weaviate/weaviate_test.go

@@ -196,6 +196,49 @@ func TestMetadataSearch(t *testing.T) {
 	require.Equal(t, "city", docs[0].Metadata["type"])
 }
 
+func TestDeduplicater(t *testing.T) {
+	t.Parallel()
+
+	scheme, host := getValues(t)
+	llm, err := openai.New()
+	require.NoError(t, err)
+	e, err := embeddings.NewEmbedder(llm)
+	require.NoError(t, err)
+
+	store, err := New(
+		WithScheme(scheme),
+		WithHost(host),
+		WithEmbedder(e),
+		WithNameSpace(uuid.New().String()),
+		WithIndexName(randomizedCamelCaseClass()),
+		WithQueryAttrs([]string{"type"}),
+	)
+	require.NoError(t, err)
+
+	err = createTestClass(context.Background(), store)
+	require.NoError(t, err)
+
+	_, err = store.AddDocuments(context.Background(), []schema.Document{
+		{PageContent: "tokyo", Metadata: map[string]any{
+			"type": "city",
+		}},
+		{PageContent: "potato", Metadata: map[string]any{
+			"type": "vegetable",
+		}},
+	}, vectorstores.WithDeduplicater(
+		func(ctx context.Context, doc schema.Document) bool {
+			return doc.PageContent == "tokyo"
+		},
+	))
+	require.NoError(t, err)
+
+	docs, err := store.MetadataSearch(context.Background(), 2)
+	require.NoError(t, err)
+	require.Len(t, docs, 1)
+	require.Equal(t, "potato", docs[0].PageContent)
+	require.Equal(t, "vegetable", docs[0].Metadata["type"])
+}
+
 func TestSimilaritySearchWithInvalidScoreThreshold(t *testing.T) {
 	t.Parallel()