Add filter-based deletion to Redis vector store

Add string-based filter deletion alongside the Filter.Expression-based deletion for Redis vector store, providing consistent deletion capabilities with other vector store implementations. Key changes: - Add delete(Filter.Expression) implementation using Redis FT.SEARCH and JSON.DEL - Configure metadata fields properly to support numeric and tag operations - Support both simple and complex filter expressions - Handle Redis-specific JSON string responses in tests - Add comprehensive integration tests for filter deletion cases This maintains consistency with other vector store implementations while utilizing Redis Search capabilities for efficient metadata-based deletion. Signed-off-by: Soby Chacko <[email protected]>
spring-projects · Jan 28, 2025 · 886a5e1 · 886a5e1
1 parent c5dd768
commit 886a5e1
Show file tree

Hide file tree

Showing 2 changed files with 134 additions and 2 deletions.
diff --git a/...-redis-store/src/main/java/org/springframework/ai/vectorstore/redis/RedisVectorStore.java b/...-redis-store/src/main/java/org/springframework/ai/vectorstore/redis/RedisVectorStore.java
@@ -53,6 +53,7 @@
 import org.springframework.ai.vectorstore.AbstractVectorStoreBuilder;
 import org.springframework.ai.vectorstore.SearchRequest;
 import org.springframework.ai.vectorstore.VectorStore;
+import org.springframework.ai.vectorstore.filter.Filter;
 import org.springframework.ai.vectorstore.filter.FilterExpressionConverter;
 import org.springframework.ai.vectorstore.observation.AbstractObservationVectorStore;
 import org.springframework.ai.vectorstore.observation.VectorStoreObservationContext;
@@ -296,6 +297,45 @@ public Optional<Boolean> doDelete(List<String> idList) {
 		}
 	}
 
+	@Override
+	protected void doDelete(Filter.Expression filterExpression) {
+		Assert.notNull(filterExpression, "Filter expression must not be null");
+
+		try {
+			String filterStr = this.filterExpressionConverter.convertExpression(filterExpression);
+
+			List<String> matchingIds = new ArrayList<>();
+			SearchResult searchResult = this.jedis.ftSearch(this.indexName, filterStr);
+
+			for (redis.clients.jedis.search.Document doc : searchResult.getDocuments()) {
+				String docId = doc.getId();
+				matchingIds.add(docId.replace(key(""), "")); // Remove the key prefix to
+																// get original ID
+			}
+
+			if (!matchingIds.isEmpty()) {
+				try (Pipeline pipeline = this.jedis.pipelined()) {
+					for (String id : matchingIds) {
+						pipeline.jsonDel(key(id));
+					}
+					List<Object> responses = pipeline.syncAndReturnAll();
+					Optional<Object> errResponse = responses.stream().filter(Predicate.not(RESPONSE_DEL_OK)).findAny();
+
+					if (errResponse.isPresent()) {
+						logger.error(() -> "Could not delete document: " + errResponse.get());
+						throw new IllegalStateException("Failed to delete some documents");
+					}
+				}
+
+				logger.debug(() -> "Deleted " + matchingIds.size() + " documents matching filter expression");
+			}
+		}
+		catch (Exception e) {
+			logger.error(e, () -> "Failed to delete documents by filter");
+			throw new IllegalStateException("Failed to delete documents by filter", e);
+		}
+	}
+
 	@Override
 	public List<Document> doSimilaritySearch(SearchRequest request) {
 

diff --git a/...edis-store/src/test/java/org/springframework/ai/vectorstore/redis/RedisVectorStoreIT.java b/...edis-store/src/test/java/org/springframework/ai/vectorstore/redis/RedisVectorStoreIT.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2023-2024 the original author or authors.
+ * Copyright 2023-2025 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -22,6 +22,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
+import java.util.stream.Collectors;
 
 import com.redis.testcontainers.RedisStackContainer;
 import org.junit.jupiter.api.BeforeEach;
@@ -36,6 +37,7 @@
 import org.springframework.ai.transformers.TransformersEmbeddingModel;
 import org.springframework.ai.vectorstore.SearchRequest;
 import org.springframework.ai.vectorstore.VectorStore;
+import org.springframework.ai.vectorstore.filter.Filter;
 import org.springframework.ai.vectorstore.redis.RedisVectorStore.MetadataField;
 import org.springframework.boot.SpringBootConfiguration;
 import org.springframework.boot.autoconfigure.AutoConfigurations;
@@ -53,6 +55,7 @@
  * @author Julien Ruaux
  * @author Eddú Meléndez
  * @author Thomas Vitale
+ * @author Soby Chacko
  */
 @Testcontainers
 class RedisVectorStoreIT {
@@ -260,6 +263,90 @@ void searchWithThreshold() {
 		});
 	}
 
+	@Test
+	void deleteByFilter() {
+		this.contextRunner.run(context -> {
+			VectorStore vectorStore = context.getBean(VectorStore.class);
+
+			var bgDocument = new Document("The World is Big and Salvation Lurks Around the Corner",
+					Map.of("country", "BG", "year", 2020));
+			var nlDocument = new Document("The World is Big and Salvation Lurks Around the Corner",
+					Map.of("country", "NL"));
+			var bgDocument2 = new Document("The World is Big and Salvation Lurks Around the Corner",
+					Map.of("country", "BG", "year", 2023));
+
+			vectorStore.add(List.of(bgDocument, nlDocument, bgDocument2));
+
+			Filter.Expression filterExpression = new Filter.Expression(Filter.ExpressionType.EQ,
+					new Filter.Key("country"), new Filter.Value("BG"));
+
+			vectorStore.delete(filterExpression);
+
+			List<Document> results = vectorStore
+				.similaritySearch(SearchRequest.builder().query("The World").topK(5).similarityThresholdAll().build());
+
+			assertThat(results).hasSize(1);
+			assertThat(results.get(0).getMetadata()).containsEntry("country", "NL");
+		});
+	}
+
+	@Test
+	void deleteWithStringFilterExpression() {
+		this.contextRunner.run(context -> {
+			VectorStore vectorStore = context.getBean(VectorStore.class);
+
+			var bgDocument = new Document("The World is Big and Salvation Lurks Around the Corner",
+					Map.of("country", "BG", "year", 2020));
+			var nlDocument = new Document("The World is Big and Salvation Lurks Around the Corner",
+					Map.of("country", "NL"));
+			var bgDocument2 = new Document("The World is Big and Salvation Lurks Around the Corner",
+					Map.of("country", "BG", "year", 2023));
+
+			vectorStore.add(List.of(bgDocument, nlDocument, bgDocument2));
+
+			vectorStore.delete("country == 'BG'");
+
+			List<Document> results = vectorStore
+				.similaritySearch(SearchRequest.builder().query("The World").topK(5).similarityThresholdAll().build());
+
+			assertThat(results).hasSize(1);
+			assertThat(results.get(0).getMetadata()).containsEntry("country", "NL");
+		});
+	}
+
+	@Test
+	void deleteWithComplexFilterExpression() {
+		this.contextRunner.run(context -> {
+			VectorStore vectorStore = context.getBean(VectorStore.class);
+
+			var doc1 = new Document("Content 1", Map.of("type", "A", "priority", 1));
+			var doc2 = new Document("Content 2", Map.of("type", "A", "priority", 2));
+			var doc3 = new Document("Content 3", Map.of("type", "B", "priority", 1));
+
+			vectorStore.add(List.of(doc1, doc2, doc3));
+
+			// Complex filter expression: (type == 'A' AND priority > 1)
+			Filter.Expression priorityFilter = new Filter.Expression(Filter.ExpressionType.GT,
+					new Filter.Key("priority"), new Filter.Value(1));
+			Filter.Expression typeFilter = new Filter.Expression(Filter.ExpressionType.EQ, new Filter.Key("type"),
+					new Filter.Value("A"));
+			Filter.Expression complexFilter = new Filter.Expression(Filter.ExpressionType.AND, typeFilter,
+					priorityFilter);
+
+			vectorStore.delete(complexFilter);
+
+			var results = vectorStore
+				.similaritySearch(SearchRequest.builder().query("Content").topK(5).similarityThresholdAll().build());
+
+			assertThat(results).hasSize(2);
+			assertThat(results.stream().map(doc -> doc.getMetadata().get("type")).collect(Collectors.toList()))
+				.containsExactlyInAnyOrder("A", "B");
+			assertThat(results.stream()
+				.map(doc -> Integer.parseInt(doc.getMetadata().get("priority").toString()))
+				.collect(Collectors.toList())).containsExactlyInAnyOrder(1, 1);
+		});
+	}
+
 	@SpringBootConfiguration
 	@EnableAutoConfiguration(exclude = { DataSourceAutoConfiguration.class })
 	public static class TestApplication {
@@ -271,7 +358,12 @@ public RedisVectorStore vectorStore(EmbeddingModel embeddingModel,
 				.builder(new JedisPooled(jedisConnectionFactory.getHostName(), jedisConnectionFactory.getPort()),
 						embeddingModel)
 				.metadataFields(MetadataField.tag("meta1"), MetadataField.tag("meta2"), MetadataField.tag("country"),
-						MetadataField.numeric("year"))
+						MetadataField.numeric("year"), MetadataField.numeric("priority"), // Add
+																							// priority
+																							// as
+																							// numeric
+						MetadataField.tag("type") // Add type as tag
+				)
 				.initializeSchema(true)
 				.build();
 		}