Initial commit for integrating Faiss index as a standalone index per …

…shard

Signed-off-by: Navneet Verma <navneev@amazon.com>

jni/src/faiss_wrapper.cpp

@@ -169,6 +169,33 @@ void knn_jni::faiss_wrapper::InsertToIndex(knn_jni::JNIUtilInterface * jniUtil,
     indexService->insertToIndex(dim, numIds, threadCount, vectorsAddress, ids, index_ptr);
 }
 
+
+void knn_jni::faiss_wrapper::InsertSingleVectorToIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jintArray idsJ, jfloatArray vector, jint dimJ,
+                                         jlong index_ptr, jint threadCount, IndexService* indexService) {
+    if (idsJ == nullptr) {
+        throw std::runtime_error("IDs cannot be null");
+    }
+
+    if(dimJ <= 0) {
+        throw std::runtime_error("Vectors dimensions cannot be less than or equal to 0");
+    }
+
+    // Dimension
+    int dim = (int)dimJ;
+
+    // Number of vectors
+    int numIds = jniUtil->GetJavaIntArrayLength(env, idsJ);
+
+    // Vectors address
+    int64_t vectorsAddress = (int64_t)vectorsAddressJ;
+
+    // Ids
+    auto ids = jniUtil->ConvertJavaIntArrayToCppIntVector(env, idsJ);
+
+    // Create index
+    indexService->insertToIndex(dim, numIds, threadCount, vectorsAddress, ids, index_ptr);
+}
+
 void knn_jni::faiss_wrapper::WriteIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env,
                                          jstring indexPathJ, jlong index_ptr, IndexService* indexService) {
 

src/main/java/org/opensearch/knn/jni/FaissService.java

@@ -33,7 +33,7 @@
  *      src/main/java/org/opensearch/knn/index/query/KNNQueryResult.java
  *      src/main/java/org/opensearch/knn/common/KNNConstants.java
  */
-class FaissService {
+public class FaissService {
 
     static {
         AccessController.doPrivileged((PrivilegedAction<Void>) () -> {

src/main/java/org/opensearch/knn/service/FaissIndex.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.knn.service;
+
+import org.opensearch.knn.jni.FaissService;
+
+import java.util.Map;
+
+public class FaissIndex {
+
+    private long memoryAddress = 0;
+
+    // We should be calling this only once. Ideally we should call this once when the shard is getting created.
+    // We will figure out a way to do this, later
+    public void initIndex(int dim, Map<String, Object> parameters) {
+        memoryAddress = FaissService.initIndex(0, dim , parameters);
+    }
+
+    public void indexData(int docId, float[] vector) {
+
+    }
+
+    public void searchIndex(float[] queryVector, int k) {
+
+    }
+
+
+
+}

src/main/java/org/opensearch/knn/service/VectorEngineService.java

@@ -5,9 +5,7 @@
 
 package org.opensearch.knn.service;
 
-import lombok.AccessLevel;
 import lombok.Builder;
-import lombok.NoArgsConstructor;
 import lombok.Value;
 import lombok.extern.log4j.Log4j2;
 import org.opensearch.knn.index.SpaceType;
@@ -22,14 +20,14 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 
-@NoArgsConstructor(access = AccessLevel.PRIVATE)
 @Log4j2
 public class VectorEngineService {
     private final Map<OSLuceneDocId, Integer> luceneDocIdToVectorEngineDocId = new ConcurrentHashMap<>();
     // Keeping the OSLuceneDocIds as list to ensure that if IndexReader is open we can reuse the docIds
     private final Map<Integer, List<OSLuceneDocId>> vectorEngineDocIdToLuceneDocId = new ConcurrentHashMap<>();
     private final Map<Integer, float[]> vectorEngineDocIdToVector = new ConcurrentHashMap<>();
     private final AtomicInteger currentVectorDocId = new AtomicInteger(0);
+    private final FaissIndex faissIndex;
 
     private static VectorEngineService INSTANCE = null;
 
@@ -40,12 +38,20 @@ public static VectorEngineService getInstance() {
         return INSTANCE;
     }
 
+    private VectorEngineService() {
+        faissIndex = new FaissIndex();
+    }
+
+
     public void ingestData(final OSLuceneDocId luceneDocId, float[] vector, final SpaceType spaceType) {
         log.debug("SpaceType during ingestion is : {}", spaceType);
         luceneDocIdToVectorEngineDocId.put(luceneDocId, currentVectorDocId.intValue());
         int currentDocId = currentVectorDocId.intValue();
         vectorEngineDocIdToLuceneDocId.getOrDefault(currentDocId, Collections.synchronizedList(new LinkedList<>())).add(luceneDocId);
+        // we should remove this
         vectorEngineDocIdToVector.put(currentDocId, vector);
+        // Here we should have a Faiss Index, before we increment the currentVectorDocId.
+        faissIndex.indexData(currentDocId, vector);
         currentVectorDocId.incrementAndGet();
     }