From 7e6b5d1a30f22a1b5120b1dc05e8229cde84c26f Mon Sep 17 00:00:00 2001 From: writinwaters <93570324+writinwaters@users.noreply.github.com> Date: Thu, 2 May 2024 12:45:08 +0800 Subject: [PATCH 1/9] Update benchmark.md (#1158) Added versions of ES, qdrant, and infinity ### What problem does this PR solve? _Briefly describe what this PR aims to solve. Include background context that will help reviewers understand the purpose of the PR._ ### Type of change - [x] Documentation Update --- docs/references/benchmark.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/docs/references/benchmark.md b/docs/references/benchmark.md index c592271372..18adc69b63 100644 --- a/docs/references/benchmark.md +++ b/docs/references/benchmark.md @@ -3,6 +3,23 @@ sidebar_position: 1 slug: /benchmark --- # Benchmark +This document compares the following key specifications of Elasticsearch, Qdrant, and Infinity: + +- QPS +- Recall +- Time to insert & build index +- Time to import & build index +- Disk usage +- Peak memory usage + +## Versions +| | QPS | +| ----------------- | --------- | +| **Elasticsearch** | v8.13.0 | +| **Qdrant** | v1.8.2 | +| **Infinity** | v0.1.0 | + +## Run Benchmark 1. Install necessary dependencies. @@ -173,4 +190,4 @@ python remote_benchmark_knn.py -t 16 -r 1 -d gist_1m - **Dataset**: SIFT1M; **topk**: 100; **recall**: 97%+ - **P99 QPS**: 15,688 (16 clients) - **P99 Latency**: 0.36 ms -- **Memory usage**: 408 MB \ No newline at end of file +- **Memory usage**: 408 MB From 4ab73313782a9f898a1042a1026c21d4018cb52d Mon Sep 17 00:00:00 2001 From: Zhichang Yu Date: Thu, 2 May 2024 15:53:19 +0800 Subject: [PATCH 2/9] Let thrift client reconnect on insert failure (#1156) Let thrift client reconnect on insert failure Updated requirements.txt Issue link:#1125 ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue) --- python/benchmark/clients/base_client.py | 23 +++++++++++----- .../benchmark/clients/elasticsearch_client.py | 14 +++++----- python/benchmark/clients/infinity_client.py | 13 +++++---- python/benchmark/clients/qdrant_client.py | 7 ++--- python/benchmark/configs/infinity_enwiki.json | 2 +- python/benchmark/requirements.txt | 4 +++ python/benchmark/run.py | 9 ++++--- python/infinity/remote_thrift/client.py | 27 +++++++++++++------ python/pyproject.toml | 20 +++++++------- python/requirements.txt | 18 ++++++------- 10 files changed, 82 insertions(+), 55 deletions(-) create mode 100644 python/benchmark/requirements.txt diff --git a/python/benchmark/clients/base_client.py b/python/benchmark/clients/base_client.py index 8099ca54c5..83c747885a 100644 --- a/python/benchmark/clients/base_client.py +++ b/python/benchmark/clients/base_client.py @@ -1,12 +1,11 @@ import argparse from abc import abstractmethod -from typing import Any, List, Optional, Dict, Union -from enum import Enum +from typing import Any import subprocess -import sys import os -from urllib.parse import urlparse import time +import logging + class BaseClient: """ @@ -25,6 +24,13 @@ def __init__(self, """ pass + @abstractmethod + def upload(self): + """ + Upload data and build indexes (parameters are parsed by __init__). + """ + pass + @abstractmethod def search(self) -> list[list[Any]]: """ @@ -32,7 +38,7 @@ def search(self) -> list[list[Any]]: The function returns id list. """ pass - + def download_data(self, url, target_path): """ Download dataset and extract it into path. @@ -59,6 +65,11 @@ def run_experiment(self, args): """ run experiment and save results. """ + if args.import_data: + start_time = time.time() + self.upload() + finish_time = time.time() + logging.info(f"upload finish, cost time = {finish_time - start_time}") if args.query: results = self.search() - self.check_and_save_results(results) \ No newline at end of file + self.check_and_save_results(results) diff --git a/python/benchmark/clients/elasticsearch_client.py b/python/benchmark/clients/elasticsearch_client.py index 2d978233b2..0d7cb5a432 100644 --- a/python/benchmark/clients/elasticsearch_client.py +++ b/python/benchmark/clients/elasticsearch_client.py @@ -3,12 +3,12 @@ from elasticsearch import Elasticsearch, helpers import json import time -from typing import List, Optional +from typing import List import os import h5py import uuid import numpy as np -import csv +import logging from .base_client import BaseClient @@ -74,7 +74,7 @@ def upload(self): for i, line in enumerate(data_file): row = line.strip().split('\t') if len(row) != len(headers): - print(f"row = {i}, row_len = {len(row)}, not equal headers len, skip") + logging.info(f"row = {i}, row_len = {len(row)}, not equal headers len, skip") continue row_dict = {header: value for header, value in zip(headers, row)} current_batch.append({"_index": self.collection_name, "_id": uuid.UUID(int=i).hex, "_source": row_dict}) @@ -133,7 +133,7 @@ def search(self) -> list[list[Any]]: The function returns id list. """ query_path = os.path.join(self.path_prefix, self.data["query_path"]) - print(query_path) + logging.info(query_path) results = [] _, ext = os.path.splitext(query_path) if ext == '.json' or ext == '.jsonl': @@ -184,7 +184,7 @@ def search(self) -> list[list[Any]]: latency = (end - start) * 1000 result = [(uuid.UUID(hex=hit['_id']).int, hit['_score']) for hit in result['hits']['hits']] result.append(latency) - print(f"{line[:-1]}, {latency}") + logging.info(f"{line[:-1]}, {latency}") results.append(result) else: raise TypeError("Unsupported file type") @@ -214,7 +214,7 @@ def check_and_save_results(self, results: List[List[Any]]): precisions.append(precision) latencies.append(result[-1]) - print( + logging.info( f'''mean_time: {np.mean(latencies)}, mean_precisions: {np.mean(precisions)}, std_time: {np.std(latencies)}, min_time: {np.min(latencies)}, \n max_time: {np.max(latencies)}, p95_time: {np.percentile(latencies, 95)}, @@ -223,7 +223,7 @@ def check_and_save_results(self, results: List[List[Any]]): latencies = [] for result in results: latencies.append(result[-1]) - print( + logging.info( f'''mean_time: {np.mean(latencies)}, std_time: {np.std(latencies)}, max_time: {np.max(latencies)}, min_time: {np.min(latencies)}, p95_time: {np.percentile(latencies, 95)}, p99_time: {np.percentile(latencies, 99)}''') diff --git a/python/benchmark/clients/infinity_client.py b/python/benchmark/clients/infinity_client.py index 1c202448b9..e7e8e056e8 100644 --- a/python/benchmark/clients/infinity_client.py +++ b/python/benchmark/clients/infinity_client.py @@ -5,13 +5,12 @@ import time import numpy as np from typing import Any, List +import logging import infinity import infinity.index as index from infinity import NetworkAddress from .base_client import BaseClient -import infinity.remote_thrift.infinity_thrift_rpc.ttypes as ttypes -import csv class InfinityClient(BaseClient): def __init__(self, @@ -93,9 +92,9 @@ def upload(self): for i, line in enumerate(data_file): row = line.strip().split('\t') if (i % 100000 == 0): - print(f"row {i}") + logging.info(f"row {i}") if len(row) != len(headers): - print(f"row = {i}, row_len = {len(row)}, not equal headers len, skip") + logging.info(f"row = {i}, row_len = {len(row)}, not equal headers len, skip") continue row_dict = {header: value for header, value in zip(headers, row)} current_batch.append(row_dict) @@ -166,7 +165,7 @@ def search(self) -> list[list[Any]]: latency = (time.time() - start) * 1000 result = [(row_id[0], score) for row_id, score in zip(res['ROW_ID'], res['SCORE'])] result.append(latency) - print(f"{query}, {latency}") + logging.info(f"{query}, {latency}") results.append(result) else: raise TypeError("Unsupported file type") @@ -197,7 +196,7 @@ def check_and_save_results(self, results: List[List[Any]]): precisions.append(precision) latencies.append(result[-1]) - print( + logging.info( f'''mean_time: {np.mean(latencies)}, mean_precisions: {np.mean(precisions)}, std_time: {np.std(latencies)}, min_time: {np.min(latencies)}, max_time: {np.max(latencies)}, p95_time: {np.percentile(latencies, 95)}, @@ -206,7 +205,7 @@ def check_and_save_results(self, results: List[List[Any]]): latencies = [] for result in results: latencies.append(result[-1]) - print( + logging.info( f'''mean_time: {np.mean(latencies)}, std_time: {np.std(latencies)}, max_time: {np.max(latencies)}, min_time: {np.min(latencies)}, p95_time: {np.percentile(latencies, 95)}, p99_time: {np.percentile(latencies, 99)}''') diff --git a/python/benchmark/clients/qdrant_client.py b/python/benchmark/clients/qdrant_client.py index 43c5273271..db78665849 100644 --- a/python/benchmark/clients/qdrant_client.py +++ b/python/benchmark/clients/qdrant_client.py @@ -6,7 +6,8 @@ import time import json import h5py -from typing import Any, List, Optional +from typing import Any +import logging from .base_client import BaseClient @@ -137,7 +138,7 @@ def search(self) -> list[list[Any]]: with_payload=False ) end = time.time() - print(f"latency of search: {(end - start)*1000:.2f} milliseconds") + logging.info(f"latency of search: {(end - start)*1000:.2f} milliseconds") results.append(result) elif ext == '.hdf5' and self.data['mode'] == 'vector': with h5py.File(query_path, 'r') as f: @@ -150,7 +151,7 @@ def search(self) -> list[list[Any]]: ) results.append(result) end = time.time() - print(f"latency of KNN search: {(end - start)*1000/len(f['test']):.2f} milliseconds") + logging.info(f"latency of KNN search: {(end - start)*1000/len(f['test']):.2f} milliseconds") else: raise TypeError("Unsupported file type") return results diff --git a/python/benchmark/configs/infinity_enwiki.json b/python/benchmark/configs/infinity_enwiki.json index 235a84c0a4..f69ebb25cf 100644 --- a/python/benchmark/configs/infinity_enwiki.json +++ b/python/benchmark/configs/infinity_enwiki.json @@ -9,7 +9,7 @@ "query_link": "to_be_set", "mode": "fulltext", "topK": 10, - "use_import": true, + "use_import": false, "schema": { "doctitle": {"type": "varchar", "default":""}, "docdate": {"type": "varchar", "default":""}, diff --git a/python/benchmark/requirements.txt b/python/benchmark/requirements.txt new file mode 100644 index 0000000000..7c1abfcadc --- /dev/null +++ b/python/benchmark/requirements.txt @@ -0,0 +1,4 @@ +elasticsearch~=8.13.0 +h5py~=3.11.0 +qdrant_client~=1.9.0 + diff --git a/python/benchmark/run.py b/python/benchmark/run.py index a96360fa8d..7f13f665d7 100644 --- a/python/benchmark/run.py +++ b/python/benchmark/run.py @@ -1,10 +1,10 @@ import argparse import os +import logging from clients.elasticsearch_client import ElasticsearchClient from clients.infinity_client import InfinityClient from clients.qdrant_client import QdrantClient -from generate_query_json import generate_query_json from generate_query_json import generate_query_txt ENGINES = ['infinity', 'qdrant', 'elasticsearch'] @@ -53,6 +53,7 @@ def get_client(engine: str, config: str, options: argparse.Namespace): raise ValueError(f"Unknown engine: {engine}") if __name__ == '__main__': + logging.basicConfig(level=logging.INFO, format='%(asctime)-15s %(levelname)-8s %(message)s') args = parse_args() config_paths = generate_config_paths(args) @@ -65,9 +66,9 @@ def get_client(engine: str, config: str, options: argparse.Namespace): for path, engine in config_paths: if not os.path.exists(path): - print(f"qdrant does not support full text search") + logging.info("qdrant does not support full text search") continue - print("Running", engine, "with", path) + logging.info("Running {} with {}".format(engine, path)) client = get_client(engine, path, args) client.run_experiment(args) - print("Finished", engine, "with", path) + logging.info("Finished {} with {}".format(engine, path)) diff --git a/python/infinity/remote_thrift/client.py b/python/infinity/remote_thrift/client.py index c883eca40c..2696bf85ac 100644 --- a/python/infinity/remote_thrift/client.py +++ b/python/infinity/remote_thrift/client.py @@ -27,9 +27,13 @@ class ThriftInfinityClient: def __init__(self, uri: URI): self.uri = uri + self.transport = None self.reconnect() def reconnect(self): + if self.transport is not None: + self.transport.close() + self.transport = None # self.transport = TTransport.TFramedTransport(TSocket.TSocket(self.uri.ip, self.uri.port)) # async self.transport = TTransport.TBufferedTransport( TSocket.TSocket(self.uri.ip, self.uri.port)) # sync @@ -126,7 +130,8 @@ def list_indexes(self, db_name: str, table_name: str): def insert(self, db_name: str, table_name: str, column_names: list[str], fields: list[Field]): retry = 0 - while retry <= 10: + inner_ex = None + while retry <= 2: try: res = self.client.Insert(InsertRequest(session_id=self.session_id, db_name=db_name, @@ -135,12 +140,14 @@ def insert(self, db_name: str, table_name: str, column_names: list[str], fields: fields=fields)) return res except TTransportException as ex: - if ex.type == ex.END_OF_FILE: - self.reconnect() - retry += 1 - else: - break - return CommonResponse(ErrorCode.TOO_MANY_CONNECTIONS, "retry insert failed") + #import traceback + #traceback.print_exc() + self.reconnect() + inner_ex = ex + retry += 1 + except Exception as ex: + inner_ex = ex + return CommonResponse(ErrorCode.TOO_MANY_CONNECTIONS, "insert failed with exception: " + str(inner_ex)) # Can be used in compact mode # def insert(self, db_name: str, table_name: str, column_names: list[str], fields: list[Field]): @@ -198,7 +205,11 @@ def update(self, db_name: str, table_name: str, where_expr, update_expr_array): update_expr_array=update_expr_array)) def disconnect(self): - res = self.client.Disconnect(CommonRequest(session_id=self.session_id)) + res = None + try: + res = self.client.Disconnect(CommonRequest(session_id=self.session_id)) + except Exception: + pass self.transport.close() return res diff --git a/python/pyproject.toml b/python/pyproject.toml index 47e11b8ebd..7e861930df 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -2,16 +2,16 @@ name = "infinity_sdk" version = "0.1.0" dependencies = [ - "sqlglot==11.7.1", - "pydantic", - "thrift", - "setuptools", - "pytest", - "pandas", - "numpy", - "pyarrow", - "openpyxl", - "polars" + "sqlglot~=11.7.1", + "pydantic~=2.7.1", + "thrift~=0.20.0", + "setuptools~=69.5.1", + "pytest~=8.2.0", + "pandas~=2.2.2", + "numpy~=1.26.4", + "pyarrow~=16.0.0", + "polars~=0.20.23", + "openpyxl~=3.1.2" ] description = "infinity" readme = "README.md" diff --git a/python/requirements.txt b/python/requirements.txt index 46c4cc25d3..fc151a38c6 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,10 +1,10 @@ -sqlglot==11.7.1 -pydantic~=1.10.12 +sqlglot~=11.7.1 +pydantic~=2.7.1 thrift~=0.20.0 -setuptools~=68.0.0 -pytest~=7.4.0 -pandas~=2.1.1 -openpyxl -numpy~=1.26.0 -polars~=0.19.0 -pyarrow +setuptools~=69.5.1 +pytest~=8.2.0 +pandas~=2.2.2 +numpy~=1.26.4 +pyarrow~=16.0.0 +polars~=0.20.23 +openpyxl~=3.1.2 From fde9efdc1abd49dd5c34052b149d3a8644972242 Mon Sep 17 00:00:00 2001 From: Zhichang Yu Date: Thu, 2 May 2024 16:07:55 +0800 Subject: [PATCH 3/9] Updated tests.yml --- .github/workflows/tests.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5cc6f30d82..14eaf37630 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,7 +2,10 @@ name: tests on: push: - branches: [ main, libcxx ] + branches: + - 'main' + - '*.*.*' + - 'libcxx' paths-ignore: - 'docs/**' - '*.md' From d458095c5133d0cef8888458c36ba64fc7579340 Mon Sep 17 00:00:00 2001 From: Zhichang Yu Date: Tue, 30 Apr 2024 17:45:05 +0800 Subject: [PATCH 4/9] Updated slow_test.yml --- .github/workflows/slow_test.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/slow_test.yml b/.github/workflows/slow_test.yml index 585ec8779d..68cf29f5be 100644 --- a/.github/workflows/slow_test.yml +++ b/.github/workflows/slow_test.yml @@ -1,8 +1,6 @@ name: slow_tests on: - schedule: - - cron: '0 21 * * *' # This schedule runs every 21:00:00Z(05:00:00+08:00) # The "create tags" trigger is specifically focused on the creation of new tags, while the "push tags" trigger is activated when tags are pushed, including both new tag creations and updates to existing tags. create: tags: @@ -31,7 +29,7 @@ jobs: if: ${{ !cancelled() && !failure() }} run: | TZ=$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}') - sudo docker rm -f infinity_build && sudo docker run -d --privileged --name infinity_build -e TZ=$TZ -v $PWD:/infinity -v /boot:/boot infiniflow/infinity_builder:ubuntu2310 + sudo docker rm -f infinity_build && sudo docker run -d --privileged --name infinity_build -e TZ=$TZ -v $PWD:/infinity -v /boot:/boot infiniflow/infinity_builder:centos7 - name: Build release version if: ${{ !cancelled() && !failure() }} @@ -43,7 +41,7 @@ jobs: - name: Prepare dataset if: ${{ !cancelled() && !failure() }} - run: sudo mkdir -p test/data/benchmark && sudo ln -s $HOME/benchmark_dataset/dbpedia-entity test/data/benchmark/dbpedia-entity && sudo ln -s $HOME/benchmark_dataset/sift1M test/data/benchmark/sift_1m + run: sudo mkdir -p test/data/benchmark && sudo ln -s $HOME/benchmark/dbpedia-entity test/data/benchmark/dbpedia-entity && sudo ln -s $HOME/benchmark/sift1M test/data/benchmark/sift_1m - name: benchmark test if: ${{ !cancelled() && !failure() }} From 96b770c3739fd1dd6b0f6a5f2f6a981e54d7d21a Mon Sep 17 00:00:00 2001 From: Zhichang Yu Date: Thu, 2 May 2024 17:28:51 +0800 Subject: [PATCH 5/9] Updated README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a66fc0e7e5..82ef6976fa 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ Supports a wide range of data types including strings, numerics, vectors, and mo ```bash sudo mkdir -p /var/infinity && sudo chown -R $USER /var/infinity docker pull infiniflow/infinity:nightly -docker run -d --name infinity -v /var/infinity/:/var/infinity --network=host infiniflow/infinity:nightly +docker run -d --name infinity -v /var/infinity/:/var/infinity --ulimit nofile=500000:500000 --network=host infiniflow/infinity:nightly ``` #### Deploy Infinity using binary package on Linux x86_64 From d2b5e5f3240a1d0cd2ff8ee8cee6935a6815653c Mon Sep 17 00:00:00 2001 From: Jin Hai Date: Fri, 3 May 2024 10:26:43 +0800 Subject: [PATCH 6/9] Fix benchmark doc error (#1159) ### What problem does this PR solve? _Briefly describe what this PR aims to solve. Include background context that will help reviewers understand the purpose of the PR._ ### Type of change - [x] Documentation Update Signed-off-by: Jin Hai --- docs/references/benchmark.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/references/benchmark.md b/docs/references/benchmark.md index 18adc69b63..8ead83d631 100644 --- a/docs/references/benchmark.md +++ b/docs/references/benchmark.md @@ -13,11 +13,11 @@ This document compares the following key specifications of Elasticsearch, Qdrant - Peak memory usage ## Versions -| | QPS | -| ----------------- | --------- | -| **Elasticsearch** | v8.13.0 | -| **Qdrant** | v1.8.2 | -| **Infinity** | v0.1.0 | +| | Version | +| ----------------- |---------| +| **Elasticsearch** | v8.13.0 | +| **Qdrant** | v1.8.2 | +| **Infinity** | v0.1.0 | ## Run Benchmark From f314ccbd75f2dc619044149d3daa26e79ae808ad Mon Sep 17 00:00:00 2001 From: Zhichang Yu Date: Sat, 4 May 2024 19:58:16 +0800 Subject: [PATCH 7/9] Declare fma requirement at CMakeLists.txt (#1162) Declare fma requirement at CMakeLists.txt Issue link:#1161 ### Type of change - [x] Documentation Update --- src/CMakeLists.txt | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 98ac6075e0..4902539d01 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -16,16 +16,26 @@ add_subdirectory(parser) # add_definitions(-msse4.2 -mfma) # add_definitions(-mavx2 -mf16c -mpopcnt) -execute_process(COMMAND grep -q avx2 /proc/cpuinfo +execute_process(COMMAND grep -q fma /proc/cpuinfo + RESULT_VARIABLE SUPPORT_FMA + OUTPUT_QUIET + ERROR_QUIET) + +execute_process(COMMAND grep -q sse4_2 /proc/cpuinfo + RESULT_VARIABLE SUPPORT_SSE42 + OUTPUT_QUIET + ERROR_QUIET) + +execute_process(COMMAND grep -q avx2 /proc/cpuinfo RESULT_VARIABLE SUPPORT_AVX2 - OUTPUT_QUIET - ERROR_QUIET) + OUTPUT_QUIET + ERROR_QUIET) + +execute_process(COMMAND grep -q avx512 /proc/cpuinfo + RESULT_VARIABLE SUPPORT_AVX512 + OUTPUT_QUIET + ERROR_QUIET) -execute_process(COMMAND grep -q avx512 /proc/cpuinfo -RESULT_VARIABLE SUPPORT_AVX512 -OUTPUT_QUIET -ERROR_QUIET) - file(GLOB_RECURSE main_cpp @@ -210,6 +220,14 @@ target_include_directories(infinity_core PUBLIC "${CMAKE_SOURCE_DIR}/third_party target_include_directories(infinity_core PUBLIC "${CMAKE_SOURCE_DIR}/third_party/base64/include") target_include_directories(infinity_core PUBLIC "${CMAKE_SOURCE_DIR}/third_party/oatpp/src") +if (NOT SUPPORT_FMA EQUAL 0) + message(FATAL_ERROR "This project requires the processor support fused multiply-add (FMA) instructions.") +endif() + +if (NOT SUPPORT_SSE42 EQUAL 0) + message(FATAL_ERROR "This project requires the processor support sse4_2 instructions.") +endif() + if (SUPPORT_AVX2 EQUAL 0 OR SUPPORT_AVX512 EQUAL 0) message("Compiled by AVX2 or AVX512") add_definitions(-march=native) From 44722e269fd25341863577f791897304a56224d0 Mon Sep 17 00:00:00 2001 From: Jin Hai Date: Sat, 4 May 2024 22:59:20 +0800 Subject: [PATCH 8/9] Add specific pysdk version to README (#1160) ### What problem does this PR solve? Infinity now have different development branch, the pysdk version in README might be different. ### Type of change - [x] Documentation Update Signed-off-by: Jin Hai --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 82ef6976fa..1ceea4954b 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ See [Build from Source](docs/build_from_source.md). `infinity-sdk` requires Python 3.10+. ```bash -pip3 install infinity-sdk +pip3 install infinity-sdk==v0.1.0 ``` ### Import necessary modules From 4ba389ea2d5c1613eaa3e27598a8daaf85bab80b Mon Sep 17 00:00:00 2001 From: Vissidarte-Herman Date: Mon, 6 May 2024 15:44:00 +0800 Subject: [PATCH 9/9] Added quickstart --- docs/_category_.json | 8 -- docs/getstarted/_category_.json | 8 ++ docs/{ => getstarted}/build_from_source.md | 4 +- docs/getstarted/quickstart.md | 108 +++++++++++++++++++++ docs/references/_category_.json | 2 +- 5 files changed, 119 insertions(+), 11 deletions(-) delete mode 100644 docs/_category_.json create mode 100644 docs/getstarted/_category_.json rename docs/{ => getstarted}/build_from_source.md (99%) create mode 100644 docs/getstarted/quickstart.md diff --git a/docs/_category_.json b/docs/_category_.json deleted file mode 100644 index 4aeba66477..0000000000 --- a/docs/_category_.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "label": "Build from Source", - "position": 1, - "link": { - "type": "generated-index", - "description": "building infinity from source" - } -} diff --git a/docs/getstarted/_category_.json b/docs/getstarted/_category_.json new file mode 100644 index 0000000000..437d33fa23 --- /dev/null +++ b/docs/getstarted/_category_.json @@ -0,0 +1,8 @@ +{ + "label": "Get started", + "position": 0, + "link": { + "type": "generated-index", + "description": "quickstart and more" + } +} diff --git a/docs/build_from_source.md b/docs/getstarted/build_from_source.md similarity index 99% rename from docs/build_from_source.md rename to docs/getstarted/build_from_source.md index 98ce0d7ec3..b9e3d9d320 100644 --- a/docs/build_from_source.md +++ b/docs/getstarted/build_from_source.md @@ -1,6 +1,6 @@ --- -sidebar_position: 1 -slug: / +sidebar_position: 2 +slug: /build_from_source --- # Build from Source diff --git a/docs/getstarted/quickstart.md b/docs/getstarted/quickstart.md new file mode 100644 index 0000000000..16388cd4ce --- /dev/null +++ b/docs/getstarted/quickstart.md @@ -0,0 +1,108 @@ +--- +sidebar_position: 1 +slug: / +--- + +# Quickstart + +## Prerequisites + +CPU >= 4 cores, with FMA and SSE4_2 +RAM >= 16 GB +Disk >= 50 GB +OS: Linux x86_64 or aarch64 +Glibc >=2.17 + +## Deploy Infinity database + +### Deploy Infinity using Docker on Linux x86_64 and MacOS x86_64 + +```bash +sudo mkdir -p /var/infinity && sudo chown -R $USER /var/infinity +docker pull infiniflow/infinity:nightly +docker run -d --name infinity -v /var/infinity/:/var/infinity --ulimit nofile=500000:500000 --network=host infiniflow/infinity:nightly +``` + +### Deploy Infinity using binary package on Linux x86_64 + +You can download the binary package (deb, rpm, or tgz) for your respective host operating system from https://github.com/infiniflow/infinity/releases. The prebuilt packages are compatible with Linux distributions based on glibc 2.17 or later, for example, RHEL 7, Debian 8, Ubuntu 14.04. + +Fedora/RHEL/CentOS/OpenSUSE +```bash +sudo rpm -i infinity-0.2.0-dev-x86_64.rpm +sudo systemctl start infinity +``` + +Ubuntu/Debian +```bash +sudo dpkg -i infinity-0.2.0-dev-x86_64.deb +sudo systemctl start infinity +``` +### 🛠️ Build from Source + +See [Build from Source](./build_from_source.md). + +## Install a Python client + +`infinity-sdk` requires Python 3.10+. + +```bash +pip3 install infinity-sdk +``` + +## Import necessary modules + +```python +import infinity +import infinity.index as index +from infinity.common import REMOTE_HOST +from infinity.common import ConflictType +``` + + + +## Connect to the remote server + +```python +infinity_obj = infinity.connect(REMOTE_HOST) +``` + + +## Get a database + +```python +db = infinity_obj.get_database("default_db") +``` + + +## Create a table + +```python +# Drop my_table if it already exists +db.drop_table("my_table", ConflictType.Ignore) +# Create a table named "my_table" +table = db.create_table( + "my_table", { + "num": {"type": "integer"}, + "body": {"type": "varchar"}, + "vec": {"type": "vector, 4, float"} + }) +``` + + +## Insert two records + +```python +table.insert([{"num": 1, "body": "unnecessary and harmful", "vec": [1.0, 1.2, 0.8, 0.9]}]) +table.insert([{"num": 2, "body": "Office for Harmful Blooms", "vec": [4.0, 4.2, 4.3, 4.5]}]) +``` + + +## Execute a vector search + +```python +res = table.output(["*"]).knn("vec", [3.0, 2.8, 2.7, 3.1], "float", "ip", 2).to_pl() +print(res) +``` + +> 💡 For more information about the Python API, see the [Python API Reference](../references/pysdk_api_reference.md). \ No newline at end of file diff --git a/docs/references/_category_.json b/docs/references/_category_.json index a1c174746c..d9b8336b14 100644 --- a/docs/references/_category_.json +++ b/docs/references/_category_.json @@ -1,6 +1,6 @@ { "label": "References", - "position": 2, + "position": 3, "link": { "type": "generated-index", "description": "miscellaneous references"