diff --git a/benchmarks/perf-tool/.pylintrc b/benchmarks/perf-tool/.pylintrc deleted file mode 100644 index 15bf4ccc3..000000000 --- a/benchmarks/perf-tool/.pylintrc +++ /dev/null @@ -1,443 +0,0 @@ -# This Pylint rcfile contains a best-effort configuration to uphold the -# best-practices and style described in the Google Python style guide: -# https://google.github.io/styleguide/pyguide.html -# -# Its canonical open-source location is: -# https://google.github.io/styleguide/pylintrc - -[MASTER] - -fail-under=9.0 - -# Files or directories to be skipped. They should be base names, not paths. -ignore=third_party - -# Files or directories matching the regex patterns are skipped. The regex -# matches against base names, not paths. -ignore-patterns= - -# Pickle collected data for later comparisons. -persistent=no - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -load-plugins= - -# Use multiple processes to speed up Pylint. -jobs=4 - -# Allow loading of arbitrary C extensions. Extensions are imported into the -# active Python interpreter and may run arbitrary code. -unsafe-load-any-extension=no - - -[MESSAGES CONTROL] - -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED -confidence= - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). See also the "--disable" option for examples. -#enable= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once).You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use"--disable=all --enable=classes -# --disable=W" -disable=abstract-method, - apply-builtin, - arguments-differ, - attribute-defined-outside-init, - backtick, - bad-option-value, - basestring-builtin, - buffer-builtin, - c-extension-no-member, - consider-using-enumerate, - cmp-builtin, - cmp-method, - coerce-builtin, - coerce-method, - delslice-method, - div-method, - duplicate-code, - eq-without-hash, - execfile-builtin, - file-builtin, - filter-builtin-not-iterating, - fixme, - getslice-method, - global-statement, - hex-method, - idiv-method, - implicit-str-concat-in-sequence, - import-error, - import-self, - import-star-module-level, - inconsistent-return-statements, - input-builtin, - intern-builtin, - invalid-str-codec, - locally-disabled, - long-builtin, - long-suffix, - map-builtin-not-iterating, - misplaced-comparison-constant, - missing-function-docstring, - metaclass-assignment, - next-method-called, - next-method-defined, - no-absolute-import, - no-else-break, - no-else-continue, - no-else-raise, - no-else-return, - no-init, # added - no-member, - no-name-in-module, - no-self-use, - nonzero-method, - oct-method, - old-division, - old-ne-operator, - old-octal-literal, - old-raise-syntax, - parameter-unpacking, - print-statement, - raising-string, - range-builtin-not-iterating, - raw_input-builtin, - rdiv-method, - reduce-builtin, - relative-import, - reload-builtin, - round-builtin, - setslice-method, - signature-differs, - standarderror-builtin, - suppressed-message, - sys-max-int, - too-few-public-methods, - too-many-ancestors, - too-many-arguments, - too-many-boolean-expressions, - too-many-branches, - too-many-instance-attributes, - too-many-locals, - too-many-nested-blocks, - too-many-public-methods, - too-many-return-statements, - too-many-statements, - trailing-newlines, - unichr-builtin, - unicode-builtin, - unnecessary-pass, - unpacking-in-except, - useless-else-on-loop, - useless-object-inheritance, - useless-suppression, - using-cmp-argument, - wrong-import-order, - xrange-builtin, - zip-builtin-not-iterating, - - -[REPORTS] - -# Set the output format. Available formats are text, parseable, colorized, msvs -# (visual studio) and html. You can also give a reporter class, eg -# mypackage.mymodule.MyReporterClass. -output-format=text - -# Put messages in a separate file for each module / package specified on the -# command line instead of printing them on stdout. Reports (if any) will be -# written in a file name "pylint_global.[txt|html]". This option is deprecated -# and it will be removed in Pylint 2.0. -files-output=no - -# Tells whether to display a full report or only the messages -reports=no - -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details -#msg-template= - - -[BASIC] - -# Good variable names which should always be accepted, separated by a comma -good-names=main,_ - -# Bad variable names which should always be refused, separated by a comma -bad-names= - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -name-group= - -# Include a hint for the correct naming format with invalid-name -include-naming-hint=no - -# List of decorators that produce properties, such as abc.abstractproperty. Add -# to this list to register other decorators that produce valid properties. -property-classes=abc.abstractproperty,cached_property.cached_property,cached_property.threaded_cached_property,cached_property.cached_property_with_ttl,cached_property.threaded_cached_property_with_ttl - -# Regular expression matching correct function names -function-rgx=^(?:(?PsetUp|tearDown|setUpModule|tearDownModule)|(?P_?[A-Z][a-zA-Z0-9]*)|(?P_?[a-z][a-z0-9_]*))$ - -# Regular expression matching correct variable names -variable-rgx=^[a-z][a-z0-9_]*$ - -# Regular expression matching correct constant names -const-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$ - -# Regular expression matching correct attribute names -attr-rgx=^_{0,2}[a-z][a-z0-9_]*$ - -# Regular expression matching correct argument names -argument-rgx=^[a-z][a-z0-9_]*$ - -# Regular expression matching correct class attribute names -class-attribute-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$ - -# Regular expression matching correct inline iteration names -inlinevar-rgx=^[a-z][a-z0-9_]*$ - -# Regular expression matching correct class names -class-rgx=^_?[A-Z][a-zA-Z0-9]*$ - -# Regular expression matching correct module names -module-rgx=^(_?[a-z][a-z0-9_]*|__init__)$ - -# Regular expression matching correct method names -method-rgx=(?x)^(?:(?P_[a-z0-9_]+__|runTest|setUp|tearDown|setUpTestCase|tearDownTestCase|setupSelf|tearDownClass|setUpClass|(test|assert)_*[A-Z0-9][a-zA-Z0-9_]*|next)|(?P_{0,2}[A-Z][a-zA-Z0-9_]*)|(?P_{0,2}[a-z][a-z0-9_]*))$ - -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=(__.*__|main|test.*|.*test|.*Test)$ - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=10 - - -[TYPECHECK] - -# List of decorators that produce context managers, such as -# contextlib.contextmanager. Add to this list to register other decorators that -# produce valid context managers. -contextmanager-decorators=contextlib.contextmanager,contextlib2.contextmanager - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis. It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules= - -# List of class names for which member attributes should not be checked (useful -# for classes with dynamically set attributes). This supports the use of -# qualified names. -ignored-classes=optparse.Values,thread._local,_thread._local - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= - - -[FORMAT] - -# Maximum number of characters on a single line. -max-line-length=80 - -# TODO(https://github.com/PyCQA/pylint/issues/3352): Direct pylint to exempt -# lines made too long by directives to pytype. - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=(?x)( - ^\s*(\#\ )??$| - ^\s*(from\s+\S+\s+)?import\s+.+$) - -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=yes - -# List of optional constructs for which whitespace checking is disabled. `dict- -# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. -# `trailing-comma` allows a space between comma and closing bracket: (a, ). -# `empty-line` allows space-only lines. -no-space-check= - -# Maximum number of lines in a module -max-module-lines=99999 - -# String used as indentation unit. The internal Google style guide mandates 2 -# spaces. Google's externaly-published style guide says 4, consistent with -# PEP 8. Here, we use 2 spaces, for conformity with many open-sourced Google -# projects (like TensorFlow). -indent-string=' ' - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 - -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=TODO - - -[STRING] - -# This flag controls whether inconsistent-quotes generates a warning when the -# character used as a quote delimiter is used inconsistently within a module. -check-quote-consistency=yes - - -[VARIABLES] - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# A regular expression matching the name of dummy variables (i.e. expectedly -# not used). -dummy-variables-rgx=^\*{0,2}(_$|unused_|dummy_) - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= - -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_,_cb - -# List of qualified module names which can have objects that can redefine -# builtins. -redefining-builtins-modules=six,six.moves,past.builtins,future.builtins,functools - - -[LOGGING] - -# Logging modules to check that the string format arguments are in logging -# function parameter format -logging-modules=logging,absl.logging,tensorflow.io.logging - - -[SIMILARITIES] - -# Minimum lines number of a similarity. -min-similarity-lines=4 - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=no - - -[SPELLING] - -# Spelling dictionary name. Available dictionaries: none. To make it working -# install python-enchant package. -spelling-dict= - -# List of comma separated words that should not be checked. -spelling-ignore-words= - -# A path to a file that contains private dictionary; one word per line. -spelling-private-dict-file= - -# Tells whether to store unknown words to indicated private dictionary in -# --spelling-private-dict-file option instead of raising a message. -spelling-store-unknown-words=no - - -[IMPORTS] - -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=regsub, - TERMIOS, - Bastion, - rexec, - sets - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -import-graph= - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -int-import-graph= - -# Force import order to recognize a module as part of the standard -# compatibility libraries. -known-standard-library= - -# Force import order to recognize a module as part of a third party library. -known-third-party=enchant, absl - -# Analyse import fallback blocks. This can be used to support both Python 2 and -# 3 compatible code, which means that the block might have code that exists -# only in one or another interpreter, leading to false positives when analysed. -analyse-fallback-blocks=no - - -[CLASSES] - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__, - __new__, - setUp - -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict, - _fields, - _replace, - _source, - _make - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls, - class_ - -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=mcs - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=StandardError, - Exception, - BaseException diff --git a/benchmarks/perf-tool/.style.yapf b/benchmarks/perf-tool/.style.yapf deleted file mode 100644 index 39b663a7a..000000000 --- a/benchmarks/perf-tool/.style.yapf +++ /dev/null @@ -1,10 +0,0 @@ -[style] -COLUMN_LIMIT: 80 -DEDENT_CLOSING_BRACKETS: True -INDENT_DICTIONARY_VALUE: True -SPLIT_ALL_COMMA_SEPARATED_VALUES: True -SPLIT_ARGUMENTS_WHEN_COMMA_TERMINATED: True -SPLIT_BEFORE_CLOSING_BRACKET: True -SPLIT_BEFORE_EXPRESSION_AFTER_OPENING_PAREN: True -SPLIT_BEFORE_FIRST_ARGUMENT: True -SPLIT_BEFORE_NAMED_ASSIGNS: True diff --git a/benchmarks/perf-tool/README.md b/benchmarks/perf-tool/README.md deleted file mode 100644 index eb4ac0dc1..000000000 --- a/benchmarks/perf-tool/README.md +++ /dev/null @@ -1,279 +0,0 @@ -# OpenSearch k-NN Benchmarking -- [Welcome!](#welcome) -- [Install Prerequisites](#install-prerequisites) -- [Usage](#usage) -- [Contributing](#contributing) - -## Welcome! - -This directory contains the code related to benchmarking the k-NN plugin. -Benchmarks can be run against any OpenSearch cluster with the k-NN plugin -installed. Benchmarks are highly configurable using the test configuration -file. - -## Install Prerequisites - -### Python - -Python 3.7 or above is required. - -### Pip - -Use pip to install the necessary requirements: - -``` -pip install -r requirements.txt -``` - -## Usage - -### Quick Start - -In order to run a benchmark, you must first create a test configuration yml -file. Checkout [this example](https://github.com/opensearch-project/k-NN/blob/main/benchmarks/perf-tool/sample-configs) file -for benchmarking *faiss*'s IVF method. This file contains the definition for -the benchmark that you want to run. At the top are -[test parameters](#test-parameters). These define high level settings of the -test, such as the endpoint of the OpenSearch cluster. - -Next, you define the actions that the test will perform. These actions are -referred to as steps. First, you can define "setup" steps. These are steps that -are run once at the beginning of the execution to configure the cluster how you -want it. These steps do not contribute to the final metrics. - -After that, you define the "steps". These are the steps that the test will be -collecting metrics on. Each step emits certain metrics. These are run -multiple times, depending on the test parameter "num_runs". At the end of the -execution of all of the runs, the metrics from each run are collected and -averaged. - -Lastly, you define the "cleanup" steps. The "cleanup" steps are executed after -each test run. For instance, if you are measuring index performance, you may -want to delete the index after each run. - -To run the test, execute the following command: -``` -python knn-perf-tool.py [--log LOGLEVEL] test config-path.yml output.json - ---log log level of tool, options are: info, debug, warning, error, critical -``` - -The output will be a json document containing the results. - -Additionally, you can get the difference between two test runs using the diff -command: -``` -python knn-perf-tool.py [--log LOGLEVEL] diff result1.json result2.json - ---log log level of tool, options are: info, debug, warning, error, critical -``` - -The output will be the delta between the two metrics. - -### Test Parameters - -| Parameter Name | Description | Default | -| ----------- | ----------- | ----------- | -| endpoint | Endpoint OpenSearch cluster is running on | localhost | -| test_name | Name of test | No default | -| test_id | String ID of test | No default | -| num_runs | Number of runs to execute steps | 1 | -| show_runs | Whether to output each run in addition to the total summary | false | -| setup | List of steps to run once before metric collection starts | [] | -| steps | List of steps that make up one test run. Metrics will be collected on these steps. | No default | -| cleanup | List of steps to run after each test run | [] | - -### Steps - -Included are the list of steps that are currently supported. Each step contains -a set of parameters that are passed in the test configuration file and a set -of metrics that the test produces. - -#### create_index - -Creates an OpenSearch index. - -##### Parameters -| Parameter Name | Description | Default | -| ----------- | ----------- | ----------- | -| index_name | Name of index to create | No default | -| index_spec | Path to index specification | No default | - -##### Metrics - -| Metric Name | Description | Unit | -| ----------- | ----------- | ----------- | -| took | Time to execute step end to end. | ms | - -#### disable_refresh - -Disables refresh for all indices in the cluster. - -##### Parameters - -| Parameter Name | Description | Default | -| ----------- | ----------- | ----------- | - -##### Metrics - -| Metric Name | Description | Unit | -| ----------- | ----------- | ----------- | -| took | Time to execute step end to end. | ms | - -#### refresh_index - -Refreshes an OpenSearch index. - -##### Parameters - -| Parameter Name | Description | Default | -| ----------- | ----------- | ----------- | -| index_name | Name of index to refresh | No default | - -##### Metrics - -| Metric Name | Description | Unit | -| ----------- | ----------- | ----------- | -| took | Time to execute step end to end. | ms | -| store_kb | Size of index after refresh completes | KB | - -#### force_merge - -Force merges an index to a specified number of segments. - -##### Parameters - -| Parameter Name | Description | Default | -| ----------- | ----------- | ----------- | -| index_name | Name of index to force merge | No default | -| max_num_segments | Number of segments to force merge to | No default | - -##### Metrics - -| Metric Name | Description | Unit | -| ----------- | ----------- | ----------- | -| took | Time to execute step end to end. | ms | - -#### train_model - -Trains a model. - -##### Parameters - -| Parameter Name | Description | Default | -| ----------- | ----------- | ----------- | -| model_id | Model id to set | Test | -| train_index | Index to pull training data from | No default | -| train_field | Field to pull training data from | No default | -| dimension | Dimension of model | No default | -| description | Description of model | No default | -| max_training_vector_count | Number of training vectors to used | No default | -| method_spec | Path to method specification | No default | - -##### Metrics - -| Metric Name | Description | Unit | -| ----------- | ----------- | ----------- | -| took | Time to execute step end to end | ms | - -#### delete_model - -Deletes a model from the cluster. - -##### Parameters - -| Parameter Name | Description | Default | -| ----------- | ----------- | ----------- | -| model_id | Model id to delete | Test | - -##### Metrics - -| Metric Name | Description | Unit | -| ----------- | ----------- | ----------- | -| took | Time to execute step end to end | ms | - -#### delete_index - -Deletes an index from the cluster. - -##### Parameters - -| Parameter Name | Description | Default | -| ----------- | ----------- | ----------- | -| index_name | Name of index to delete | No default | - -##### Metrics - -| Metric Name | Description | Unit | -| ----------- | ----------- | ----------- | -| took | Time to execute step end to end | ms | - -#### ingest - -Ingests a dataset of vectors into the cluster. - -##### Parameters - -| Parameter Name | Description | Default | -| ----------- | ----------- | ----------- | -| index_name | Name of index to ingest into | No default | -| field_name | Name of field to ingest into | No default | -| bulk_size | Documents per bulk request | 300 | -| dataset_format | Format the data-set is in. Currently hdf5 and bigann is supported. The hdf5 file must be organized in the same way that the ann-benchmarks organizes theirs. | 'hdf5' | -| dataset_path | Path to data-set | No default | -| doc_count | Number of documents to create from data-set | Size of the data-set | - -##### Metrics - -| Metric Name | Description | Unit | -| ----------- | ----------- | ----------- | -| took | Total time to ingest the dataset into the index.| ms | - -#### query - -Runs a set of queries against an index. - -##### Parameters - -| Parameter Name | Description | Default | -| ----------- | ----------- | ----------- | -| k | Number of neighbors to return on search | 100 | -| r | r value in Recall@R | 1 | -| index_name | Name of index to search | No default | -| field_name | Name field to search | No default | -| calculate_recall | Whether to calculate recall values | False | -| dataset_format | Format the dataset is in. Currently hdf5 and bigann is supported. The hdf5 file must be organized in the same way that the ann-benchmarks organizes theirs. | 'hdf5' | -| dataset_path | Path to dataset | No default | -| neighbors_format | Format the neighbors dataset is in. Currently hdf5 and bigann is supported. The hdf5 file must be organized in the same way that the ann-benchmarks organizes theirs. | 'hdf5' | -| neighbors_path | Path to neighbors dataset | No default | -| query_count | Number of queries to create from data-set | Size of the data-set | - -##### Metrics - -| Metric Name | Description | Unit | -| ----------- | ----------- | ----------- | -| took | Took times returned per query aggregated as total, p50, p90 and p99 (when applicable) | ms | -| memory_kb | Native memory k-NN is using at the end of the query workload | KB | -| recall@R | ratio of top R results from the ground truth neighbors that are in the K results returned by the plugin | float 0.0-1.0 | -| recall@K | ratio of results returned that were ground truth nearest neighbors | float 0.0-1.0 | - -## Contributing - -### Linting - -Use pylint to lint the code: -``` -pylint knn-perf-tool.py okpt/**/*.py okpt/**/**/*.py -``` - -### Formatting - -We use yapf and the google style to format our code. After installing yapf, you can format your code by running: - -``` -yapf --style google knn-perf-tool.py okpt/**/*.py okpt/**/**/*.py -``` - -### Updating requirements - -Add new requirements to "requirements.in" and run `pip-compile` diff --git a/benchmarks/perf-tool/dataset/data.hdf5 b/benchmarks/perf-tool/dataset/data.hdf5 deleted file mode 100644 index c9268606d..000000000 Binary files a/benchmarks/perf-tool/dataset/data.hdf5 and /dev/null differ diff --git a/benchmarks/perf-tool/knn-perf-tool.py b/benchmarks/perf-tool/knn-perf-tool.py deleted file mode 100644 index 48eedc427..000000000 --- a/benchmarks/perf-tool/knn-perf-tool.py +++ /dev/null @@ -1,10 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. -"""Script for user to run the testing tool.""" - -import okpt.main - -okpt.main.main() diff --git a/benchmarks/perf-tool/okpt/__init__.py b/benchmarks/perf-tool/okpt/__init__.py deleted file mode 100644 index c3bffc54c..000000000 --- a/benchmarks/perf-tool/okpt/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. - diff --git a/benchmarks/perf-tool/okpt/diff/diff.py b/benchmarks/perf-tool/okpt/diff/diff.py deleted file mode 100644 index 23f424ab9..000000000 --- a/benchmarks/perf-tool/okpt/diff/diff.py +++ /dev/null @@ -1,142 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. - -"""Provides the Diff class.""" - -from enum import Enum -from typing import Any, Dict, Tuple - - -class InvalidTestResultsError(Exception): - """Exception raised when the test results are invalid. - - The results can be invalid if they have different fields, non-numeric - values, or if they don't follow the standard result format. - """ - def __init__(self, msg: str): - self.message = msg - super().__init__(self.message) - - -def _is_numeric(a) -> bool: - return isinstance(a, (int, float)) - - -class TestResultFields(str, Enum): - METADATA = 'metadata' - RESULTS = 'results' - TEST_PARAMETERS = 'test_parameters' - - -class TestResultNames(str, Enum): - BASE = 'base_result' - CHANGED = 'changed_result' - - -class Diff: - """Diff class for validating and diffing two test result files. - - Methods: - diff: Returns the diff between two test results. (changed - base) - """ - def __init__( - self, - base_result: Dict[str, - Any], - changed_result: Dict[str, - Any], - metadata: bool - ): - """Initializes test results and validate them.""" - self.base_result = base_result - self.changed_result = changed_result - self.metadata = metadata - - # make sure results have proper test result fields - is_valid, key, result = self._validate_keys() - if not is_valid: - raise InvalidTestResultsError( - f'{result} has a missing or invalid key `{key}`.' - ) - - self.base_results = self.base_result[TestResultFields.RESULTS] - self.changed_results = self.changed_result[TestResultFields.RESULTS] - - # make sure results have the same fields - is_valid, key, result = self._validate_structure() - if not is_valid: - raise InvalidTestResultsError( - f'key `{key}` is not present in {result}.' - ) - - # make sure results have numeric values - is_valid, key, result = self._validate_types() - if not is_valid: - raise InvalidTestResultsError( - f'key `{key}` in {result} points to a non-numeric value.' - ) - - def _validate_keys(self) -> Tuple[bool, str, str]: - """Ensure both test results have `metadata` and `results` keys.""" - check_keydict = lambda key, res: key in res and isinstance( - res[key], dict) - - # check if results have a `metadata` field and if `metadata` is a dict - if self.metadata: - if not check_keydict(TestResultFields.METADATA, self.base_result): - return (False, TestResultFields.METADATA, TestResultNames.BASE) - if not check_keydict(TestResultFields.METADATA, - self.changed_result): - return ( - False, - TestResultFields.METADATA, - TestResultNames.CHANGED - ) - # check if results have a `results` field and `results` is a dict - if not check_keydict(TestResultFields.RESULTS, self.base_result): - return (False, TestResultFields.RESULTS, TestResultNames.BASE) - if not check_keydict(TestResultFields.RESULTS, self.changed_result): - return (False, TestResultFields.RESULTS, TestResultNames.CHANGED) - return (True, '', '') - - def _validate_structure(self) -> Tuple[bool, str, str]: - """Ensure both test results have the same keys.""" - for k in self.base_results: - if not k in self.changed_results: - return (False, k, TestResultNames.CHANGED) - for k in self.changed_results: - if not k in self.base_results: - return (False, k, TestResultNames.BASE) - return (True, '', '') - - def _validate_types(self) -> Tuple[bool, str, str]: - """Ensure both test results have numeric values.""" - for k, v in self.base_results.items(): - if not _is_numeric(v): - return (False, k, TestResultNames.BASE) - for k, v in self.changed_results.items(): - if not _is_numeric(v): - return (False, k, TestResultNames.BASE) - return (True, '', '') - - def diff(self) -> Dict[str, Any]: - """Return the diff between the two test results. (changed - base)""" - results_diff = { - key: self.changed_results[key] - self.base_results[key] - for key in self.base_results - } - - # add metadata if specified - if self.metadata: - return { - f'{TestResultNames.BASE}_{TestResultFields.METADATA}': - self.base_result[TestResultFields.METADATA], - f'{TestResultNames.CHANGED}_{TestResultFields.METADATA}': - self.changed_result[TestResultFields.METADATA], - 'diff': - results_diff - } - return results_diff diff --git a/benchmarks/perf-tool/okpt/io/args.py b/benchmarks/perf-tool/okpt/io/args.py deleted file mode 100644 index f8c5d8809..000000000 --- a/benchmarks/perf-tool/okpt/io/args.py +++ /dev/null @@ -1,178 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. - -"""Parses and defines command line arguments for the program. - -Defines the subcommands `test` and `diff` and the corresponding -files that are required by each command. - -Functions: - define_args(): Define the command line arguments. - get_args(): Returns a dictionary of the command line args. -""" - -import argparse -import sys -from dataclasses import dataclass -from io import TextIOWrapper -from typing import Union - -_read_type = argparse.FileType('r') -_write_type = argparse.FileType('w') - - -def _add_config(parser, name, **kwargs): - """"Add configuration file path argument.""" - opts = { - 'type': _read_type, - 'help': 'Path of configuration file.', - 'metavar': 'config_path', - **kwargs, - } - parser.add_argument(name, **opts) - - -def _add_result(parser, name, **kwargs): - """"Add results files paths argument.""" - opts = { - 'type': _read_type, - 'help': 'Path of one result file.', - 'metavar': 'result_path', - **kwargs, - } - parser.add_argument(name, **opts) - - -def _add_results(parser, name, **kwargs): - """"Add results files paths argument.""" - opts = { - 'nargs': '+', - 'type': _read_type, - 'help': 'Paths of result files.', - 'metavar': 'result_paths', - **kwargs, - } - parser.add_argument(name, **opts) - - -def _add_output(parser, name, **kwargs): - """"Add output file path argument.""" - opts = { - 'type': _write_type, - 'help': 'Path of output file.', - 'metavar': 'output_path', - **kwargs, - } - parser.add_argument(name, **opts) - - -def _add_metadata(parser, name, **kwargs): - opts = { - 'action': 'store_true', - **kwargs, - } - parser.add_argument(name, **opts) - - -def _add_test_cmd(subparsers): - test_parser = subparsers.add_parser('test') - _add_config(test_parser, 'config') - _add_output(test_parser, 'output') - - -def _add_diff_cmd(subparsers): - diff_parser = subparsers.add_parser('diff') - _add_metadata(diff_parser, '--metadata') - _add_result( - diff_parser, - 'base_result', - help='Base test result.', - metavar='base_result' - ) - _add_result( - diff_parser, - 'changed_result', - help='Changed test result.', - metavar='changed_result' - ) - _add_output(diff_parser, '--output', default=sys.stdout) - - -@dataclass -class TestArgs: - log: str - command: str - config: TextIOWrapper - output: TextIOWrapper - - -@dataclass -class DiffArgs: - log: str - command: str - metadata: bool - base_result: TextIOWrapper - changed_result: TextIOWrapper - output: TextIOWrapper - - -def get_args() -> Union[TestArgs, DiffArgs]: - """Define, parse and return command line args. - - Returns: - A dict containing the command line args. - """ - parser = argparse.ArgumentParser( - description= - 'Run performance tests against the OpenSearch plugin and various ANN ' - 'libaries.' - ) - - def define_args(): - """Define tool commands.""" - - # add log level arg - parser.add_argument( - '--log', - default='info', - type=str, - choices=['debug', - 'info', - 'warning', - 'error', - 'critical'], - help='Log level of the tool.' - ) - - subparsers = parser.add_subparsers( - title='commands', - dest='command', - help='sub-command help' - ) - subparsers.required = True - - # add subcommands - _add_test_cmd(subparsers) - _add_diff_cmd(subparsers) - - define_args() - args = parser.parse_args() - if args.command == 'test': - return TestArgs( - log=args.log, - command=args.command, - config=args.config, - output=args.output - ) - else: - return DiffArgs( - log=args.log, - command=args.command, - metadata=args.metadata, - base_result=args.base_result, - changed_result=args.changed_result, - output=args.output - ) diff --git a/benchmarks/perf-tool/okpt/io/config/parsers/base.py b/benchmarks/perf-tool/okpt/io/config/parsers/base.py deleted file mode 100644 index 795aab1b2..000000000 --- a/benchmarks/perf-tool/okpt/io/config/parsers/base.py +++ /dev/null @@ -1,67 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. - -"""Base Parser class. - -Classes: - BaseParser: Base class for config parsers. - -Exceptions: - ConfigurationError: An error in the configuration syntax. -""" - -import os -from io import TextIOWrapper - -import cerberus - -from okpt.io.utils import reader - - -class ConfigurationError(Exception): - """Exception raised for errors in the tool configuration. - - Attributes: - message -- explanation of the error - """ - - def __init__(self, message: str): - self.message = f'{message}' - super().__init__(self.message) - - -def _get_validator_from_schema_name(schema_name: str): - """Get the corresponding Cerberus validator from a schema name.""" - curr_file_dir = os.path.dirname(os.path.abspath(__file__)) - schemas_dir = os.path.join(os.path.dirname(curr_file_dir), 'schemas') - schema_file_path = os.path.join(schemas_dir, f'{schema_name}.yml') - schema_obj = reader.parse_yaml_from_path(schema_file_path) - return cerberus.Validator(schema_obj) - - -class BaseParser: - """Base class for config parsers. - - Attributes: - validator: Cerberus validator for a particular schema - errors: Cerberus validation errors (if any are found during validation) - - Methods: - parse: Parse config. - """ - - def __init__(self, schema_name: str): - self.validator = _get_validator_from_schema_name(schema_name) - self.errors = '' - - def parse(self, file_obj: TextIOWrapper): - """Convert file object to dict, while validating against config schema.""" - config_obj = reader.parse_yaml(file_obj) - is_config_valid = self.validator.validate(config_obj) - if not is_config_valid: - raise ConfigurationError(self.validator.errors) - - return self.validator.document diff --git a/benchmarks/perf-tool/okpt/io/config/parsers/test.py b/benchmarks/perf-tool/okpt/io/config/parsers/test.py deleted file mode 100644 index 34b1752c7..000000000 --- a/benchmarks/perf-tool/okpt/io/config/parsers/test.py +++ /dev/null @@ -1,74 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. - -"""Provides ToolParser. - -Classes: - ToolParser: Tool config parser. -""" -from dataclasses import dataclass -from io import TextIOWrapper -from typing import List - -from okpt.io.config.parsers import base -from okpt.test.steps.base import Step, StepConfig -from okpt.test.steps.factory import create_step - - -@dataclass -class TestConfig: - test_name: str - test_id: str - endpoint: str - num_runs: int - show_runs: bool - setup: List[Step] - steps: List[Step] - cleanup: List[Step] - - -class TestParser(base.BaseParser): - """Parser for Test config. - - Methods: - parse: Parse and validate the Test config. - """ - - def __init__(self): - super().__init__('test') - - def parse(self, file_obj: TextIOWrapper) -> TestConfig: - """See base class.""" - config_obj = super().parse(file_obj) - - implicit_step_config = dict() - if 'endpoint' in config_obj: - implicit_step_config['endpoint'] = config_obj['endpoint'] - - # Each step should have its own parse - take the config object and check if its valid - setup = [] - if 'setup' in config_obj: - setup = [create_step(StepConfig(step["name"], step, implicit_step_config)) for step in config_obj['setup']] - - steps = [create_step(StepConfig(step["name"], step, implicit_step_config)) for step in config_obj['steps']] - - cleanup = [] - if 'cleanup' in config_obj: - cleanup = [create_step(StepConfig(step["name"], step, implicit_step_config)) for step - in config_obj['cleanup']] - - test_config = TestConfig( - endpoint=config_obj['endpoint'], - test_name=config_obj['test_name'], - test_id=config_obj['test_id'], - num_runs=config_obj['num_runs'], - show_runs=config_obj['show_runs'], - setup=setup, - steps=steps, - cleanup=cleanup - ) - - return test_config diff --git a/benchmarks/perf-tool/okpt/io/config/parsers/util.py b/benchmarks/perf-tool/okpt/io/config/parsers/util.py deleted file mode 100644 index cecb9f2d0..000000000 --- a/benchmarks/perf-tool/okpt/io/config/parsers/util.py +++ /dev/null @@ -1,116 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. - -"""Utility functions for parsing""" - - -from okpt.io.config.parsers.base import ConfigurationError -from okpt.io.dataset import HDF5DataSet, BigANNNeighborDataSet, \ - BigANNVectorDataSet, DataSet, Context - - -def parse_dataset(dataset_format: str, dataset_path: str, - context: Context) -> DataSet: - if dataset_format == 'hdf5': - return HDF5DataSet(dataset_path, context) - - if dataset_format == 'bigann' and context == Context.NEIGHBORS: - return BigANNNeighborDataSet(dataset_path) - - if dataset_format == 'bigann': - return BigANNVectorDataSet(dataset_path) - - raise Exception("Unsupported data-set format") - - -def parse_string_param(key: str, first_map, second_map, default) -> str: - value = first_map.get(key) - if value is not None: - if type(value) is str: - return value - raise ConfigurationError("Invalid type for {}".format(key)) - - value = second_map.get(key) - if value is not None: - if type(value) is str: - return value - raise ConfigurationError("Invalid type for {}".format(key)) - - if default is None: - raise ConfigurationError("{} must be set".format(key)) - return default - - -def parse_int_param(key: str, first_map, second_map, default) -> int: - value = first_map.get(key) - if value is not None: - if type(value) is int: - return value - raise ConfigurationError("Invalid type for {}".format(key)) - - value = second_map.get(key) - if value is not None: - if type(value) is int: - return value - raise ConfigurationError("Invalid type for {}".format(key)) - - if default is None: - raise ConfigurationError("{} must be set".format(key)) - return default - - -def parse_bool_param(key: str, first_map, second_map, default) -> bool: - value = first_map.get(key) - if value is not None: - if type(value) is bool: - return value - raise ConfigurationError("Invalid type for {}".format(key)) - - value = second_map.get(key) - if value is not None: - if type(value) is bool: - return value - raise ConfigurationError("Invalid type for {}".format(key)) - - if default is None: - raise ConfigurationError("{} must be set".format(key)) - return default - - -def parse_dict_param(key: str, first_map, second_map, default) -> dict: - value = first_map.get(key) - if value is not None: - if type(value) is dict: - return value - raise ConfigurationError("Invalid type for {}".format(key)) - - value = second_map.get(key) - if value is not None: - if type(value) is dict: - return value - raise ConfigurationError("Invalid type for {}".format(key)) - - if default is None: - raise ConfigurationError("{} must be set".format(key)) - return default - - -def parse_list_param(key: str, first_map, second_map, default) -> list: - value = first_map.get(key) - if value is not None: - if type(value) is list: - return value - raise ConfigurationError("Invalid type for {}".format(key)) - - value = second_map.get(key) - if value is not None: - if type(value) is list: - return value - raise ConfigurationError("Invalid type for {}".format(key)) - - if default is None: - raise ConfigurationError("{} must be set".format(key)) - return default diff --git a/benchmarks/perf-tool/okpt/io/config/schemas/test.yml b/benchmarks/perf-tool/okpt/io/config/schemas/test.yml deleted file mode 100644 index 1939a8a31..000000000 --- a/benchmarks/perf-tool/okpt/io/config/schemas/test.yml +++ /dev/null @@ -1,29 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. - -# defined using the cerberus validation API -# https://docs.python-cerberus.org/en/stable/index.html -endpoint: - type: string - default: "localhost" -test_name: - type: string -test_id: - type: string -num_runs: - type: integer - default: 1 - min: 1 - max: 10000 -show_runs: - type: boolean - default: false -setup: - type: list -steps: - type: list -cleanup: - type: list diff --git a/benchmarks/perf-tool/okpt/io/dataset.py b/benchmarks/perf-tool/okpt/io/dataset.py deleted file mode 100644 index 4f8bc22a2..000000000 --- a/benchmarks/perf-tool/okpt/io/dataset.py +++ /dev/null @@ -1,218 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. - -"""Defines DataSet interface and implements particular formats - -A DataSet is the basic functionality that it can be read in chunks, or -read completely and reset to the start. - -Currently, we support HDF5 formats from ann-benchmarks and big-ann-benchmarks -datasets. - -Classes: - HDF5DataSet: Format used in ann-benchmarks - BigANNNeighborDataSet: Neighbor format for big-ann-benchmarks - BigANNVectorDataSet: Vector format for big-ann-benchmarks -""" -import os -from abc import ABC, ABCMeta, abstractmethod -from enum import Enum -from typing import cast -import h5py -import numpy as np - -import struct - - -class Context(Enum): - """DataSet context enum. Can be used to add additional context for how a - data-set should be interpreted. - """ - INDEX = 1 - QUERY = 2 - NEIGHBORS = 3 - - -class DataSet(ABC): - """DataSet interface. Used for reading data-sets from files. - - Methods: - read: Read a chunk of data from the data-set - size: Gets the number of items in the data-set - reset: Resets internal state of data-set to beginning - """ - __metaclass__ = ABCMeta - - @abstractmethod - def read(self, chunk_size: int): - pass - - @abstractmethod - def size(self): - pass - - @abstractmethod - def reset(self): - pass - - -class HDF5DataSet(DataSet): - """ Data-set format corresponding to `ANN Benchmarks - `_ - """ - - def __init__(self, dataset_path: str, context: Context): - file = h5py.File(dataset_path) - self.data = cast(h5py.Dataset, file[self._parse_context(context)]) - self.current = 0 - - def read(self, chunk_size: int): - if self.current >= self.size(): - return None - - end_i = self.current + chunk_size - if end_i > self.size(): - end_i = self.size() - - v = cast(np.ndarray, self.data[self.current:end_i]) - self.current = end_i - return v - - def size(self): - return self.data.len() - - def reset(self): - self.current = 0 - - @staticmethod - def _parse_context(context: Context) -> str: - if context == Context.NEIGHBORS: - return "neighbors" - - if context == Context.INDEX: - return "train" - - if context == Context.QUERY: - return "test" - - raise Exception("Unsupported context") - - -class BigANNNeighborDataSet(DataSet): - """ Data-set format for neighbor data-sets for `Big ANN Benchmarks - `_""" - - def __init__(self, dataset_path: str): - self.file = open(dataset_path, 'rb') - self.file.seek(0, os.SEEK_END) - num_bytes = self.file.tell() - self.file.seek(0) - - if num_bytes < 8: - raise Exception("File is invalid") - - self.num_queries = int.from_bytes(self.file.read(4), "little") - self.k = int.from_bytes(self.file.read(4), "little") - - # According to the website, the number of bytes that will follow will - # be: num_queries X K x sizeof(uint32_t) bytes + num_queries X K x - # sizeof(float) - if (num_bytes - 8) != 2 * (self.num_queries * self.k * 4): - raise Exception("File is invalid") - - self.current = 0 - - def read(self, chunk_size: int): - if self.current >= self.size(): - return None - - end_i = self.current + chunk_size - if end_i > self.size(): - end_i = self.size() - - v = [[int.from_bytes(self.file.read(4), "little") for _ in - range(self.k)] for _ in range(end_i - self.current)] - - self.current = end_i - return v - - def size(self): - return self.num_queries - - def reset(self): - self.file.seek(8) - self.current = 0 - - -class BigANNVectorDataSet(DataSet): - """ Data-set format for vector data-sets for `Big ANN Benchmarks - `_ - """ - - def __init__(self, dataset_path: str): - self.file = open(dataset_path, 'rb') - self.file.seek(0, os.SEEK_END) - num_bytes = self.file.tell() - self.file.seek(0) - - if num_bytes < 8: - raise Exception("File is invalid") - - self.num_points = int.from_bytes(self.file.read(4), "little") - self.dimension = int.from_bytes(self.file.read(4), "little") - bytes_per_num = self._get_data_size(dataset_path) - - if (num_bytes - 8) != self.num_points * self.dimension * bytes_per_num: - raise Exception("File is invalid") - - self.reader = self._value_reader(dataset_path) - self.current = 0 - - def read(self, chunk_size: int): - if self.current >= self.size(): - return None - - end_i = self.current + chunk_size - if end_i > self.size(): - end_i = self.size() - - v = np.asarray([self._read_vector() for _ in - range(end_i - self.current)]) - self.current = end_i - return v - - def _read_vector(self): - return np.asarray([self.reader(self.file) for _ in - range(self.dimension)]) - - def size(self): - return self.num_points - - def reset(self): - self.file.seek(8) # Seek to 8 bytes to skip re-reading metadata - self.current = 0 - - @staticmethod - def _get_data_size(file_name): - ext = file_name.split('.')[-1] - if ext == "u8bin": - return 1 - - if ext == "fbin": - return 4 - - raise Exception("Unknown extension") - - @staticmethod - def _value_reader(file_name): - ext = file_name.split('.')[-1] - if ext == "u8bin": - return lambda file: float(int.from_bytes(file.read(1), "little")) - - if ext == "fbin": - return lambda file: struct.unpack(' TextIOWrapper: - """Given a file path, get a readable file object. - - Args: - file path - - Returns: - Writeable file object - """ - return open(path, 'r', encoding='UTF-8') - - -def parse_yaml(file: TextIOWrapper) -> Dict[str, Any]: - """Parses YAML file from file object. - - Args: - file: file object to parse - - Returns: - A dict representing the YAML file. - """ - return yaml.load(file, Loader=yaml.SafeLoader) - - -def parse_yaml_from_path(path: str) -> Dict[str, Any]: - """Parses YAML file from file path. - - Args: - path: file path to parse - - Returns: - A dict representing the YAML file. - """ - file = reader.get_file_obj(path) - return parse_yaml(file) - - -def parse_json(file: TextIOWrapper) -> Dict[str, Any]: - """Parses JSON file from file object. - - Args: - file: file object to parse - - Returns: - A dict representing the JSON file. - """ - return json.load(file) - - -def parse_json_from_path(path: str) -> Dict[str, Any]: - """Parses JSON file from file path. - - Args: - path: file path to parse - - Returns: - A dict representing the JSON file. - """ - file = reader.get_file_obj(path) - return json.load(file) diff --git a/benchmarks/perf-tool/okpt/io/utils/writer.py b/benchmarks/perf-tool/okpt/io/utils/writer.py deleted file mode 100644 index 1f14bfd94..000000000 --- a/benchmarks/perf-tool/okpt/io/utils/writer.py +++ /dev/null @@ -1,40 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. -"""Provides functions for writing to file. - -Functions: - get_file_obj(): Get a writeable file object. - write_json(): Writes a python dictionary to a JSON file -""" - -import json -from io import TextIOWrapper -from typing import Any, Dict, TextIO, Union - - -def get_file_obj(path: str) -> TextIOWrapper: - """Get a writeable file object from a file path. - - Args: - file path - - Returns: - Writeable file object - """ - return open(path, 'w', encoding='UTF-8') - - -def write_json(data: Dict[str, Any], - file: Union[TextIOWrapper, TextIO], - pretty=False): - """Writes a dictionary to a JSON file. - - Args: - data: A dict to write to JSON. - file: Path of output file. - """ - indent = 2 if pretty else 0 - json.dump(data, file, indent=indent) diff --git a/benchmarks/perf-tool/okpt/main.py b/benchmarks/perf-tool/okpt/main.py deleted file mode 100644 index 3e6e022d4..000000000 --- a/benchmarks/perf-tool/okpt/main.py +++ /dev/null @@ -1,55 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. - -""" Runner script that serves as the main controller of the testing tool.""" - -import logging -import sys -from typing import cast - -from okpt.diff import diff -from okpt.io import args -from okpt.io.config.parsers import test -from okpt.io.utils import reader, writer -from okpt.test import runner - - -def main(): - """Main function of entry module.""" - cli_args = args.get_args() - output = cli_args.output - if cli_args.log: - log_level = getattr(logging, cli_args.log.upper()) - logging.basicConfig(level=log_level) - - if cli_args.command == 'test': - cli_args = cast(args.TestArgs, cli_args) - - # parse config - parser = test.TestParser() - test_config = parser.parse(cli_args.config) - logging.info('Configs are valid.') - - # run tests - test_runner = runner.TestRunner(test_config=test_config) - test_result = test_runner.execute() - - # write test results - logging.debug( - f'Test Result:\n {writer.write_json(test_result, sys.stdout, pretty=True)}' - ) - writer.write_json(test_result, output, pretty=True) - elif cli_args.command == 'diff': - cli_args = cast(args.DiffArgs, cli_args) - - # parse test results - base_result = reader.parse_json(cli_args.base_result) - changed_result = reader.parse_json(cli_args.changed_result) - - # get diff - diff_result = diff.Diff(base_result, changed_result, - cli_args.metadata).diff() - writer.write_json(data=diff_result, file=output, pretty=True) diff --git a/benchmarks/perf-tool/okpt/test/__init__.py b/benchmarks/perf-tool/okpt/test/__init__.py deleted file mode 100644 index ff4fd04d1..000000000 --- a/benchmarks/perf-tool/okpt/test/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. diff --git a/benchmarks/perf-tool/okpt/test/profile.py b/benchmarks/perf-tool/okpt/test/profile.py deleted file mode 100644 index d96860f9a..000000000 --- a/benchmarks/perf-tool/okpt/test/profile.py +++ /dev/null @@ -1,86 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. - -"""Provides decorators to profile functions. - -The decorators work by adding a `measureable` (time, memory, etc) field to a -dictionary returned by the wrapped function. So the wrapped functions must -return a dictionary in order to be profiled. -""" -import functools -import time -from typing import Callable - - -class TimerStoppedWithoutStartingError(Exception): - """Error raised when Timer is stopped without having been started.""" - - def __init__(self): - super().__init__() - self.message = 'Timer must call start() before calling end().' - - -class _Timer(): - """Timer class for timing. - - Methods: - start: Starts the timer. - end: Stops the timer and returns the time elapsed since start. - - Raises: - TimerStoppedWithoutStartingError: Timer must start before ending. - """ - - def __init__(self): - self.start_time = None - - def start(self): - """Starts the timer.""" - self.start_time = time.perf_counter() - - def end(self) -> float: - """Stops the timer. - - Returns: - The time elapsed in milliseconds. - """ - # ensure timer has started before ending - if self.start_time is None: - raise TimerStoppedWithoutStartingError() - - elapsed = (time.perf_counter() - self.start_time) * 1000 - self.start_time = None - return elapsed - - -def took(f: Callable): - """Profiles a functions execution time. - - Args: - f: Function to profile. - - Returns: - A function that wraps the passed in function and adds a time took field - to the return value. - """ - - @functools.wraps(f) - def wrapper(*args, **kwargs): - """Wrapper function.""" - timer = _Timer() - timer.start() - result = f(*args, **kwargs) - time_took = timer.end() - - # if result already has a `took` field, don't modify the result - if isinstance(result, dict) and 'took' in result: - return result - # `result` may not be a dictionary, so it may not be unpackable - elif isinstance(result, dict): - return {**result, 'took': time_took} - return {'took': time_took} - - return wrapper diff --git a/benchmarks/perf-tool/okpt/test/runner.py b/benchmarks/perf-tool/okpt/test/runner.py deleted file mode 100644 index 150154691..000000000 --- a/benchmarks/perf-tool/okpt/test/runner.py +++ /dev/null @@ -1,107 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. - -"""Provides a test runner class.""" -import logging -import platform -import sys -from datetime import datetime -from typing import Any, Dict, List - -import psutil - -from okpt.io.config.parsers import test -from okpt.test.test import Test, get_avg - - -def _aggregate_runs(runs: List[Dict[str, Any]]): - """Aggregates and averages a list of test results. - - Args: - results: A list of test results. - num_runs: Number of times the tests were ran. - - Returns: - A dictionary containing the averages of the test results. - """ - aggregate: Dict[str, Any] = {} - for run in runs: - for key, value in run.items(): - if key in aggregate: - aggregate[key].append(value) - else: - aggregate[key] = [value] - - aggregate = {key: get_avg(value) for key, value in aggregate.items()} - return aggregate - - -class TestRunner: - """Test runner class for running tests and aggregating the results. - - Methods: - execute: Run the tests and aggregate the results. - """ - - def __init__(self, test_config: test.TestConfig): - """"Initializes test state.""" - self.test_config = test_config - self.test = Test(test_config) - - def _get_metadata(self): - """"Retrieves the test metadata.""" - svmem = psutil.virtual_memory() - return { - 'test_name': - self.test_config.test_name, - 'test_id': - self.test_config.test_id, - 'date': - datetime.now().strftime('%m/%d/%Y %H:%M:%S'), - 'python_version': - sys.version, - 'os_version': - platform.platform(), - 'processor': - platform.processor() + ', ' + - str(psutil.cpu_count(logical=True)) + ' cores', - 'memory': - str(svmem.used) + ' (used) / ' + str(svmem.available) + - ' (available) / ' + str(svmem.total) + ' (total)', - } - - def execute(self) -> Dict[str, Any]: - """Runs the tests and aggregates the results. - - Returns: - A dictionary containing the aggregate of test results. - """ - logging.info('Setting up tests.') - self.test.setup() - logging.info('Beginning to run tests.') - runs = [] - for i in range(self.test_config.num_runs): - logging.info( - f'Running test {i + 1} of {self.test_config.num_runs}' - ) - runs.append(self.test.execute()) - - logging.info('Finished running tests.') - aggregate = _aggregate_runs(runs) - - # add metadata to test results - test_result = { - 'metadata': - self._get_metadata(), - 'results': - aggregate - } - - # include info about all test runs if specified in config - if self.test_config.show_runs: - test_result['runs'] = runs - - return test_result diff --git a/benchmarks/perf-tool/okpt/test/steps/base.py b/benchmarks/perf-tool/okpt/test/steps/base.py deleted file mode 100644 index 829980421..000000000 --- a/benchmarks/perf-tool/okpt/test/steps/base.py +++ /dev/null @@ -1,60 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. -"""Provides base Step interface.""" - -from dataclasses import dataclass -from typing import Any, Dict, List - -from okpt.test import profile - - -@dataclass -class StepConfig: - step_name: str - config: Dict[str, object] - implicit_config: Dict[str, object] - - -class Step: - """Test step interface. - - Attributes: - label: Name of the step. - - Methods: - execute: Run the step and return a step response with the label and - corresponding measures. - """ - - label = 'base_step' - - def __init__(self, step_config: StepConfig): - self.step_config = step_config - - def _action(self): - """Step logic/behavior to be executed and profiled.""" - pass - - def _get_measures(self) -> List[str]: - """Gets the measures for a particular test""" - pass - - def execute(self) -> List[Dict[str, Any]]: - """Execute step logic while profiling various measures. - - Returns: - Dict containing step label and various step measures. - """ - action = self._action - - # profile the action with measure decorators - add if necessary - action = getattr(profile, 'took')(action) - - result = action() - if isinstance(result, dict): - return [{'label': self.label, **result}] - - raise ValueError('Invalid return by a step') diff --git a/benchmarks/perf-tool/okpt/test/steps/factory.py b/benchmarks/perf-tool/okpt/test/steps/factory.py deleted file mode 100644 index cb0789dec..000000000 --- a/benchmarks/perf-tool/okpt/test/steps/factory.py +++ /dev/null @@ -1,35 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. -"""Factory for creating steps.""" - -from okpt.io.config.parsers.base import ConfigurationError -from okpt.test.steps.base import Step, StepConfig - -from okpt.test.steps.steps import CreateIndexStep, DisableRefreshStep, RefreshIndexStep, DeleteIndexStep, \ - TrainModelStep, DeleteModelStep, ForceMergeStep, IngestStep, QueryStep - - -def create_step(step_config: StepConfig) -> Step: - if step_config.step_name == CreateIndexStep.label: - return CreateIndexStep(step_config) - elif step_config.step_name == DisableRefreshStep.label: - return DisableRefreshStep(step_config) - elif step_config.step_name == RefreshIndexStep.label: - return RefreshIndexStep(step_config) - elif step_config.step_name == TrainModelStep.label: - return TrainModelStep(step_config) - elif step_config.step_name == DeleteModelStep.label: - return DeleteModelStep(step_config) - elif step_config.step_name == DeleteIndexStep.label: - return DeleteIndexStep(step_config) - elif step_config.step_name == IngestStep.label: - return IngestStep(step_config) - elif step_config.step_name == QueryStep.label: - return QueryStep(step_config) - elif step_config.step_name == ForceMergeStep.label: - return ForceMergeStep(step_config) - - raise ConfigurationError(f'Invalid step {step_config.step_name}') diff --git a/benchmarks/perf-tool/okpt/test/steps/steps.py b/benchmarks/perf-tool/okpt/test/steps/steps.py deleted file mode 100644 index 7230bc1f4..000000000 --- a/benchmarks/perf-tool/okpt/test/steps/steps.py +++ /dev/null @@ -1,557 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. -"""Provides steps for OpenSearch tests. - -Some of the OpenSearch operations return a `took` field in the response body, -so the profiling decorators aren't needed for some functions. -""" -import json -from typing import Any, Dict, List - -import numpy as np -import requests -import time - -from opensearchpy import OpenSearch, RequestsHttpConnection - -from okpt.io.config.parsers.base import ConfigurationError -from okpt.io.config.parsers.util import parse_string_param, parse_int_param, parse_dataset, parse_bool_param -from okpt.io.dataset import Context -from okpt.io.utils.reader import parse_json_from_path -from okpt.test.steps import base -from okpt.test.steps.base import StepConfig - - -class OpenSearchStep(base.Step): - """See base class.""" - - def __init__(self, step_config: StepConfig): - super().__init__(step_config) - self.endpoint = parse_string_param('endpoint', step_config.config, - step_config.implicit_config, - 'localhost') - default_port = 9200 if self.endpoint == 'localhost' else 80 - self.port = parse_int_param('port', step_config.config, - step_config.implicit_config, default_port) - self.opensearch = get_opensearch_client(str(self.endpoint), - int(self.port)) - - -class CreateIndexStep(OpenSearchStep): - """See base class.""" - - label = 'create_index' - - def __init__(self, step_config: StepConfig): - super().__init__(step_config) - self.index_name = parse_string_param('index_name', step_config.config, - {}, None) - index_spec = parse_string_param('index_spec', step_config.config, {}, - None) - self.body = parse_json_from_path(index_spec) - if self.body is None: - raise ConfigurationError('Index body must be passed in') - - def _action(self): - """Creates an OpenSearch index, applying the index settings/mappings. - - Returns: - An OpenSearch index creation response body. - """ - self.opensearch.indices.create(index=self.index_name, body=self.body) - return {} - - def _get_measures(self) -> List[str]: - return ['took'] - - -class DisableRefreshStep(OpenSearchStep): - """See base class.""" - - label = 'disable_refresh' - - def _action(self): - """Disables the refresh interval for an OpenSearch index. - - Returns: - An OpenSearch index settings update response body. - """ - self.opensearch.indices.put_settings( - body={'index': { - 'refresh_interval': -1 - }}) - - return {} - - def _get_measures(self) -> List[str]: - return ['took'] - - -class RefreshIndexStep(OpenSearchStep): - """See base class.""" - - label = 'refresh_index' - - def __init__(self, step_config: StepConfig): - super().__init__(step_config) - self.index_name = parse_string_param('index_name', step_config.config, - {}, None) - - def _action(self): - while True: - try: - self.opensearch.indices.refresh(index=self.index_name) - return {'store_kb': get_index_size_in_kb(self.opensearch, - self.index_name)} - except: - pass - - def _get_measures(self) -> List[str]: - return ['took', 'store_kb'] - - -class ForceMergeStep(OpenSearchStep): - """See base class.""" - - label = 'force_merge' - - def __init__(self, step_config: StepConfig): - super().__init__(step_config) - self.index_name = parse_string_param('index_name', step_config.config, - {}, None) - self.max_num_segments = parse_int_param('max_num_segments', - step_config.config, {}, None) - - def _action(self): - while True: - try: - self.opensearch.indices.forcemerge( - index=self.index_name, - max_num_segments=self.max_num_segments) - return {} - except: - pass - - def _get_measures(self) -> List[str]: - return ['took'] - - -class TrainModelStep(OpenSearchStep): - """See base class.""" - - label = 'train_model' - - def __init__(self, step_config: StepConfig): - super().__init__(step_config) - - self.model_id = parse_string_param('model_id', step_config.config, {}, - 'Test') - self.train_index_name = parse_string_param('train_index', - step_config.config, {}, None) - self.train_index_field = parse_string_param('train_field', - step_config.config, {}, - None) - self.dimension = parse_int_param('dimension', step_config.config, {}, - None) - self.description = parse_string_param('description', step_config.config, - {}, 'Default') - self.max_training_vector_count = parse_int_param( - 'max_training_vector_count', step_config.config, {}, 10000000000000) - - method_spec = parse_string_param('method_spec', step_config.config, {}, - None) - self.method = parse_json_from_path(method_spec) - if self.method is None: - raise ConfigurationError('method must be passed in') - - def _action(self): - """Train a model for an index. - - Returns: - The trained model - """ - - # Build body - body = { - 'training_index': self.train_index_name, - 'training_field': self.train_index_field, - 'description': self.description, - 'dimension': self.dimension, - 'method': self.method, - 'max_training_vector_count': self.max_training_vector_count - } - - # So, we trained the model. Now we need to wait until we have to wait - # until the model is created. Poll every - # 1/10 second - requests.post('http://' + self.endpoint + ':' + str(self.port) + - '/_plugins/_knn/models/' + str(self.model_id) + '/_train', - json.dumps(body), - headers={'content-type': 'application/json'}) - - sleep_time = 0.1 - timeout = 100000 - i = 0 - while i < timeout: - time.sleep(sleep_time) - model_response = get_model(self.endpoint, self.port, self.model_id) - if 'state' in model_response.keys() and model_response['state'] == \ - 'created': - return {} - i += 1 - - raise TimeoutError('Failed to create model') - - def _get_measures(self) -> List[str]: - return ['took'] - - -class DeleteModelStep(OpenSearchStep): - """See base class.""" - - label = 'delete_model' - - def __init__(self, step_config: StepConfig): - super().__init__(step_config) - - self.model_id = parse_string_param('model_id', step_config.config, {}, - 'Test') - - def _action(self): - """Train a model for an index. - - Returns: - The trained model - """ - delete_model(self.endpoint, self.port, self.model_id) - return {} - - def _get_measures(self) -> List[str]: - return ['took'] - - -class DeleteIndexStep(OpenSearchStep): - """See base class.""" - - label = 'delete_index' - - def __init__(self, step_config: StepConfig): - super().__init__(step_config) - - self.index_name = parse_string_param('index_name', step_config.config, - {}, None) - - def _action(self): - """Delete the index - - Returns: - An empty dict - """ - delete_index(self.opensearch, self.index_name) - return {} - - def _get_measures(self) -> List[str]: - return ['took'] - - -class IngestStep(OpenSearchStep): - """See base class.""" - - label = 'ingest' - - def __init__(self, step_config: StepConfig): - super().__init__(step_config) - self.index_name = parse_string_param('index_name', step_config.config, - {}, None) - self.field_name = parse_string_param('field_name', step_config.config, - {}, None) - self.bulk_size = parse_int_param('bulk_size', step_config.config, {}, - 300) - self.implicit_config = step_config.implicit_config - dataset_format = parse_string_param('dataset_format', - step_config.config, {}, 'hdf5') - dataset_path = parse_string_param('dataset_path', step_config.config, - {}, None) - self.dataset = parse_dataset(dataset_format, dataset_path, - Context.INDEX) - - input_doc_count = parse_int_param('doc_count', step_config.config, {}, - self.dataset.size()) - self.doc_count = min(input_doc_count, self.dataset.size()) - - def _action(self): - - def action(doc_id): - return {'index': {'_index': self.index_name, '_id': doc_id}} - - # Maintain minimal state outside of this loop. For large data sets, too - # much state may cause out of memory failure - for i in range(0, self.doc_count, self.bulk_size): - partition = self.dataset.read(self.bulk_size) - if partition is None: - break - body = bulk_transform(partition, self.field_name, action, i) - bulk_index(self.opensearch, self.index_name, body) - - self.dataset.reset() - - return {} - - def _get_measures(self) -> List[str]: - return ['took'] - - -class QueryStep(OpenSearchStep): - """See base class.""" - - label = 'query' - - def __init__(self, step_config: StepConfig): - super().__init__(step_config) - self.k = parse_int_param('k', step_config.config, {}, 100) - self.r = parse_int_param('r', step_config.config, {}, 1) - self.index_name = parse_string_param('index_name', step_config.config, - {}, None) - self.field_name = parse_string_param('field_name', step_config.config, - {}, None) - self.calculate_recall = parse_bool_param('calculate_recall', - step_config.config, {}, False) - dataset_format = parse_string_param('dataset_format', - step_config.config, {}, 'hdf5') - dataset_path = parse_string_param('dataset_path', - step_config.config, {}, None) - self.dataset = parse_dataset(dataset_format, dataset_path, - Context.QUERY) - - input_query_count = parse_int_param('query_count', - step_config.config, {}, - self.dataset.size()) - self.query_count = min(input_query_count, self.dataset.size()) - - neighbors_format = parse_string_param('neighbors_format', - step_config.config, {}, 'hdf5') - neighbors_path = parse_string_param('neighbors_path', - step_config.config, {}, None) - self.neighbors = parse_dataset(neighbors_format, neighbors_path, - Context.NEIGHBORS) - self.implicit_config = step_config.implicit_config - - def _action(self): - - def get_body(vec): - return { - 'size': self.k, - 'query': { - 'knn': { - self.field_name: { - 'vector': vec, - 'k': self.k - } - } - } - } - - results = {} - query_responses = [] - for _ in range(self.query_count): - query = self.dataset.read(1) - if query is None: - break - query_responses.append( - query_index(self.opensearch, self.index_name, - get_body(query[0]), [self.field_name])) - - results['took'] = [ - float(query_response['took']) for query_response in query_responses - ] - results['memory_kb'] = get_cache_size_in_kb(self.endpoint, 80) - - if self.calculate_recall: - ids = [[int(hit['_id']) - for hit in query_response['hits']['hits']] - for query_response in query_responses] - results['recall@K'] = recall_at_r(ids, self.neighbors, - self.k, self.k, self.query_count) - self.neighbors.reset() - results[f'recall@{str(self.r)}'] = recall_at_r( - ids, self.neighbors, self.r, self.k, self.query_count) - self.neighbors.reset() - - self.dataset.reset() - - return results - - def _get_measures(self) -> List[str]: - measures = ['took', 'memory_kb'] - - if self.calculate_recall: - measures.extend(['recall@K', f'recall@{str(self.r)}']) - - return measures - - -# Helper functions - (AKA not steps) -def bulk_transform(partition: np.ndarray, field_name: str, action, - offset: int) -> List[Dict[str, Any]]: - """Partitions and transforms a list of vectors into OpenSearch's bulk - injection format. - Args: - offset: to start counting from - partition: An array of vectors to transform. - field_name: field name for action - action: Bulk API action. - Returns: - An array of transformed vectors in bulk format. - """ - actions = [] - _ = [ - actions.extend([action(i + offset), None]) - for i in range(len(partition)) - ] - actions[1::2] = [{field_name: vec} for vec in partition.tolist()] - return actions - - -def delete_index(opensearch: OpenSearch, index_name: str): - """Deletes an OpenSearch index. - - Args: - opensearch: An OpenSearch client. - index_name: Name of the OpenSearch index to be deleted. - """ - opensearch.indices.delete(index=index_name, ignore=[400, 404]) - - -def get_model(endpoint, port, model_id): - """ - Retrieve a model from an OpenSearch cluster - Args: - endpoint: Endpoint OpenSearch is running on - port: Port OpenSearch is running on - model_id: ID of model to be deleted - Returns: - Get model response - """ - response = requests.get('http://' + endpoint + ':' + str(port) + - '/_plugins/_knn/models/' + model_id, - headers={'content-type': 'application/json'}) - return response.json() - - -def delete_model(endpoint, port, model_id): - """ - Deletes a model from OpenSearch cluster - Args: - endpoint: Endpoint OpenSearch is running on - port: Port OpenSearch is running on - model_id: ID of model to be deleted - Returns: - Deleted model response - """ - response = requests.delete('http://' + endpoint + ':' + str(port) + - '/_plugins/_knn/models/' + model_id, - headers={'content-type': 'application/json'}) - return response.json() - - -def get_opensearch_client(endpoint: str, port: int): - """ - Get an opensearch client from an endpoint and port - Args: - endpoint: Endpoint OpenSearch is running on - port: Port OpenSearch is running on - Returns: - OpenSearch client - - """ - # TODO: fix for security in the future - return OpenSearch( - hosts=[{ - 'host': endpoint, - 'port': port - }], - use_ssl=False, - verify_certs=False, - connection_class=RequestsHttpConnection, - timeout=60, - ) - - -def recall_at_r(results, neighbor_dataset, r, k, query_count): - """ - Calculates the recall@R for a set of queries against a ground truth nearest - neighbor set - Args: - results: 2D list containing ids of results returned by OpenSearch. - results[i][j] i refers to query, j refers to - result in the query - neighbor_dataset: 2D dataset containing ids of the true nearest - neighbors for a set of queries - r: number of top results to check if they are in the ground truth k-NN - set. - k: k value for the query - query_count: number of queries - Returns: - Recall at R - """ - correct = 0.0 - for query in range(query_count): - true_neighbors = neighbor_dataset.read(1) - if true_neighbors is None: - break - true_neighbors_set = set(true_neighbors[0][:k]) - for j in range(r): - if results[query][j] in true_neighbors_set: - correct += 1.0 - - return correct / (r * query_count) - - -def get_index_size_in_kb(opensearch, index_name): - """ - Gets the size of an index in kilobytes - Args: - opensearch: opensearch client - index_name: name of index to look up - Returns: - size of index in kilobytes - """ - return int( - opensearch.indices.stats(index_name, metric='store')['indices'] - [index_name]['total']['store']['size_in_bytes']) / 1024 - - -def get_cache_size_in_kb(endpoint, port): - """ - Gets the size of the k-NN cache in kilobytes - Args: - endpoint: endpoint of OpenSearch cluster - port: port of endpoint OpenSearch is running on - Returns: - size of cache in kilobytes - """ - response = requests.get('http://' + endpoint + ':' + str(port) + - '/_plugins/_knn/stats', - headers={'content-type': 'application/json'}) - stats = response.json() - - keys = stats['nodes'].keys() - - total_used = 0 - for key in keys: - total_used += int(stats['nodes'][key]['graph_memory_usage']) - return total_used - - -def query_index(opensearch: OpenSearch, index_name: str, body: dict, - excluded_fields: list): - return opensearch.search(index=index_name, - body=body, - _source_excludes=excluded_fields) - - -def bulk_index(opensearch: OpenSearch, index_name: str, body: List): - return opensearch.bulk(index=index_name, body=body, timeout='5m') diff --git a/benchmarks/perf-tool/okpt/test/test.py b/benchmarks/perf-tool/okpt/test/test.py deleted file mode 100644 index dbd65d053..000000000 --- a/benchmarks/perf-tool/okpt/test/test.py +++ /dev/null @@ -1,180 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. - -"""Provides a base Test class.""" -from math import floor -from typing import Any, Dict, List - -from okpt.io.config.parsers.test import TestConfig -from okpt.test.steps.base import Step - - -def get_avg(values: List[Any]): - """Get average value of a list. - - Args: - values: A list of values. - - Returns: - The average value in the list. - """ - valid_total = len(values) - running_sum = 0.0 - - for value in values: - if value == -1: - valid_total -= 1 - continue - running_sum += value - - if valid_total == 0: - return -1 - return running_sum / valid_total - - -def _pxx(values: List[Any], p: float): - """Calculates the pXX statistics for a given list. - - Args: - values: List of values. - p: Percentile (between 0 and 1). - - Returns: - The corresponding pXX metric. - """ - lowest_percentile = 1 / len(values) - highest_percentile = (len(values) - 1) / len(values) - - # return -1 if p is out of range or if the list doesn't have enough elements - # to support the specified percentile - if p < 0 or p > 1: - return -1.0 - elif p < lowest_percentile or p > highest_percentile: - return -1.0 - else: - return float(values[floor(len(values) * p)]) - - -def _aggregate_steps(step_results: List[Dict[str, Any]], - measure_labels=None): - """Aggregates the steps for a given Test. - - The aggregation process extracts the measures from each step and calculates - the total time spent performing each step measure, including the - percentile metrics, if possible. - - The aggregation process also extracts the test measures by simply summing - up the respective step measures. - - A step measure is formatted as `{step_name}_{measure_name}`, for example, - {bulk_index}_{took} or {query_index}_{memory}. The braces are not included - in the actual key string. - - Percentile/Total step measures are give as - `{step_name}_{measure_name}_{percentile|total}`. - - Test measures are just step measure sums so they just given as - `test_{measure_name}`. - - Args: - steps: List of test steps to be aggregated. - measures: List of step metrics to account for. - - Returns: - A complete test result. - """ - if measure_labels is None: - measure_labels = ['took'] - test_measures = { - f'test_{measure_label}': 0 - for measure_label in measure_labels - } - step_measures: Dict[str, Any] = {} - - # iterate over all test steps - for step in step_results: - step_label = step['label'] - - step_measure_labels = list(step.keys()) - step_measure_labels.remove('label') - - # iterate over all measures in each test step - for measure_label in step_measure_labels: - - step_measure = step[measure_label] - step_measure_label = f'{step_label}_{measure_label}' - - # Add cumulative test measures from steps to test measures - if measure_label in measure_labels: - test_measures[f'test_{measure_label}'] += sum(step_measure) if \ - isinstance(step_measure, list) else step_measure - - if step_measure_label in step_measures: - _ = step_measures[step_measure_label].extend(step_measure) \ - if isinstance(step_measure, list) else \ - step_measures[step_measure_label].append(step_measure) - else: - step_measures[step_measure_label] = step_measure if \ - isinstance(step_measure, list) else [step_measure] - - aggregate = {**test_measures} - # calculate the totals and percentile statistics for each step measure - # where relevant - for step_measure_label, step_measure in step_measures.items(): - step_measure.sort() - - aggregate[step_measure_label + '_total'] = float(sum(step_measure)) - - p50 = _pxx(step_measure, 0.50) - if p50 != -1: - aggregate[step_measure_label + '_p50'] = p50 - p90 = _pxx(step_measure, 0.90) - if p90 != -1: - aggregate[step_measure_label + '_p90'] = p90 - p99 = _pxx(step_measure, 0.99) - if p99 != -1: - aggregate[step_measure_label + '_p99'] = p99 - - return aggregate - - -class Test: - """A base Test class, representing a collection of steps to profiled and - aggregated. - - Methods: - setup: Performs test setup. Usually for steps not intended to be - profiled. - run_steps: Runs the test steps, aggregating the results into the - `step_results` instance field. - cleanup: Perform test cleanup. Useful for clearing the state of a - persistent process like OpenSearch. Cleanup steps are executed after - each run. - execute: Runs steps, cleans up, and aggregates the test result. - """ - def __init__(self, test_config: TestConfig): - """Initializes the test state. - """ - self.test_config = test_config - self.setup_steps: List[Step] = test_config.setup - self.test_steps: List[Step] = test_config.steps - self.cleanup_steps: List[Step] = test_config.cleanup - - def setup(self): - _ = [step.execute() for step in self.setup_steps] - - def _run_steps(self): - step_results = [] - _ = [step_results.extend(step.execute()) for step in self.test_steps] - return step_results - - def _cleanup(self): - _ = [step.execute() for step in self.cleanup_steps] - - def execute(self): - results = self._run_steps() - self._cleanup() - return _aggregate_steps(results) diff --git a/benchmarks/perf-tool/requirements.in b/benchmarks/perf-tool/requirements.in deleted file mode 100644 index fd3555aab..000000000 --- a/benchmarks/perf-tool/requirements.in +++ /dev/null @@ -1,7 +0,0 @@ -Cerberus -opensearch-py -PyYAML -numpy -h5py -requests -psutil diff --git a/benchmarks/perf-tool/requirements.txt b/benchmarks/perf-tool/requirements.txt deleted file mode 100644 index 3b9946aad..000000000 --- a/benchmarks/perf-tool/requirements.txt +++ /dev/null @@ -1,39 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: -# -# pip-compile -# -cached-property==1.5.2 - # via h5py -cerberus==1.3.4 - # via -r requirements.in -certifi==2023.7.22 - # via - # opensearch-py - # requests -charset-normalizer==2.0.4 - # via requests -h5py==3.3.0 - # via -r requirements.in -idna==3.2 - # via requests -numpy==1.22.1 - # via - # -r requirements.in - # h5py -opensearch-py==1.0.0 - # via -r requirements.in -psutil==5.8.0 - # via -r requirements.in -pyyaml==5.4.1 - # via -r requirements.in -requests==2.31.0 - # via -r requirements.in -urllib3==1.26.6 - # via - # opensearch-py - # requests - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/benchmarks/perf-tool/sample-configs/faiss-sift-ivf/index-spec.json b/benchmarks/perf-tool/sample-configs/faiss-sift-ivf/index-spec.json deleted file mode 100644 index 5542ef387..000000000 --- a/benchmarks/perf-tool/sample-configs/faiss-sift-ivf/index-spec.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "settings": { - "index": { - "knn": true, - "number_of_shards": 3, - "number_of_replicas": 0 - } - }, - "mappings": { - "properties": { - "target_field": { - "type": "knn_vector", - "model_id": "test-model" - } - } - } -} diff --git a/benchmarks/perf-tool/sample-configs/faiss-sift-ivf/method-spec.json b/benchmarks/perf-tool/sample-configs/faiss-sift-ivf/method-spec.json deleted file mode 100644 index 1aa7f809f..000000000 --- a/benchmarks/perf-tool/sample-configs/faiss-sift-ivf/method-spec.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "name":"ivf", - "engine":"faiss", - "parameters":{ - "nlist":16, - "nprobes": 4 - } -} diff --git a/benchmarks/perf-tool/sample-configs/faiss-sift-ivf/test.yml b/benchmarks/perf-tool/sample-configs/faiss-sift-ivf/test.yml deleted file mode 100644 index c8fb42ec4..000000000 --- a/benchmarks/perf-tool/sample-configs/faiss-sift-ivf/test.yml +++ /dev/null @@ -1,60 +0,0 @@ -endpoint: localhost -test_name: faiss_sift_ivf -test_id: "Test workflow for faiss ivf" -num_runs: 3 -show_runs: true -setup: - - name: delete_model - model_id: test-model - - name: delete_index - index_name: target_index - - name: delete_index - index_name: train_index - - name: create_index - index_name: train_index - index_spec: sample-configs/faiss-sift-ivf/train-index-spec.json - - name: ingest - index_name: train_index - field_name: train_field - bulk_size: 500 - dataset_format: hdf5 - dataset_path: ../dataset/sift-128-euclidean.hdf5 - - name: refresh_index - index_name: train_index -steps: - - name: train_model - model_id: test-model - train_index: train_index - train_field: train_field - dimension: 128 - method_spec: sample-configs/faiss-sift-ivf/method-spec.json - max_training_vector_count: 1000000000 - - name: create_index - index_name: target_index - index_spec: sample-configs/faiss-sift-ivf/index-spec.json - - name: ingest - index_name: target_index - field_name: target_field - bulk_size: 500 - dataset_format: hdf5 - dataset_path: ../dataset/sift-128-euclidean.hdf5 - - name: refresh_index - index_name: target_index - - name: force_merge - index_name: target_index - max_num_segments: 10 - - name: query - k: 100 - r: 1 - calculate_recall: true - index_name: target_index - field_name: target_field - dataset_format: hdf5 - dataset_path: ../dataset/sift-128-euclidean.hdf5 - neighbors_format: hdf5 - neighbors_path: ../dataset/sift-128-euclidean.hdf5 -cleanup: - - name: delete_model - model_id: test-model - - name: delete_index - index_name: target_index diff --git a/benchmarks/perf-tool/sample-configs/faiss-sift-ivf/train-index-spec.json b/benchmarks/perf-tool/sample-configs/faiss-sift-ivf/train-index-spec.json deleted file mode 100644 index 00a418e4f..000000000 --- a/benchmarks/perf-tool/sample-configs/faiss-sift-ivf/train-index-spec.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "settings": { - "index": { - "number_of_shards": 3, - "number_of_replicas": 0 - } - }, - "mappings": { - "properties": { - "train_field": { - "type": "knn_vector", - "dimension": 128 - } - } - } -} diff --git a/benchmarks/perf-tool/sample-configs/nmslib-sift-hnsw/index-spec.json b/benchmarks/perf-tool/sample-configs/nmslib-sift-hnsw/index-spec.json deleted file mode 100644 index 75abe7baa..000000000 --- a/benchmarks/perf-tool/sample-configs/nmslib-sift-hnsw/index-spec.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "settings": { - "index": { - "knn": true, - "knn.algo_param.ef_search": 512, - "refresh_interval": "10s", - "number_of_shards": 1, - "number_of_replicas": 0 - } - }, - "mappings": { - "properties": { - "target_field": { - "type": "knn_vector", - "dimension": 128, - "method": { - "name": "hnsw", - "space_type": "l2", - "engine": "nmslib", - "parameters": { - "ef_construction": 512, - "m": 16 - } - } - } - } - } -} diff --git a/benchmarks/perf-tool/sample-configs/nmslib-sift-hnsw/test.yml b/benchmarks/perf-tool/sample-configs/nmslib-sift-hnsw/test.yml deleted file mode 100644 index deea1ad47..000000000 --- a/benchmarks/perf-tool/sample-configs/nmslib-sift-hnsw/test.yml +++ /dev/null @@ -1,36 +0,0 @@ -endpoint: localhost -test_name: nmslib_sift_hnsw -test_id: "Test workflow for nmslib hnsw" -num_runs: 2 -show_runs: false -setup: - - name: delete_index - index_name: target_index -steps: - - name: create_index - index_name: target_index - index_spec: sample-configs/nmslib-sift-hnsw/index-spec.json - - name: ingest - index_name: target_index - field_name: target_field - bulk_size: 500 - dataset_format: hdf5 - dataset_path: ../dataset/sift-128-euclidean.hdf5 - - name: refresh_index - index_name: target_index - - name: force_merge - index_name: target_index - max_num_segments: 10 - - name: query - k: 100 - r: 1 - calculate_recall: true - index_name: target_index - field_name: target_field - dataset_format: hdf5 - dataset_path: ../dataset/sift-128-euclidean.hdf5 - neighbors_format: hdf5 - neighbors_path: ../dataset/sift-128-euclidean.hdf5 -cleanup: - - name: delete_index - index_name: target_index