diff --git a/advocacy_docs/edb-postgres-ai/ai-accelerator/capabilities.mdx b/advocacy_docs/edb-postgres-ai/ai-accelerator/capabilities.mdx index 1b263fbabf8..cc2672f1c69 100644 --- a/advocacy_docs/edb-postgres-ai/ai-accelerator/capabilities.mdx +++ b/advocacy_docs/edb-postgres-ai/ai-accelerator/capabilities.mdx @@ -18,18 +18,18 @@ This storage location can be an S3 bucket or a local file system. The storage locations can be used to create a volume, suitable for a retriever to use to access the data it contains. -### A model is registered +### A model is created -A [model](models) is registered with the Pipelines system. This model can be a machine learning model, a deep learning model, or any other type of model that can be used for AI tasks. +A [model](models) is created with the Pipelines system. This model can be a machine learning model, a deep learning model, or any other type of model that can be used for AI tasks. -### A retriever is registered +### A retriever is created -A retriever is registered with the Pipelines system. A retriever is a function that retrieves data from a table or volume and returns it in a format that can be used by the model. +A retriever is created with the Pipelines system. A retriever is a function that retrieves data from a table or volume and returns it in a format that can be used by the model. By default, a retriever only needs: * a name -* the name of a registered model to use +* the name of a model to use If the retriever is for a table, it also needs: @@ -42,10 +42,10 @@ If, on the other hand, the retriever is for a volume, it needs: * the name of the volume * the name of the column in the volume that contains the data -When a retriever is registered, by default it will create a vector table to store the embeddings of the data that is retrieved. +When a retriever is created, by default it will create a vector table to store the embeddings of the data that is retrieved. This table will have a column to store the embeddings and a column to store the key of the data. -The name of the vector table and the name of the vector column and the key column can be specified when the retriever is registered; this is useful if you are migrating to aidb and want to use an existing vector table. +The name of the vector table and the name of the vector column and the key column can be specified when the retriever is created; this is useful if you are migrating to aidb and want to use an existing vector table. ### Embeddings are created @@ -65,4 +65,4 @@ While auto-embedding is enabled, the embeddings are always up-to-date and applic ### Cleanup -If the embeddings are no longer required, the retriever can be unregistered, the vector table can be dropped and the model can be unregistered too. +If the embeddings are no longer required, the retriever can be deleted, the vector table can be dropped and the model can be deleted. diff --git a/advocacy_docs/edb-postgres-ai/ai-accelerator/gettingstarted/index.mdx b/advocacy_docs/edb-postgres-ai/ai-accelerator/gettingstarted/index.mdx index abe67250290..ad31b506daf 100644 --- a/advocacy_docs/edb-postgres-ai/ai-accelerator/gettingstarted/index.mdx +++ b/advocacy_docs/edb-postgres-ai/ai-accelerator/gettingstarted/index.mdx @@ -58,14 +58,14 @@ INSERT 0 9 So now we have a table with some data in it, food products and some very personal opinions about them. -## Registering a Retriever +## Creating a Retriever -The first step to using Pipelines with this data is to register a retriever. A retriever is a way to access the data in the table and use it in AI workflows. +The first step to using Pipelines with this data is to create a retriever. A retriever is a way to access the data in the table and use it in AI workflows. ```sql -select aidb.register_retriever_for_table('products_retriever', 't5', 'products', 'description', 'Text'); +select aidb.create_retriever_for_table('products_retriever', 't5', 'products', 'description', 'Text'); __OUTPUT__ - register_retriever_for_table + create_retriever_for_table ------------------------------ products_retriever (1 row) @@ -73,7 +73,7 @@ __OUTPUT__ ## Querying the retriever -Now that we have a retriever registered, we can query it to get similar results based on the data in the table. +Now that we have created a retriever, we can query it to get similar results based on the data in the table. ```sql select * from aidb.retrieve_key('products_retriever','I like it',5); @@ -189,6 +189,6 @@ __OUTPUT__ ## Further reading -In the [Models](../models) section, you can learn how to register more models with Pipelines, including external models from OpenAI API compatible services. +In the [Models](../models) section, you can learn how to create more models with Pipelines, including external models from OpenAI API compatible services. In the [Retrievers](../retrievers) section, you can learn more about how to use retrievers with external data sources, local files or S3 storage, and how to use the retriever functions to get the data you need. diff --git a/advocacy_docs/edb-postgres-ai/ai-accelerator/index.mdx b/advocacy_docs/edb-postgres-ai/ai-accelerator/index.mdx index 57cb755e571..89ec27c4c44 100644 --- a/advocacy_docs/edb-postgres-ai/ai-accelerator/index.mdx +++ b/advocacy_docs/edb-postgres-ai/ai-accelerator/index.mdx @@ -4,7 +4,7 @@ navTitle: "AI Accelerator" directoryDefaults: product: "EDB Postgres AI" iconName: BrainCircuit -indexCards: simple +indexCards: extra description: "All about the EDB Postgres AI - AI Accelerator suite of tools including Pipelines and PGvector." navigation: - overview @@ -15,7 +15,7 @@ navigation: - limitations - compatibility - installing -- "#Piplelines components" +- "#Pipelines components" - models - retrievers - pgfs diff --git a/advocacy_docs/edb-postgres-ai/ai-accelerator/models/index.mdx b/advocacy_docs/edb-postgres-ai/ai-accelerator/models/index.mdx index 9603f222942..09846294913 100644 --- a/advocacy_docs/edb-postgres-ai/ai-accelerator/models/index.mdx +++ b/advocacy_docs/edb-postgres-ai/ai-accelerator/models/index.mdx @@ -6,13 +6,15 @@ navigation: - using-models - primitives - supported-models +- openai-api-compatibility --- -Pipelines has a model registry that manages configured instances of models. Any Pipelines functions that use models, such as embedding and retrieving, must reference a registered model. +Pipelines has a model registry that manages configured instances of models. Any Pipelines functions that use models, such as embedding and retrieving, must reference a created model. -* Learn how to [register models](./using-models) in Pipelines. +* Learn how to [create models](./using-models) in Pipelines. * Discover the [primitives](./primitives) that can be used to interact with models. * See the [supported models](./supported-models) that come with Pipelines. +* Learn how to use [OpenAI API compatible services](./openai-api-compatibility) with Pipelines. ## Next Steps diff --git a/advocacy_docs/edb-postgres-ai/ai-accelerator/models/openai-api-compatibility.mdx b/advocacy_docs/edb-postgres-ai/ai-accelerator/models/openai-api-compatibility.mdx new file mode 100644 index 00000000000..42be732e6d2 --- /dev/null +++ b/advocacy_docs/edb-postgres-ai/ai-accelerator/models/openai-api-compatibility.mdx @@ -0,0 +1,69 @@ +--- +title: "Using an OpenAI compatible API with Pipelines" +navTitle: "OpenAI Compatible Models" +description: "Using an OpenAI compatible API with Pipelines by setting options and credentials." +--- + +To make use of an OpenAI compliant API, you can use the openai_embeddings or openai_completions model providers. Note that a retriever will need to encode first so you can only use the embeddings model provider with a retriever. + +## Why use an OpenAI compatible API? + +Some examples of why you might want to use an OpenAI compatible API include: + +* If you have a local system running [Ollama](https://ollama.com) and, assuming you have configured [Ollama as a server](https://github.com/ollama/ollama/blob/main/docs/faq.md#how-do-i-configure-ollama-server), then you may want that local system to handle embeddings. + +* If you have access to a service which provides different or specifically tuned models, you can use it instead of other models. + +## Creating the model + +The starting point for this process is creating a model. When you create a model, you can pass `options` and `credentials` to the registration. The defaults will point to the OpenAI service endpoint, so by overriding that, you can point to any service. Here is an example that will create a model that uses a local Ollama server: + +```sql +select aidb.create_model( +'my_local_ollama', +'openai_embeddings', +'{"model":"llama3.3", "url":"http://llama.local:11434/v1/embeddings", "dimensions":8192}'::JSONB, +'{"api_key":""}'::JSONB); +``` + +### Model name and model provider + +The model name is the first parameter and set to “my_local_ollama” which we will use later. + +We specify the model provider as “openai_embeddings” which is the provider that defaults to using OpenAI servers, but can be overridden by the configuration (the next parameter), to talk to any compliant server. + +### Configuration + +The next parameter is the configuration. This is a JSON string, which when expanded has three parameters, the model, the url and the dimensions. + +```json +'{"model":"llama3.3", "url":"http://llama.local:11434/v1/embeddings", "dimensions":8192}'::JSONB +``` + +In this case, we are setting the model to [“llama3.3”](https://ollama.com/library/llama3.3), a relatively new and powerful model. Remember to run `ollama run llama3.3` to pull and start the model on the server. + +The next json setting is the important one, overriding the endpoint that the aidb model will use. + +* Our server is running on a machine called `llama.local`. +* It has port 11434 (the default port for Ollama) open to service requests over HTTP (not HTTPS in this case). +* The path to the endpoint on the server `/v1/embeddings`; the same as OpenAI. + +Putting those components together we get `[`http://llama.local:11434/v1/embeddings`](http://art.local:11434/v1/embeddings","api_key":"","dimensions":8192}'::JSONB)` as our end point. + +The last JSON parameter in this example is “dimensions” which is a hint to the system about how many vector values to expect from the model. If we [look up llama3.3’s properties](https://ollama.com/library/llama3.3/blobs/4824460d29f2) we can see the `llama.embedding_length` value is 8192\. The provider defaults to 1536 (with some hard-wired exceptions depending on model) but it doesn’t know about llama3.3, so we have to pass the dimension value of 8192 in the configuration. + +That completes the configuration parameter. + +### Credentials + +The last parameter is the credentials parameter, which is another JSON string. It’s usually used for carrying the `api_key` for the OpenAI service and any other necessary credential information. It is not part of the configuration and by being separate, it can be securely hidden from users with lesser permissions. For our ollama connection, we don’t need an api\_key, but the model provider currently requires that one is specified. We can specify an empty string for the api\_key to satisfy this requirement. + +## Using the model + +Use the model name you created earlier to use the model just like any other Pipelines model. Here is an example of how to use the model to get an embedding: + +```sql +select aidb.encode_text('my_local_ollama','I like it'); +``` + +Pipelines will take care of all the connection management leaving you to focus on your data and the model results. diff --git a/advocacy_docs/edb-postgres-ai/ai-accelerator/models/supported-models/bert.mdx b/advocacy_docs/edb-postgres-ai/ai-accelerator/models/supported-models/bert.mdx index f483e7f3a5d..6471449872b 100644 --- a/advocacy_docs/edb-postgres-ai/ai-accelerator/models/supported-models/bert.mdx +++ b/advocacy_docs/edb-postgres-ai/ai-accelerator/models/supported-models/bert.mdx @@ -31,17 +31,18 @@ Read more about [BERT on Wikipedia](https://en.wikipedia.org/wiki/BERT_(language * sentence-transformers/paraphrase-multilingual-mpnet-base-v2 * sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2 -## Register the default implementation - +## Creating the default model ```sql -SELECT aidb.register_model('my_bert_model', 'bert_local'); +SELECT aidb.create_model('my_bert_model', 'bert_local'); ``` -## Register another model +## Creating a specific model + +You can specify a model and revision in the options JSONB object. In this example, we are creating a `sentence-transformers/all-distilroberta-v1` model with the name `another_bert_model`: ```sql -select aidb.register_model( +select aidb.create_model( 'another_bert_model', 'bert_local', '{"model": "sentence-transformers/all-distilroberta-v1", "revision": "main"}'::JSONB diff --git a/advocacy_docs/edb-postgres-ai/ai-accelerator/models/supported-models/clip.mdx b/advocacy_docs/edb-postgres-ai/ai-accelerator/models/supported-models/clip.mdx index 124e26e51fa..4881e3bf0a3 100644 --- a/advocacy_docs/edb-postgres-ai/ai-accelerator/models/supported-models/clip.mdx +++ b/advocacy_docs/edb-postgres-ai/ai-accelerator/models/supported-models/clip.mdx @@ -24,15 +24,15 @@ Read more about [CLIP on OpenAI's website](https://openai.com/research/clip/). * openai/clip-vit-base-patch32 (default) -## Register the default implementation +## Creating the default model ```sql -SELECT aidb.register_model('my_clip_model', 'clip_local'); +SELECT aidb.create_model('my_clip_model', 'clip_local'); ``` There is only one model, the default `openai/clip-vit-base-patch32`, so we do not need to specify the model in the configuration. No credentials are required for the CLIP model. -## Register another model +## Creating a specific model There are no other model configurations available for the CLIP model. diff --git a/advocacy_docs/edb-postgres-ai/ai-accelerator/models/supported-models/openai-completions.mdx b/advocacy_docs/edb-postgres-ai/ai-accelerator/models/supported-models/openai-completions.mdx index b4172289534..affa3bc4abf 100644 --- a/advocacy_docs/edb-postgres-ai/ai-accelerator/models/supported-models/openai-completions.mdx +++ b/advocacy_docs/edb-postgres-ai/ai-accelerator/models/supported-models/openai-completions.mdx @@ -21,18 +21,18 @@ See a list of supported OpenAI models [here](https://platform.openai.com/docs/mo * Any text generation model that is supported by OpenAI. This includes models such as GPT-4o, GPT-4o mini, GPT-4 and GPT-3.5. -## Registering the default model +## Creating the default model -There is no default model for OpenAI Completions. You can register any supported OpenAI model using the `aidb.register_model` function. See [Registering a model](#registering-a-model). +There is no default model for OpenAI Completions. You can create any supported OpenAI model using the `aidb.create_model` function. See [Creating a model](#creating-a-specific-model). -## Registering a model +## Creating a specific model -You can register any supported OpenAI model using the `aidb.register_model` function. +You can create any supported OpenAI model using the `aidb.create_model` function. -In this example, we are registering a GPT-4o model with the name `my_openai_model`: +In this example, we are creating a GPT-4o model with the name `my_openai_model`: ```sql -SELECT aidb.register_model( +SELECT aidb.create_model( 'my_openai_model', 'openai_completions', '{"model": "gpt-4o"}::JSONB, diff --git a/advocacy_docs/edb-postgres-ai/ai-accelerator/models/supported-models/openai-embeddings.mdx b/advocacy_docs/edb-postgres-ai/ai-accelerator/models/supported-models/openai-embeddings.mdx index 50a0369b8ef..4a18cdd522d 100644 --- a/advocacy_docs/edb-postgres-ai/ai-accelerator/models/supported-models/openai-embeddings.mdx +++ b/advocacy_docs/edb-postgres-ai/ai-accelerator/models/supported-models/openai-embeddings.mdx @@ -22,22 +22,22 @@ See a list of supported OpenAI models [here](https://platform.openai.com/docs/gu * Any text embedding model that is supported by OpenAI. This includes `text-embedding-3-small`, `text-embedding-3-large`, and `text-embedding-ada-002`. * Defaults to `text-embedding-3-small`. -## Registering the default model +## Creating the default model ```sql -SELECT aidb.register_model('my_openai_embeddings', +SELECT aidb.create_model('my_openai_embeddings', 'openai_embeddings', credentials=>'{"api_key": "sk-abc123xyz456def789ghi012jkl345mn"'::JSONB); ``` As we are defaulting the model to `text-embedding-3-small`, we do not need to specify the model in the configuration. But we do need to pass an OpenAI API key in the credentials, and for that we have to pass credentials as a named parameter. -## Registering a model +## Creating a specific model -You can register any supported OpenAI embedding model using the `aidb.register_model` function. In this example, we are registering a `text-embedding-3-large` model with the name `my_openai_model`: +You can create any supported OpenAI embedding model using the `aidb.create_model` function. In this example, we are creating a `text-embedding-3-large` model with the name `my_openai_model`: ```sql -SELECT aidb.register_model( +SELECT aidb.create_model( 'my_openai_model', 'openai_embeddings', '{"model": "text-embedding-3-large"}'::JSONB, @@ -55,6 +55,20 @@ The following configuration settings are available for OpenAI models: * `url` - The URL of the OpenAI model to use. This is optional and can be used to specify a custom model URL. Defaults to `https://api.openai.com/v1/chat/completions`. * `max_concurrent_requests` - The maximum number of concurrent requests to make to the OpenAI model. Defaults to `25`. +## Available OpenAI Embeddings models + +* sentence-transformers/all-MiniLM-L6-v2 (default) +* sentence-transformers/all-MiniLM-L6-v1 +* sentence-transformers/all-MiniLM-L12-v1 +* sentence-transformers/msmarco-bert-base-dot-v5 +* sentence-transformers/multi-qa-MiniLM-L6-dot-v1 +* sentence-transformers/paraphrase-TinyBERT-L6-v2 +* sentence-transformers/all-distilroberta-v1 +* sentence-transformers/all-MiniLM-L6-v2 +* sentence-transformers/multi-qa-MiniLM-L6-cos-v1 +* sentence-transformers/paraphrase-multilingual-mpnet-base-v2 +* sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2 + ## Model credentials The following credentials are required for OpenAI models: diff --git a/advocacy_docs/edb-postgres-ai/ai-accelerator/models/supported-models/t5.mdx b/advocacy_docs/edb-postgres-ai/ai-accelerator/models/supported-models/t5.mdx index 5c6bbf20900..9a65671a977 100644 --- a/advocacy_docs/edb-postgres-ai/ai-accelerator/models/supported-models/t5.mdx +++ b/advocacy_docs/edb-postgres-ai/ai-accelerator/models/supported-models/t5.mdx @@ -27,16 +27,16 @@ Read more about [T5 on Wikipedia](https://en.wikipedia.org/wiki/T5_(language_mod * t5-3b * t5-11b -## Registering the default model +## Creating the default model ```sql -SELECT aidb.register_model('my_t5_model', 't5_local'); +SELECT aidb.create_model('my_t5_model', 't5_local'); ``` -## Registering a specific model +## Creating a specific model ```sql -SELECT aidb.register_model( +SELECT aidb.create_model( 'another_t5_model', 't5_local', '{"model": "t5-large", "revision": "main"}'::JSONB diff --git a/advocacy_docs/edb-postgres-ai/ai-accelerator/models/using-models.mdx b/advocacy_docs/edb-postgres-ai/ai-accelerator/models/using-models.mdx index 188a0584147..fa6a66197e0 100644 --- a/advocacy_docs/edb-postgres-ai/ai-accelerator/models/using-models.mdx +++ b/advocacy_docs/edb-postgres-ai/ai-accelerator/models/using-models.mdx @@ -1,22 +1,22 @@ --- title: Using Models in AI Accelerator Pipelines navTitle: Using Models -description: How to register and use models in AI Accelerator Pipelines. +description: How to create and use models in AI Accelerator Pipelines. --- -Pipelines has a model registry that manages configured instances of models. Any Pipelines functions that use models, such as embedding and retrieving, must reference a registered model. +Pipelines has a model registry that manages configured instances of models. Any Pipelines functions that use models, such as embedding and retrieving, must reference a model in this registry. ## Discover the preloaded models -Pipelines comes with a set of pre-registerd models that you can use out of the box. +Pipelines comes with a set of pre-created models that you can use out of the box. To find them, you can run the following query: ```sql -SELECT * FROM aidb.list_registered_models(); +SELECT * FROM aidb.list_models(); ``` -This will return a list of all the models that are currently registered in the system. If you have not registered any models, you'll see the default models that come with Pipelines. +This will return a list of all the models that are currently created in the system. If you have not created any models, you'll see the default models that come with Pipelines. ```text name | provider | options @@ -27,38 +27,58 @@ This will return a list of all the models that are currently registered in the s dummy | dummy | {"config={}"} ``` -The `bert`, `clip`, and `t5` models are all registered and ready to use. The `dummy` model is a placeholder model that can be used for testing purposes. +The `bert`, `clip`, and `t5` models are all pre-created and ready to use. The `dummy` model is a placeholder model that can be used for testing purposes. -## Registering a Model +## Creating a Model -You can also register your own models. To do this, you can use the `aidb.register_model` function. Here is an example of how to register a model: +You can also create your own models. To do this, you can use the `aidb.create_model` function. Here is an example of how to create a model: ```sql -SELECT aidb.register_model('my_model', 'bert_local'); +SELECT aidb.create_model('my_model', 'bert_local'); ``` -This will register a model named `my_model` that uses the default `bert_local` model provider. But, this is essentially the same as using the bert model thats already registered. +This will create a model named `my_model` that uses the default `bert_local` model provider. But, this is essentially the same as using the bert model thats already created. -## Registering a Model with a Configuration +## Discovering the Model Providers -You can also pass options to the model when registering it. For example, you can specify the model configuration: +You can also find out what model providers are available by running the following query: ```sql -SELECT aidb.register_model('my_model', +SELECT * FROM aidb.model_providers; +__OUTPUT__ + server_name | server_options +--------------------+---------------- + t5_local | + openai_embeddings | + openai_completions | + bert_local | + clip_local | + dummy | +``` + +This will return a list of all the model providers that are currently available in the system. You can find out more about these providers and their capabilities in the [Supported Models](./supported-models) section. + +## Creating a Model with a Configuration + +You can also pass options to the model when creating it. For example, you can specify the model configuration: + +```sql +SELECT aidb.create_model('my_model', 'bert_local', - '{"model": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", "revision": "main"}'::JSONB); + '{"model": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", + "revision": "main"}'::JSONB); ``` -This will register a model named `my_model` that uses the `bert_local` model provider and the `sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2` model from HuggingFace. The `revision` option specifies the version of the model to use. The options are passed as a JSONB object, with a single quoted string that is then cast to JSONB. Within the string are the key-value pairs that define the model configuration in a single JSON object. +This will create a model named `my_model` that uses the `bert_local` model provider and the `sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2` model from HuggingFace. The `revision` option specifies the version of the model to use. The options are passed as a JSONB object, with a single quoted string that is then cast to JSONB. Within the string are the key-value pairs that define the model configuration in a single JSON object. -## Registering a Model with Configuration and Credentials +## Creating a Model with Configuration and Credentials -This is where the other [supported models](./supported-models) come in. You can register a different model by specifying the model name in the configuration. The `OpenAI Completions` and `OpenAI Embeddings` models are both models which you can register to make use of OpenAI's completions and embeddings APIs. +This is where the other [supported models](./supported-models) come in. You can create a different model by specifying the model name in the configuration. The `OpenAI Completions` and `OpenAI Embeddings` models are both models which you can create to make use of OpenAI's completions and embeddings APIs. -You need to provide more information to the `aidb.register_model` function when registering a model like these. Completions has a number of options, including selecting which model it will use on OpenAI. Both Completions and Embeddings requires API credentials. Here is an example of how to register the OpenAI Completions model: +You need to provide more information to the `aidb.create_model` function when creating a model like this. Completions has a number of options, including selecting which model it will use on OpenAI. Both Completions and Embeddings require API credentials. Here is an example of how to create the OpenAI Completions model: ```sql -SELECT aidb.register_model( +SELECT aidb.create_model( 'my_openai_model', 'openai_completions', '{"model": "gpt-4o"}'::JSONB, @@ -68,10 +88,10 @@ SELECT aidb.register_model( You should replace the `api_key` value with your own OpenAI API key. Now you can use the `my_openai_model` model in your Pipelines functions and, in this example, leverage the GPT-4o model from OpenAI. -You can also register the OpenAI Embeddings model in a similar way. +You can also create the OpenAI Embeddings model in a similar way. ```sql -SELECT aidb.register_model( +SELECT aidb.create_model( 'my_openai_embeddings', 'openai_embeddings', '{"model": "text-embedding-3-large"}'::JSONB, @@ -79,11 +99,11 @@ SELECT aidb.register_model( }; ``` -This will register the `text-embedding-3-large` model with the name `my_openai_embeddings`. You can now use this model in your Pipelines functions to generate embeddings for text data. +This will create the `text-embedding-3-large` model with the name `my_openai_embeddings`. You can now use this model in your Pipelines functions to generate embeddings for text data. ## Using models with OpenAI compatible APIs -These OpenAI models work with any OpenAI compatible API. This allows you to connect and use an even wider range of models, just by passing the appropriate API endpoint to the `url` option in the `aidb.register_model` function's options. +These OpenAI models work with any OpenAI compatible API. This allows you to connect and use an even wider range of models, just by passing the appropriate API endpoint to the `url` option in the `aidb.create_model` function's options. For more information about the OpenAI models, see the [OpenAI Completions](./supported-models/openai-completions) and [OpenAI Embeddings](./supported-models/openai-embeddings) pages. diff --git a/advocacy_docs/edb-postgres-ai/ai-accelerator/pgfs/functions.mdx b/advocacy_docs/edb-postgres-ai/ai-accelerator/pgfs/functions.mdx index 6c0c636bc5c..4d00c4ad0fb 100644 --- a/advocacy_docs/edb-postgres-ai/ai-accelerator/pgfs/functions.mdx +++ b/advocacy_docs/edb-postgres-ai/ai-accelerator/pgfs/functions.mdx @@ -13,19 +13,22 @@ The PGFS extension provides a set of functions to create and manage storage loca We start with creating a storage location. A storage location is a reference to a location in an external file system. You can create a storage location with the `pgfs.create_storage_location` function. ```sql -select pgfs.create_storage_location('my_storage', 's3://my_bucket'); +select pgfs.create_storage_location('my_storage', 's3://my_bucket','','{}'::JSONB,'{}'::JSONB); ``` The create_strorage_location function takes a name for the storage location and then a URL for the location. The URL should be prefixed with `s3:` for an S3-compatible bucket or `file:` for a local file system. -The function also takes an optional `msl_id` parameter, which is not used. +The function also takes an optional `msl_id` parameter, which is not used. The function also takes an `options` parameter and credentials parameter. They are not optional and should be passed as empty JSON objects if not used. ### Creating a storage location with options and credentials -There are two other optional parameters, `options` and `credentials`. +Using the `options` and `credentials` parameters allows the passing of a range of other settings. + The `options` parameter is a JSON object that can be used to pass additional options to the storage location. The `credentials` parameter is a JSON object that can be used to pass credentials to the storage location. +The difference between `option` and `credentials` is that while options remains visible to users querying the extension, credentials are hidden to all users except superusers and the user that creates the storage location. + For example, you can create a storage location with options and credentials like this: ```sql @@ -62,12 +65,12 @@ You can also update a storage location with the `pgfs.update_storage_location` f select pgfs.update_storage_location('my_storage', 's3://my_bucket', null, '{"region": "eu-west"}' ``` -### Dropping a storage location +### Deleting a storage location -You can drop a storage location with the `pgfs.drop_storage_location` function. +You can delete a storage location with the `pgfs.delete_storage_location` function. ```sql -select pgfs.drop_storage_location('my_storage'); +select pgfs.delete_storage_location('my_storage'); ``` This will remove the storage location with the name `my_storage` from the database. diff --git a/advocacy_docs/edb-postgres-ai/ai-accelerator/pipelines-overview.mdx b/advocacy_docs/edb-postgres-ai/ai-accelerator/pipelines-overview.mdx index 91916ed7ca6..b9d70341aeb 100644 --- a/advocacy_docs/edb-postgres-ai/ai-accelerator/pipelines-overview.mdx +++ b/advocacy_docs/edb-postgres-ai/ai-accelerator/pipelines-overview.mdx @@ -25,7 +25,7 @@ Pipelines delivers its functionality through the Pipelines aidb extension, embed Pipelines' aidb extension introduces the concept of a “retriever” that you can create for a given type and location of AI data. Currently, Pipelines supports unstructured plain text documents as well as a set of image formats. This data can either reside in regular columns of a Postgres table or it can reside in an S3-compatible object storage bucket. -A retriever encapsulates all processing that is needed to make the AI data in the provided source location searchable and retrievable through similarity. The application just needs to create a retriever via the `aidb.register_retriever_for_table()` function for Postgres tables or `aidb.register_retriever_for_volume` for externally stored data on S3 or local filesystems. +A retriever encapsulates all processing that is needed to make the AI data in the provided source location searchable and retrievable through similarity. The application just needs to create a retriever via the `aidb.create_retriever_for_table()` function for Postgres tables or `aidb.create_retriever_for_volume` for externally stored data on S3 or local filesystems. ### Auto embedding diff --git a/advocacy_docs/edb-postgres-ai/ai-accelerator/reference/index.mdx b/advocacy_docs/edb-postgres-ai/ai-accelerator/reference/index.mdx index b2aef087a57..36b66ab7bd4 100644 --- a/advocacy_docs/edb-postgres-ai/ai-accelerator/reference/index.mdx +++ b/advocacy_docs/edb-postgres-ai/ai-accelerator/reference/index.mdx @@ -15,22 +15,22 @@ navigation: * [aidb.model_providers](models#aidbmodel_providers) -### Functions -* [aidb.register_model](models#aidbregister_model) -* [aidb.list_registered_models](models#aidblist_registered_models) -* [aidb.get_registered_model](models#aidbget_registered_model) -* [aidb.delete_registered_model](models#aidbdelete_registered_model) +### Model Functions +* [aidb.create_model](models#aidbcreate_model) +* [aidb.list_models](models#aidblist_models) +* [aidb.get_model](models#aidbget_model) +* [aidb.delete_model](models#aidbdelete_model) ## Retrievers -### Tables +### Retriever Tables * [aidb.retrievers](retrievers#aidbretrievers) -### Functions +### Retriever Functions -* [aidb.register_retriever_for_table](retrievers#aidbregister_retriever_for_table) -* [aidb.register_retriever_for_volume](retrievers#aidbregister_retriever_for_volume) +* [aidb.create_retriever_for_table](retrievers#aidbcreate_retriever_for_table) +* [aidb.create_retriever_for_volume](retrievers#aidbcreate_retriever_for_volume) * [aidb.enable_auto_embedding_for_table](retrievers#aidbenable_auto_embedding_for_table) * [aidb.disable_auto_embedding_for_table](retrievers#aidbdisable_auto_embedding_for_table) * [aidb.bulk_embedding](retrievers#aidbbulk_embedding) @@ -38,11 +38,11 @@ navigation: * [aidb.retrieve_text](retrievers#aidbretrieve_text) * [aidb.delete_retriever](retrievers#aidbdelete_retriever) * [aidb.create_volume](retrievers#aidbcreate_volume) -* [aidb.drop_volume](retrievers#aidbdrop_volume) +* [aidb.delete_volume](retrievers#aidbdelete_volume) ## PGFS -### Functions +### PGFS Functions * [pgfs.create_foreign_table](pgfs#pgfscreate_foreign_table) * [pgfs.create_storage_location](pgfs#pgfscreate_storage_location) @@ -53,4 +53,3 @@ navigation: * [pgfs.update_storage_location](pgfs#pgfsupdate_storage_location) * [pgfs.set_default_storage_location](pgfs#pgfsset_default_storage_location) * [pgfs.get_default_storage_location](pgfs#pgfsget_default_storage_location) - diff --git a/advocacy_docs/edb-postgres-ai/ai-accelerator/reference/models.mdx b/advocacy_docs/edb-postgres-ai/ai-accelerator/reference/models.mdx index b3211983ab6..c830f065e85 100644 --- a/advocacy_docs/edb-postgres-ai/ai-accelerator/reference/models.mdx +++ b/advocacy_docs/edb-postgres-ai/ai-accelerator/reference/models.mdx @@ -21,9 +21,9 @@ The `aidb.model_providers` table stores information about the model providers th ## Functions -### `aidb.register_model` +### `aidb.create_model` -Registers a a new model in the system by saving its name, provider and optional configuration. +Creates a new model in the system by saving its name, provider and optional configuration. #### Parameters @@ -38,7 +38,7 @@ Registers a a new model in the system by saving its name, provider and optional #### Example ```sql -SELECT aidb.register_model( +SELECT aidb.create_model( name => 'my_t5'::text, provider => 't5_local'::character varying, config => '{"param1": "value1", "param2": "value2"}'::jsonb, @@ -49,12 +49,12 @@ SELECT aidb.register_model( or equivalently, using default values: ```sql -SELECT aidb.register_model('my_t5', 't5_local'); +SELECT aidb.create_model('my_t5', 't5_local'); ``` -### `aidb.list_registered_models` +### `aidb.list_models` -Returns a list of all registered models and their configured options. +Returns a list of all models in the registry and their configured options. #### Parameters @@ -71,7 +71,7 @@ None #### Example ```sql -SELECT * FROM aidb.list_registered_models(); +SELECT * FROM aidb.list_models(); __OUTPUT__ name | provider | options -------+------------+--------------- @@ -80,9 +80,9 @@ __OUTPUT__ t5 | t5_local | {"config={}"} ``` -### `aidb.get_registered_model` +### `aidb.get_model` -Returns the configuration for a registered model. +Returns the configuration for a model in the registry. #### Parameters @@ -101,7 +101,7 @@ Returns the configuration for a registered model. #### Example ```sql -SELECT * FROM aidb.get_registered_model('t5'); +SELECT * FROM aidb.get_model('t5'); __OUTPUT__ name | provider | options ------+----------+--------------- @@ -109,9 +109,9 @@ __OUTPUT__ (1 row) ``` -### `aidb.delete_registered_model` +### `aidb.delete_model` -Deletes a registered model. +Deletes a model from the registry. #### Parameters @@ -122,9 +122,9 @@ Deletes a registered model. #### Example ```sql -SELECT aidb.delete_registered_model('t5'); +SELECT aidb.delete_model('t5'); __OUTPUT__ - delete_registered_model + delete_model --------------------------------- (t5,t5_local,"{""config={}""}") (1 row) @@ -134,11 +134,11 @@ __OUTPUT__ | Column | Type | Description | |---------------------------|-------|----------------------------------------------------------| -| `delete_registered_model` | jsonb | The name, provider and options of the now deleted model. | +| `delete_model` | jsonb | The name, provider and options of the now deleted model. | ### `aidb.encode_text` -Encodes text using a registered model, generating a vector representation of a given text input. +Encodes text using a model, generating a vector representation of a given text input. #### Parameters @@ -149,7 +149,7 @@ Encodes text using a registered model, generating a vector representation of a g ### `aidb.encode_text_batch` -Encodes a batch of text using a registered model, generating a vector representation of a given text inputs. +Encodes a batch of text using a model, generating a vector representation of a given text inputs. #### Parameters @@ -161,7 +161,7 @@ Encodes a batch of text using a registered model, generating a vector representa ### `aidb.decode_text` -Decodes text using a registered model, generating a vector representation of a given text input. +Decodes text using a model, generating a vector representation of a given text input. #### Parameters @@ -178,7 +178,7 @@ Decodes text using a registered model, generating a vector representation of a g ### `aidb.decode_text_batch` -Decodes a batch of text using a registered model, generating a representation of a given text input. +Decodes a batch of text using a model, generating a representation of a given text input. #### Parameters @@ -195,7 +195,7 @@ Decodes a batch of text using a registered model, generating a representation of ### `aidb.encode_image` -Encodes an image using a registered model, generating a vector representation of a given image input. +Encodes an image using a model, generating a vector representation of a given image input. #### Parameters diff --git a/advocacy_docs/edb-postgres-ai/ai-accelerator/reference/pgfs.mdx b/advocacy_docs/edb-postgres-ai/ai-accelerator/reference/pgfs.mdx index 35bbc009b04..7c37cad4591 100644 --- a/advocacy_docs/edb-postgres-ai/ai-accelerator/reference/pgfs.mdx +++ b/advocacy_docs/edb-postgres-ai/ai-accelerator/reference/pgfs.mdx @@ -55,7 +55,7 @@ Creates a storage location in the database. #### Example ```sql -SELECT pgfs.create_storage_location('my_storage', 's3://my_bucket', '123e4567-e89b-12d3-a456-426614174000', '{}', '{}'); +SELECT pgfs.create_storage_location('my_storage', 's3://my_bucket', '123e4567-e89b-12d3-a456-426614174000', '{}'::JSONB, '{}'::JSONB); ``` ### `pgfs.create_storage_location_with_foreign_table` @@ -64,13 +64,13 @@ Creates a storage location in the database and associates it with a foreign tabl #### Parameters -| Parameter | Type | Default | Description | -|-------------------------|------|---------|-------------| -| `storage_location_name` | text | | | -| `url` | text | | | -| `msl_id` | uuid | | | -| `options` | json | | | -| `credentials` | json | | | +| Parameter | Type | Default | Description | +|-------------------------|------|---------|---------------------------------------------------------| +| `storage_location_name` | text | | Name for storage location | +| `url` | text | | URL for this storage location (prefix `s3:` or `file:`) | +| `msl_id` | uuid | | Unused | +| `options` | json | | Options for the storage location | +| `credentials` | json | | Credentials for the storage location | #### Example @@ -84,9 +84,9 @@ Deletes a storage location from the database. #### Parameters -| Parameter | Type | Default | Description | -|-------------------------|------|---------|-------------| -| `storage_location_name` | text | | | +| Parameter | Type | Default | Description | +|-------------------------|------|---------|---------------------------------------| +| `storage_location_name` | text | | Name of storage location | #### Example @@ -116,9 +116,9 @@ Returns information about a storage location. #### Parameters -| Parameter | Type | Default | Description | -|-------------------------|------|---------|-------------| -| `storage_location_name` | text | | | +| Parameter | Type | Default | Description | +|-------------------------|------|---------|---------------------------------------| +| `storage_location_name` | text | | Name of storage location | #### Returns diff --git a/advocacy_docs/edb-postgres-ai/ai-accelerator/reference/retrievers.mdx b/advocacy_docs/edb-postgres-ai/ai-accelerator/reference/retrievers.mdx index e40f047a628..6b7e456ca4e 100644 --- a/advocacy_docs/edb-postgres-ai/ai-accelerator/reference/retrievers.mdx +++ b/advocacy_docs/edb-postgres-ai/ai-accelerator/reference/retrievers.mdx @@ -13,23 +13,23 @@ This section provides reference documentation for Pipelines Retrievers. It inclu The `aidb.retrievers` view shows information about the retrievers that have been created in the database. -| Column | Type | Description | -|-------------------------------|------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| id | integer | | -| name | text | Name of the retriever. | -| vector_table_name | text | Name of the table where the embeddings are stored. Gets newly created if it doesn’t exist, managed by aidb. | -| vector_table_key_column | text | The column that be used to store the key that references the key in source data when computing embeddings. Recommend to use the default and let aidb manage this table. | -| vector_table_vector_column | text | The column to store embeddings in. Recommend to use the default and let aidb manage this table. | -| model_name | text | Name of the registered model to use for embedding computation and retrievals. | -| topk | integer | How many results should be returned during a retrieve by default. Similar to LIMIT in SQL. | -| distance_operator | [aidb.DistanceOperator](#aidbdistanceoperator) | During retrieval, what vector operation should be used to compare the vectors. | -| options | jsonb | Currently unused. | -| source_type | text | Type of source data the retriever is working with. Can be either 'Table' or 'Volume'. | -| source_table_name | regclass | name of the table that has the source data we compute embeddings for, and that we retrieve from. Only applicable to retrievers configured with aidb.register_retriever_for_volume(). | -| source_table_data_column | text | column name in the source table that we compute embeddings for. This is also the column that will be returned in retrieve operations. | -| source_table_data_column_type | [aidb.RetrieverSourceDataFormat](#aidbretrieversourcedataformat) | Type of data the retriever working with. Uses type [`aidb.RetrieverSourceDataFormat`](#aidbretrieversourcedataformat). Only relevant for table based retrievers. In the case of a volume based retriever, the format/type information is discovered from the volume. | -| source_table_key_column | text | column to use as key for storing the embedding in the vector table. This provides a reference from the embedding to the source data | -| source_volume_name | text | Name of the volume to use as a data source. Only applicable to retrievers configured with aidb.register_retriever_for_volume(). | +| Column | Type | Description | +|-------------------------------|------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| id | integer | | +| name | text | Name of the retriever. | +| vector_table_name | text | Name of the table where the embeddings are stored. Gets newly created if it doesn’t exist, managed by aidb. | +| vector_table_key_column | text | The column that be used to store the key that references the key in source data when computing embeddings. Recommend to use the default and let aidb manage this table. | +| vector_table_vector_column | text | The column to store embeddings in. Recommend to use the default and let aidb manage this table. | +| model_name | text | Name of the model to use for embedding computation and retrievals. | +| topk | integer | How many results should be returned during a retrieve by default. Similar to LIMIT in SQL. | +| distance_operator | [aidb.DistanceOperator](#aidbdistanceoperator) | During retrieval, what vector operation should be used to compare the vectors. | +| options | jsonb | Currently unused. | +| source_type | text | Type of source data the retriever is working with. Can be either 'Table' or 'Volume'. | +| source_table_name | regclass | name of the table that has the source data we compute embeddings for, and that we retrieve from. Only applicable to retrievers configured with aidb.create_retriever_for_table(). | +| source_table_data_column | text | column name in the source table that we compute embeddings for. This is also the column that will be returned in retrieve operations. | +| source_table_data_column_type | [aidb.RetrieverSourceDataFormat](#aidbretrieversourcedataformat) | Type of data the retriever working with. Uses type [`aidb.RetrieverSourceDataFormat`](#aidbretrieversourcedataformat). Only relevant for table based retrievers. In the case of a volume based retriever, the format/type information is discovered from the volume. | +| source_table_key_column | text | column to use as key for storing the embedding in the vector table. This provides a reference from the embedding to the source data | +| source_volume_name | text | Name of the volume to use as a data source. Only applicable to retrievers configured with aidb.create_retriever_for_volume(). | ## Types @@ -81,64 +81,64 @@ CREATE TYPE RetrieverSourceDataFormat AS ENUM ( ## Functions -### `aidb.register_retriever_for_table` +### `aidb.create_retriever_for_table` -Registers a retriever for a given table. +Creates a retriever for a given table. #### Parameters -| Parameter | Type | Default | Description | -|---------------------------------|------------------------------------------------------------------|--------------|----------------------------------------------------| -| p_name | TEXT | Required | Name of the retriever | -| p_model_name | TEXT | Required | Name of the registered model to use | -| p_source_table_name | regclass | Required | Name of the table to use as source | -| p_source_table_data_column | TEXT | Required | Column name in source table to use | -| p_source_table_data_column_type | [aidb.RetrieverSourceDataFormat](#aidbretrieversourcedataformat) | Required | Type of data in that column ("Text"."Image","PDF") | -| p_source_table_key_column | TEXT | 'id' | Column to use as key to reference the rows | -| p_vector_table_name | TEXT | NULL | | -| p_vector_table_vector_column | TEXT | 'embeddings' | | -| p_vector_table_key_column | TEXT | 'id' | | -| p_topk | INTEGER | 1 | | -| p_distance_operator | [aidb.distanceoperator](#aidbdistanceoperator) | 'L2' | | -| p_options | JSONB | '{}'::JSONB | Options | +| Parameter | Type | Default | Description | +|--------------------|------------------------------------------------------------------|--------------|----------------------------------------------------| +| name | TEXT | Required | Name of the retriever | +| model_name | TEXT | Required | Name of the model to use | +| source_table | regclass | Required | Name of the table to use as source | +| source_data_column | TEXT | Required | Column name in source table to use | +| source_data_type | [aidb.RetrieverSourceDataFormat](#aidbretrieversourcedataformat) | Required | Type of data in that column ("Text"."Image","PDF") | +| source_key_column | TEXT | 'id' | Column to use as key to reference the rows | +| vector_table | TEXT | NULL | | +| vector_data_column | TEXT | 'embeddings' | | +| vector_key_column | TEXT | 'id' | | +| topk | INTEGER | 1 | | +| distance_operator | [aidb.distanceoperator](#aidbdistanceoperator) | 'L2' | | +| options | JSONB | '{}'::JSONB | Options | #### Example ```sql -SELECT aidb.register_retriever_for_table( - p_name => 'test_retriever', - p_model_name => 'simple_model', - p_source_table_name => 'test_source_table', - p_source_table_data_column => 'content', - p_source_table_data_column_type => 'Text', +SELECT aidb.create_retriever_for_table( + name => 'test_retriever', + model_name => 'simple_model', + source_table => 'test_source_table', + source_data_column => 'content', + source_data_type => 'Text', ); ``` -### `aidb.register_retriever_for_volume` +### `aidb.create_retriever_for_volume` -Registers a retriever for a given PGFS volume. +Creates a retriever for a given PGFS volume. #### Parameters -| Parameter | Type | Default | Description | -|------------------------------|-----------------------|--------------|------------------------------| -| p_name | TEXT | Required | Name of the retriever. | -| p_model_name | TEXT | Required | Name of the model. | -| p_source_volume_name | TEXT | Required | Name of the volume. | -| p_vector_table_name | TEXT | NULL | Name of the vector table. | -| p_vector_table_vector_column | TEXT | 'embeddings' | Name of the vector column. | -| p_vector_table_key_column | TEXT | 'id' | Name of the key column. | -| p_topk | INTEGER | 1 | Number of results to return. | -| p_distance_operator | aidb.distanceoperator | 'L2' | Distance operator. | -| p_options | JSONB | '{}'::JSONB | Options. | +| Parameter | Type | Default | Description | +|--------------------|-----------------------|--------------|------------------------------| +| name | TEXT | Required | Name of the retriever. | +| model_name | TEXT | Required | Name of the model. | +| source_volume_name | TEXT | Required | Name of the volume. | +| vector_table | TEXT | NULL | Name of the vector table. | +| vector_data_column | TEXT | 'embeddings' | Name of the vector column. | +| vector_key_column | TEXT | 'id' | Name of the key column. | +| topk | INTEGER | 1 | Number of results to return. | +| distance_operator | aidb.distanceoperator | 'L2' | Distance operator. | +| options | JSONB | '{}'::JSONB | Options. | #### Example ```sql -SELECT aidb.register_retriever_for_volume( - p_name => 'demo_vol_retriever', - p_model_name => 'simple_model', - p_source_volume_name => 'demo_bucket_vol' +SELECT aidb.create_retriever_for_volume( + name => 'demo_vol_retriever', + model_name => 'simple_model', + source_volume_name => 'demo_bucket_vol' ); ``` @@ -148,9 +148,9 @@ Enables automatic embedding generation for a given table. #### Parameters -| Parameter | Type | Default | Description | -|---------------------------------|--------------------------------|--------------|-----------------------------------------------| -| p_name | TEXT | | Name of registered table which should have auto-embedding enabled.| +| Parameter | Type | Default | Description | +|---------------------------------|--------------------------------|--------------|------------------------------------------------------------| +| retriever_name | TEXT | | Name of retriever which should have auto-embedding enabled.| #### Example @@ -164,9 +164,9 @@ Enables automatic embedding generation for a given table. #### Parameters -| Parameter | Type | Default | Description | -|---------------------------------|--------------------------------|--------------|-----------------------------------------------| -| p_name | TEXT | | Name of registered table which should have auto_embedding disabled.| +| Parameter | Type | Default | Description | +|---------------------------------|--------------------------------|--------------|-------------------------------------------------------------| +| retriever_name | TEXT | | Name of retriever which should have auto_embedding disabled.| #### Example @@ -180,9 +180,9 @@ Generates embeddings for all data in a given table if there is existing data in #### Parameters -| Parameter | Type | Default | Description | -|---------------------------------|--------------------------------|--------------|-----------------------------------------------| -| retriever_name | TEXT | | Name of retriever which which should have embeddings generated.| +| Parameter | Type | Default | Description | +|---------------------------------|--------------------------------|--------------|-----------------------------------------------------------------| +| retriever_name | TEXT | | Name of retriever which which should have embeddings generated.| #### Example @@ -202,11 +202,11 @@ Retrieves a key from matching embeddings without looking up the source data. #### Parameters -| Parameter | Type | Default | Description | -|---------------------------------|--------------------------------|--------------|-----------------------------------------------| -| retriever_name | TEXT | | Name of retriever which should be used for retrieval.| -| query_string | TEXT | | Query string to be used for retrieval.| -| number_of_results | INTEGER | 0 | Number of results to be returned.| +| Parameter | Type | Default | Description | +|-------------------|---------|---------|-------------------------------------------------------| +| retriever_name | TEXT | | Name of retriever which should be used for retrieval. | +| query_string | TEXT | | Query string to be used for retrieval. | +| number_of_results | INTEGER | 0 | Number of results to be returned. | #### Example @@ -226,11 +226,11 @@ Retrieves the source text data from matching embeddings by joining the embedding #### Parameters -| Parameter | Type | Default | Description | -|---------------------------------|--------------------------------|--------------|-----------------------------------------------| -| retriever_name | TEXT | | Name of retriever which should be used for retrieval.| -| query_string | TEXT | | Query string to be used for retrieval.| -| number_of_results | INTEGER | 0 | Number of results to be returned.| +| Parameter | Type | Default | Description | +|-------------------|---------|---------|-------------------------------------------------------| +| retriever_name | TEXT | | Name of retriever which should be used for retrieval. | +| query_string | TEXT | | Query string to be used for retrieval. | +| number_of_results | INTEGER | 0 | Number of results to be returned. | #### Returns @@ -306,22 +306,22 @@ Lists all the volumes that have been created in the database. select * from aidb.list_volumes(); ``` -### `aidb.drop_volume` +### `aidb.delete_volume` -Drops a volume from the database. +Deletes a volume from the database. #### Parameters | Parameter | Type | Default | Description | |---------------------------------|--------------------------------|--------------|-----------------------------------------------| -| volume_name | TEXT | | Name of the volume to be dropped.| +| volume_name | TEXT | | Name of the volume to be deleted. | #### Example ```sql -select aidb.drop_volume('demo_bucket_vol'); +select aidb.delete_volume('demo_bucket_vol'); __OUTPUT__ - drop_volume + delete_volume ------------- (1 row) ``` diff --git a/advocacy_docs/edb-postgres-ai/ai-accelerator/rel_notes/ai-accelerator_2.0.0_rel_notes.mdx b/advocacy_docs/edb-postgres-ai/ai-accelerator/rel_notes/ai-accelerator_2.0.0_rel_notes.mdx new file mode 100644 index 00000000000..5e960ef91de --- /dev/null +++ b/advocacy_docs/edb-postgres-ai/ai-accelerator/rel_notes/ai-accelerator_2.0.0_rel_notes.mdx @@ -0,0 +1,22 @@ +--- +title: AI Accelerator - Pipelines 2.0.0 release notes +navTitle: Version 2.0.0 +--- + +Released: 13 January 2025 + +Updating the GA release of EDB Postgres AI - AI Accelerator - Pipelines, in response to feedback. + +## Highlights + +- Improved API naming in functions and arguments. + +## Enhancements + + + +
DescriptionAddresses
Renaming of functions/arguments

Management functions move to use create rather than register and delete rather than drop. +Function argument names lose the preceding `p_`` to be consistent.

+
+ + diff --git a/advocacy_docs/edb-postgres-ai/ai-accelerator/rel_notes/index.mdx b/advocacy_docs/edb-postgres-ai/ai-accelerator/rel_notes/index.mdx index 149fb8dc37a..e3daad74bea 100644 --- a/advocacy_docs/edb-postgres-ai/ai-accelerator/rel_notes/index.mdx +++ b/advocacy_docs/edb-postgres-ai/ai-accelerator/rel_notes/index.mdx @@ -1,16 +1,18 @@ --- -title: EDB Postgres AI - AI Accelerator 1.0.0 release notes +title: EDB Postgres AI - AI Accelerator release notes navTitle: Release notes -description: Release notes for EDB Postgres AI - AI Accelerator 1.0.0 +description: Release notes for EDB Postgres AI - AI Accelerator indexCards: none navigation: + - ai-accelerator_2.0.0_rel_notes - ai-accelerator_1.0.7_rel_notes --- -The EDB Postgres AI - AI Accelerator describes the latest version of AI Accelerator. The release notes provide information on what was new in each release. For the GA 1.0.0 release, differences from the technical preview are highlighted. +The EDB Postgres AI - AI Accelerator describes the latest version of AI Accelerator. The release notes provide information on what was new in each release. Differences from the technical preview are highlighted in the 1.0.7 release notes. | AI Accelerator version | Release Date | |---|---| +| [2.0.0](./ai-accelerator_2.0.0_rel_notes) | 13 Jan 2025 | | [1.0.7](./ai-accelerator_1.0.7_rel_notes) | 10 Dec 2024 | diff --git a/advocacy_docs/edb-postgres-ai/ai-accelerator/rel_notes/src/meta.yml b/advocacy_docs/edb-postgres-ai/ai-accelerator/rel_notes/src/meta.yml index c4d65dc0295..8a3eeae4882 100644 --- a/advocacy_docs/edb-postgres-ai/ai-accelerator/rel_notes/src/meta.yml +++ b/advocacy_docs/edb-postgres-ai/ai-accelerator/rel_notes/src/meta.yml @@ -1,9 +1,9 @@ product: EDB Postgres AI - AI Accelerator shortname: ai-accelerator -title: EDB Postgres AI - AI Accelerator 1.0.0 release notes -description: Release notes for EDB Postgres AI - AI Accelerator 1.0.0 +title: EDB Postgres AI - AI Accelerator release notes +description: Release notes for EDB Postgres AI - AI Accelerator intro: | - The EDB Postgres AI - AI Accelerator describes the latest version of AI Accelerator. The release notes provide information on what was new in each release. For the GA 1.0.0 release, differences from the technical preview are highlighted. + The EDB Postgres AI - AI Accelerator describes the latest version of AI Accelerator. The release notes provide information on what was new in each release. Differences from the technical preview are highlighted in the 1.0.7 release notes. columns: - 0: label: "AI Accelerator version" diff --git a/advocacy_docs/edb-postgres-ai/ai-accelerator/rel_notes/src/rel_notes_2.0.0.yml b/advocacy_docs/edb-postgres-ai/ai-accelerator/rel_notes/src/rel_notes_2.0.0.yml new file mode 100644 index 00000000000..964bec128ba --- /dev/null +++ b/advocacy_docs/edb-postgres-ai/ai-accelerator/rel_notes/src/rel_notes_2.0.0.yml @@ -0,0 +1,16 @@ +product: AI Accelerator - Pipelines +version: 2.0.0 +date: 13 January 2025 +intro: | + Updating the GA release of EDB Postgres AI - AI Accelerator - Pipelines, in response to feedback. +highlights: | + - Improved API naming in functions and arguments. +relnotes: +- relnote: Renaming of functions/arguments + details: | + Management functions move to use `create` rather than `register` and `delete` rather than `drop`. + Function argument names lose the preceding `p_`` to be consistent. + jira: AID-213 + addresses: "" + type: Enhancement + impact: High diff --git a/advocacy_docs/edb-postgres-ai/ai-accelerator/retrievers/example.mdx b/advocacy_docs/edb-postgres-ai/ai-accelerator/retrievers/example.mdx index 5c56ba23bb5..72d3ed40ef3 100644 --- a/advocacy_docs/edb-postgres-ai/ai-accelerator/retrievers/example.mdx +++ b/advocacy_docs/edb-postgres-ai/ai-accelerator/retrievers/example.mdx @@ -11,82 +11,74 @@ This is a full end-to-end example of using retrievers in EDB Postgres AI - AI Ac DROP EXTENSION aidb CASCADE; CREATE EXTENSION aidb CASCADE; -drop table if exists test_source_table cascade; -drop table if exists test_retriever_vector cascade; +drop table if exists test_source_table_ajz72eb cascade; +drop table if exists test_retriever_ajz72eb_vector cascade; -- Create source test table -CREATE TABLE test_source_table +CREATE TABLE test_source_table_ajz72eb ( id INT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, content TEXT NOT NULL, unrelated_column TEXT ); -INSERT INTO test_source_table +INSERT INTO test_source_table_ajz72eb VALUES (43941, 'Catwalk Women Brown Heels'), (55018, 'Lakme 3 in 1 Orchid Aqua Shine Lip Color'), (19337, 'United Colors of Benetton Men Stripes Black Jacket'); --- Register model -SELECT aidb.register_model('simple_model', 'bert_local'); - -SELECT aidb.register_retriever_for_table( - p_name => 'test_retriever', - p_model_name => 'simple_model', - p_source_table_name => 'test_source_table', - p_source_table_data_column => 'content', - p_source_table_data_column_type => 'Text', - p_source_table_key_column => 'id', -- Default - p_vector_table_name => 'test_source_table_vector', -- Defaults to `source_table_name + '_vector'` - p_vector_table_vector_column => 'embeddings', -- Default - p_vector_table_key_column => 'id', -- Default - p_topk => 1, -- Default - p_distance_operator => 'L2', -- Default - p_options => '{}'::JSONB -- Default +-- Create model +SELECT aidb.create_model('simple_model_ajz72eb', 'bert_local'); + +SELECT aidb.create_retriever_for_table( + name => 'test_retriever_ajz72eb', + model_name => 'simple_model_ajz72eb', + source_table => 'test_source_table_ajz72eb', + source_data_column => 'content', + source_data_type => 'Text' ); -- expect "Table" -SELECT aidb.get_retriever_data_source('test_retriever'); +SELECT aidb.get_retriever_data_source('test_retriever_ajz72eb'); SELECT * FROM aidb.retrievers; -SELECT aidb.bulk_embedding('test_retriever'); +SELECT aidb.bulk_embedding('test_retriever_ajz72eb'); -- Perform retrieval similarity search for the closest `key` -SELECT * FROM aidb.retrieve_key('test_retriever', 'orchid'); -SELECT * FROM aidb.retrieve_key('test_retriever', 'orchid', 2); -- Limit to top 2 results +SELECT * FROM aidb.retrieve_key('test_retriever_ajz72eb', 'orchid'); +SELECT * FROM aidb.retrieve_key('test_retriever_ajz72eb', 'orchid', 2); -- Limit to top 2 results -SELECT * FROM aidb.retrieve_text('test_retriever', 'orchid'); -SELECT * FROM aidb.retrieve_text('test_retriever', 'orchid', 2); -- Limit to top 2 results +SELECT * FROM aidb.retrieve_text('test_retriever_ajz72eb', 'orchid'); +SELECT * FROM aidb.retrieve_text('test_retriever_ajz72eb', 'orchid', 2); -- Limit to top 2 results -- enable the auto embedding -SELECT aidb.enable_auto_embedding_for_table('test_retriever'); +SELECT aidb.enable_auto_embedding_for_table('test_retriever_ajz72eb'); -- add additional data to test auto-embedding -INSERT INTO test_source_table +INSERT INTO test_source_table_ajz72eb VALUES (11211, 'Bicycle'), (11311, 'What is this?'), (11411, 'Elephants'); -- check embeddings -SELECT id FROM test_retriever_vector; +SELECT id FROM test_retriever_ajz72eb_vector; -- delete one of the source rows -DELETE FROM test_source_table WHERE id = 11211; +DELETE FROM test_source_table_ajz72eb WHERE id = 11211; -- check embeddings -SELECT id FROM test_retriever_vector; +SELECT id FROM test_retriever_ajz72eb_vector; -- enable the auto embedding -SELECT aidb.disable_auto_embedding_for_table('test_retriever'); -INSERT INTO test_source_table VALUES (212121, 'new value'); - +SELECT aidb.disable_auto_embedding_for_table('test_retriever_ajz72eb'); +INSERT INTO test_source_table_ajz72eb VALUES (212121, 'new value'); -select aidb.delete_retriever('test_retriever'); +select aidb.delete_retriever('test_retriever_ajz72eb'); ``` diff --git a/advocacy_docs/edb-postgres-ai/ai-accelerator/retrievers/usage.mdx b/advocacy_docs/edb-postgres-ai/ai-accelerator/retrievers/usage.mdx index cd72fc1e5ae..48d12f5112a 100644 --- a/advocacy_docs/edb-postgres-ai/ai-accelerator/retrievers/usage.mdx +++ b/advocacy_docs/edb-postgres-ai/ai-accelerator/retrievers/usage.mdx @@ -7,39 +7,39 @@ description: "Usage of retrievers in AI Accelerator Pipelines." ## Creating a new retriever configuration -There are two types of retrievers: table and volume. Given the different nature of the data sources, and the options required for each we have different functions to register them. +There are two types of retrievers: table and volume. Given the different nature of the data sources, and the options required for each we have different functions to create them. ## Retriever for a table data source -The [aidb.register_retriever_for_table](../reference/retrievers#aidbregister_retriever_for_table) function is used to create a retriever for a table data source. This is the function signature, you can see many of those are optional and have defaults. +The [aidb.create_retriever_for_table](../reference/retrievers#aidbcreate_retriever_for_table) function is used to create a retriever for a table data source. This is the function signature, you can see many of those are optional and have defaults. ``` -register_retriever_for_table( +create_retriever_for_table( ------------------------------------------------------------------------------- - p_name TEXT, - p_model_name, TEXT, - p_source_table_name regclass, - p_source_table_data_column TEXT, - p_source_table_data_column_type aidb.RetrieverSourceDataFormat, - p_source_table_key_column TEXT DEFAULT 'id', - p_vector_table_name TEXT DEFAULT NULL, - p_vector_table_vector_column TEXT DEFAULT 'embeddings', - p_vector_table_key_column TEXT DEFAULT 'id', - p_topk INTEGER DEFAULT 1, - p_distance_operator aidb.distanceoperator DEFAULT 'L2', - p_options JSONB DEFAULT '{}'::JSONB + name TEXT, + model_name, TEXT, + source_table_name regclass, + source_data_column TEXT, + source_data_type aidb.RetrieverSourceDataFormat, + source_key_column TEXT DEFAULT 'id', + vector_table TEXT DEFAULT NULL, + vector_data_column TEXT DEFAULT 'embeddings', + vector_key_column TEXT DEFAULT 'id', + topk INTEGER DEFAULT 1, + distance_operator aidb.distanceoperator DEFAULT 'L2', + options JSONB DEFAULT '{}'::JSONB ) ``` -### Example: Registering a retriever +### Example: Creating a retriever ``` sql -SELECT aidb.register_retriever_for_table( - p_name => 'test_retriever', - p_model_name => 'simple_model', - p_source_table_name => 'test_source_table', - p_source_table_data_column => 'content', - p_source_table_data_column_type => 'Text' +SELECT aidb.create_retriever_for_table( + name => 'test_retriever', + model_name => 'simple_model', + source_table_name => 'test_source_table', + source_data_column => 'content', + source_data_type => 'Text' ); ``` @@ -47,15 +47,15 @@ In this example, we use all the defaults. If you are only using Postgres tables, [skip to the next section](#creating-the-embeddings). -If you are using external data sources, you need to create a volume and register a retriever for it, which is explained in the next section. +If you are using external data sources, you need to create a volume and create a retriever for it, which is explained in the next section. ## Retriever for a volume data source ### Creating a new volume -Before we can register a retriever for a volume, we need to create a volume. The [aidb.create_volume](../reference/retrievers#aidbcreate_volume) function is used to create a volume. This is the function signature, you can see many of those are optional and have defaults. +Before we can create a retriever for a volume, we need to create a volume. The [aidb.create_volume](../reference/retrievers#aidbcreate_volume) function is used to create a volume. This is the function signature, you can see many of those are optional and have defaults. -``` +```text aidb.create_volume( ------------------------------------------------------------------------------- name TEXT, @@ -80,32 +80,32 @@ SELECT aidb.create_volume( The `server_name` comes from calling PGFS functions to create a storage location; [pgfs.create_storage_location](../reference/pgfs#pgfscreate_storage_location). The `path` is the path to the data in the storage location. -### Registering a retriever for a volume +### Creating a retriever for a volume -The [aidb.register_retriever_for_volume](../reference/retrievers#aidbregister_retriever_for_volume) function is used to create a retriever for a volume data source. This is the function signature, you can see many of those are optional and have defaults. +The [aidb.create_retriever_for_volume](../reference/retrievers#aidbcreate_retriever_for_volume) function is used to create a retriever for a volume data source. This is the function signature, you can see many of those are optional and have defaults. ``` -aidb.register_retriever_for_volume( +aidb.create_retriever_for_volume( ------------------------------------------------------------------------------- - p_name TEXT, - p_source_volume_name TEXT, - p_vector_table_name TEXT DEFAULT NULL, - p_vector_table_vector_column TEXT DEFAULT 'embeddings', - p_vector_table_key_column TEXT DEFAULT 'id', - p_model_name, TEXT, - p_topk INTEGER DEFAULT 1, - p_distance_operator aidb.distanceoperator DEFAULT 'L2', - p_options JSONB DEFAULT '{}'::JSONB + name TEXT, + model_name, TEXT, + source_volume_name TEXT, + vector_table TEXT DEFAULT NULL, + vector_data_column TEXT DEFAULT 'embeddings', + vector_key_column TEXT DEFAULT 'id', + topk INTEGER DEFAULT 1, + distance_operator aidb.distanceoperator DEFAULT 'L2', + options JSONB DEFAULT '{}'::JSONB ) ``` -### Example: Registering a retriever for a volume +### Example: Creating a retriever for a volume ``` sql -SELECT aidb.register_retriever_for_volume( - p_name => 'test_retriever_volume', - p_source_volume_name => 'test_volume', - p_model_name => 'simple_model' +SELECT aidb.create_retriever_for_volume( + name => 'test_retriever_volume', + model_name => 'simple_model', + source_volume_name => 'test_volume' ); ``` diff --git a/advocacy_docs/edb-postgres-ai/index.mdx b/advocacy_docs/edb-postgres-ai/index.mdx index fc3c02e7f4d..482609aebc1 100644 --- a/advocacy_docs/edb-postgres-ai/index.mdx +++ b/advocacy_docs/edb-postgres-ai/index.mdx @@ -12,7 +12,7 @@ navigation: - cloud-service - databases - analytics -- ai-ml +- ai-accelerator - migration-etl - downloads-and-repositories - developer-guides diff --git a/gatsby-node.js b/gatsby-node.js index d93cbae8452..b98f6ae9457 100644 --- a/gatsby-node.js +++ b/gatsby-node.js @@ -576,6 +576,7 @@ exports.createSchemaCustomization = ({ actions }) => { none simple full + extra } enum EditTargets { diff --git a/product_docs/docs/pgd/5.6/rel_notes/index.mdx b/product_docs/docs/pgd/5.6/rel_notes/index.mdx index e9940e7e004..3b71917efbc 100644 --- a/product_docs/docs/pgd/5.6/rel_notes/index.mdx +++ b/product_docs/docs/pgd/5.6/rel_notes/index.mdx @@ -2,11 +2,13 @@ title: EDB Postgres Distributed 5.6+ release notes navTitle: Release notes description: Release notes for EDB Postgres Distributed 5.6 and later +indexCards: none navigation: - pgd_5.6.1_rel_notes - pgd_5.6.0_rel_notes --- + The EDB Postgres Distributed documentation describes the latest version of EDB Postgres Distributed 5, including minor releases and patches. The release notes provide information on what was new in each release. For new functionality introduced in a minor or patch release, the content also indicates the release that introduced the feature. @@ -14,4 +16,3 @@ The EDB Postgres Distributed documentation describes the latest version of EDB P |---|---|---|---|---| | 25 Nov 2024 | [5.6.1](./pgd_5.6.1_rel_notes) | 5.6.1 | 5.6.1 | 5.6.1 | | 15 Oct 2024 | [5.6.0](./pgd_5.6.0_rel_notes) | 5.6.0 | 5.6.0 | 5.6.0 | - diff --git a/product_docs/docs/tpa/23/rel_notes/index.mdx b/product_docs/docs/tpa/23/rel_notes/index.mdx index 2eddffa4239..0a80a8767c3 100644 --- a/product_docs/docs/tpa/23/rel_notes/index.mdx +++ b/product_docs/docs/tpa/23/rel_notes/index.mdx @@ -2,6 +2,7 @@ title: Trusted Postgres Architect release notes navTitle: Release notes description: Release notes for Trusted Postgres Architect and later +indexCards: none navigation: - tpa_23.35.0_rel_notes - tpa_23.34.1_rel_notes diff --git a/src/components/card-decks.js b/src/components/card-decks.js index d4da13984f2..2042055eaa3 100644 --- a/src/components/card-decks.js +++ b/src/components/card-decks.js @@ -43,6 +43,33 @@ const FullCard = ({ card }) => { ); }; +const ExtraCard = ({ card }) => { + return ( +
+
+

+ {card.navTitle || card.title} +

+ +

{card.description}

+ +
+ {card.items.map((child) => ( + + {child.navTitle || child.title} + {child.interactive && } + + ))} +
+
+
+ ); +}; + const SimpleCard = ({ card }) => (
@@ -78,7 +105,10 @@ const CardDecks = ({ cards, cardType = "simple", deckTitle = "" }) => { xl={cardType === "simple" && 4} className="d-flex" > - {cardType === "full" ? ( + {" "} + {cardType === "extra" ? ( + + ) : cardType === "full" ? ( ) : ( diff --git a/src/components/tiles.js b/src/components/tiles.js index 74563bfc7be..c002d26a766 100644 --- a/src/components/tiles.js +++ b/src/components/tiles.js @@ -5,6 +5,7 @@ export const TileModes = { None: "none", Simple: "simple", Full: "full", + Extra: "extra", }; const Tiles = ({ mode, node }) => { diff --git a/tools/automation/generators/relgen/relgen.js b/tools/automation/generators/relgen/relgen.js index 9116f3fabbc..b4a61d520a7 100755 --- a/tools/automation/generators/relgen/relgen.js +++ b/tools/automation/generators/relgen/relgen.js @@ -36,6 +36,11 @@ function converter(markdown) { return micromark(markdown); } +function error_and_exit(message) { + console.error(`Error: ${message}`); + process.exit(1); +} + function normalizeType(type) { switch (type) { case "Feature": @@ -168,16 +173,37 @@ function makeShortDate(date) { November: "Nov", December: "Dec", }; + + const valid_short_months = [ + "Jan", + "Feb", + "Mar", + "Apr", + "May", + "Jun", + "Jul", + "Aug", + "Sep", + "Oct", + "Nov", + "Dec", + ]; + let [day, month, year] = date.split(" "); + if (month in valid_short_months) { + return `${day} ${month} ${year}`; + } let shortmonth = months[month]; - return `${day} ${shortmonth} ${year}`; + if (shortmonth === undefined) { + throw new Error(`Invalid month: ${month}`); + } + return `${day.toString().padStart(2, "0")} ${shortmonth} ${year}`; } let relindexfilename = path.join(basepath, "index.mdx"); // Use this to write the file let err = writeFileSync(relindexfilename, ""); if (err) { - console.error(err); - process.exit(1); + error_and_exit(`Error writing ${relindexfilename}`); } console.log(`Writing ${relindexfilename}`); @@ -237,14 +263,19 @@ for (let [file, relnote] of relnotes) { line += ` [${relnote.version}](./${makeRelnotefilename(meta.shortname, relnote.version)}) |`; break; case "shortdate": - line += ` ${makeShortDate(relnote.date)} |`; + try { + let shortdate = makeShortDate(relnote.date); + line += ` ${shortdate} |`; + } catch (e) { + error_and_exit(`${file}: ${e}`); + } break; default: if (col.key.startsWith("$")) { let key = col.key.replace("$", ""); line += ` ${relnote.meta[key]} |`; } else { - console.err(`Unknown column key: ${col.key}`); + error_and_exit(`${file}: Unknown column key: ${col.key}`); } break; } @@ -270,7 +301,7 @@ if (meta.precursor !== undefined) { let key = col.key.replace("$", ""); line += ` ${prec.meta[key]} |`; } else { - console.err(`Unknown column key: ${col.key}`); + error_and_exit(`Unknown column key ${col.key} in meta.yml`); } break; } @@ -292,8 +323,7 @@ function prepareRelnote(meta, file, note) { let rlout = path.join(basepath, relnotefilename + ".mdx"); let err = writeFileSync(rlout, ""); if (err) { - console.error(err); - process.exit(1); + error_and_exit(`Error writing ${rlout}`); } types = types.sort((a, b) => {