diff --git a/admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-embeddings-edit.js b/admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-embeddings-edit.js new file mode 100644 index 000000000..bc6bec1ab --- /dev/null +++ b/admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-embeddings-edit.js @@ -0,0 +1,19 @@ +import DiscourseRoute from "discourse/routes/discourse"; + +export default class AdminPluginsShowDiscourseAiEmbeddingsEdit extends DiscourseRoute { + async model(params) { + const allEmbeddings = this.modelFor("adminPlugins.show.discourse-ai-embeddings"); + const id = parseInt(params.id, 10); + const record = allEmbeddings.findBy("id", id); + record.provider_params = record.provider_params || {}; + return record; + } + + setupController(controller, model) { + super.setupController(controller, model); + controller.set( + "allEmbeddings", + this.modelFor("adminPlugins.show.discourse-ai-embeddings") + ); + } +} diff --git a/admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-embeddings-new.js b/admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-embeddings-new.js new file mode 100644 index 000000000..ea8a92392 --- /dev/null +++ b/admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-embeddings-new.js @@ -0,0 +1,17 @@ +import DiscourseRoute from "discourse/routes/discourse"; + +export default class AdminPluginsShowDiscourseAiEmbeddingsNew extends DiscourseRoute { + async model() { + const record = this.store.createRecord("ai-embedding"); + record.provider_params = {}; + return record; + } + + setupController(controller, model) { + super.setupController(controller, model); + controller.set( + "allEmbeddings", + this.modelFor("adminPlugins.show.discourse-ai-embeddings") + ); + } +} diff --git a/admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-embeddings.js b/admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-embeddings.js new file mode 100644 index 000000000..475e6f880 --- /dev/null +++ b/admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-embeddings.js @@ -0,0 +1,7 @@ +import DiscourseRoute from "discourse/routes/discourse"; + +export default class DiscourseAiAiEmbeddingsRoute extends DiscourseRoute { + model() { + return this.store.findAll("ai-embedding"); + } +} diff --git a/admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-embeddings/edit.hbs b/admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-embeddings/edit.hbs new file mode 100644 index 000000000..8ec8776fa --- /dev/null +++ b/admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-embeddings/edit.hbs @@ -0,0 +1,4 @@ + \ No newline at end of file diff --git a/admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-embeddings/index.hbs b/admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-embeddings/index.hbs new file mode 100644 index 000000000..8226d03c4 --- /dev/null +++ b/admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-embeddings/index.hbs @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-embeddings/new.hbs b/admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-embeddings/new.hbs new file mode 100644 index 000000000..8ec8776fa --- /dev/null +++ b/admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-embeddings/new.hbs @@ -0,0 +1,4 @@ + \ No newline at end of file diff --git a/app/controllers/discourse_ai/admin/ai_embeddings_controller.rb b/app/controllers/discourse_ai/admin/ai_embeddings_controller.rb index 5738be9ea..1332fc393 100644 --- a/app/controllers/discourse_ai/admin/ai_embeddings_controller.rb +++ b/app/controllers/discourse_ai/admin/ai_embeddings_controller.rb @@ -18,6 +18,7 @@ def index meta: { provider_params: EmbeddingDefinition.provider_params, providers: EmbeddingDefinition.provider_names, + distance_functions: EmbeddingDefinition.distance_functions, tokenizers: EmbeddingDefinition.tokenizer_names.map { |tn| { id: tn, name: tn.split("::").last } diff --git a/app/models/embedding_definition.rb b/app/models/embedding_definition.rb index 233720bd4..3e3b3e810 100644 --- a/app/models/embedding_definition.rb +++ b/app/models/embedding_definition.rb @@ -2,14 +2,17 @@ class EmbeddingDefinition < ActiveRecord::Base CLOUDFLARE = "cloudflare" - DISCOURSE = "discourse" HUGGING_FACE = "hugging_face" OPEN_AI = "open_ai" - GEMINI = "gemini" + GOOGLE = "google" class << self def provider_names - [CLOUDFLARE, DISCOURSE, HUGGING_FACE, OPEN_AI, GEMINI] + [CLOUDFLARE, HUGGING_FACE, OPEN_AI, GOOGLE] + end + + def distance_functions + %w[<#> <=>] end def tokenizer_names @@ -24,7 +27,7 @@ def tokenizer_names end def provider_params - { discourse: { model_name: :text }, open_ai: { model_name: :text } } + { open_ai: { model_name: :text } } end end @@ -41,13 +44,11 @@ def inference_client case provider when CLOUDFLARE cloudflare_client - when DISCOURSE - discourse_client when HUGGING_FACE hugging_face_client when OPEN_AI open_ai_client - when GEMINI + when GOOGLE gemini_client else raise "Uknown embeddings provider" @@ -91,17 +92,6 @@ def cloudflare_client DiscourseAi::Inference::CloudflareWorkersAi.new(endpoint_url, api_key) end - def discourse_client - client_url = endpoint_url - client_url = "#{client_url}/api/v1/classify" if url.starts_with?("srv://") - - DiscourseAi::Inference::DiscourseClassifier.new( - client_url, - api_key, - lookup_custom_param("model_name"), - ) - end - def hugging_face_client DiscourseAi::Inference::HuggingFaceTextEmbeddings.new(endpoint_url, api_key) end diff --git a/assets/javascripts/discourse/admin-discourse-ai-plugin-route-map.js b/assets/javascripts/discourse/admin-discourse-ai-plugin-route-map.js index e4c49570a..3e798a18d 100644 --- a/assets/javascripts/discourse/admin-discourse-ai-plugin-route-map.js +++ b/assets/javascripts/discourse/admin-discourse-ai-plugin-route-map.js @@ -20,5 +20,14 @@ export default { }); this.route("discourse-ai-spam", { path: "ai-spam" }); this.route("discourse-ai-usage", { path: "ai-usage" }); + + this.route( + "discourse-ai-embeddings", + { path: "ai-embeddings" }, + function () { + this.route("new"); + this.route("edit", { path: "/:id/edit" }); + } + ); }, }; diff --git a/assets/javascripts/discourse/admin/adapters/ai-embedding.js b/assets/javascripts/discourse/admin/adapters/ai-embedding.js new file mode 100644 index 000000000..5aa1b48ce --- /dev/null +++ b/assets/javascripts/discourse/admin/adapters/ai-embedding.js @@ -0,0 +1,21 @@ +import RestAdapter from "discourse/adapters/rest"; + +export default class Adapter extends RestAdapter { + jsonMode = true; + + basePath() { + return "/admin/plugins/discourse-ai/"; + } + + pathFor(store, type, findArgs) { + // removes underscores which are implemented in base + let path = + this.basePath(store, type, findArgs) + + store.pluralize(this.apiNameFor(type)); + return this.appendQueryParams(path, findArgs); + } + + apiNameFor() { + return "ai-embedding"; + } +} diff --git a/assets/javascripts/discourse/admin/models/ai-embedding.js b/assets/javascripts/discourse/admin/models/ai-embedding.js new file mode 100644 index 000000000..d61218f97 --- /dev/null +++ b/assets/javascripts/discourse/admin/models/ai-embedding.js @@ -0,0 +1,33 @@ +import { ajax } from "discourse/lib/ajax"; +import RestModel from "discourse/models/rest"; + +export default class AiEmbedding extends RestModel { + createProperties() { + return this.getProperties( + "id", + "display_name", + "dimensions", + "provider", + "tokenizer_class", + "dimensions", + "url", + "api_key", + "max_sequence_length", + "provider_params", + "pg_function" + ); + } + + updateProperties() { + const attrs = this.createProperties(); + attrs.id = this.id; + + return attrs; + } + + async testConfig() { + return await ajax(`/admin/plugins/discourse-ai/ai-embeddings/test.json`, { + data: { ai_embedding: this.createProperties() }, + }); + } +} diff --git a/assets/javascripts/discourse/components/ai-embedding-editor.gjs b/assets/javascripts/discourse/components/ai-embedding-editor.gjs new file mode 100644 index 000000000..d687a74be --- /dev/null +++ b/assets/javascripts/discourse/components/ai-embedding-editor.gjs @@ -0,0 +1,289 @@ +import Component from "@glimmer/component"; +import { tracked } from "@glimmer/tracking"; +import { action, computed } from "@ember/object"; +import BackButton from "discourse/components/back-button"; +import { Input } from "@ember/component"; +import ComboBox from "select-kit/components/combo-box"; +import DButton from "discourse/components/d-button"; +import i18n from "discourse-common/helpers/i18n"; +import { on } from "@ember/modifier"; +import { concat, get } from "@ember/helper"; +import { popupAjaxError } from "discourse/lib/ajax-error"; +import { later } from "@ember/runloop"; +import { service } from "@ember/service"; +import icon from "discourse-common/helpers/d-icon"; + +export default class AiEmbeddingEditor extends Component { + @service toasts; + @service router; + @service dialog; + + @tracked isSaving = false; + + @tracked testRunning = false; + @tracked testResult = null; + @tracked testError = null; + @tracked apiKeySecret = true; + + get selectedProviders() { + const t = (provName) => { + return I18n.t(`discourse_ai.embeddings.providers.${provName}`); + }; + + return this.args.embeddings.resultSetMeta.providers.map((prov) => { + return { id: prov, name: t(prov) }; + }); + } + + get distanceFunctions() { + return this.args.embeddings.resultSetMeta.distance_functions.map((df) => { + return { id: df, name: df }; + }); + } + + @computed("args.model.provider") + get metaProviderParams() { + return ( + this.args.embeddings.resultSetMeta.provider_params[ + this.args?.model?.provider + ] || {} + ); + } + + get testErrorMessage() { + return I18n.t("discourse_ai.llms.tests.failure", { error: this.testError }); + } + + get displayTestResult() { + return this.testRunning || this.testResult !== null; + } + + @action + makeApiKeySecret() { + this.apiKeySecret = true; + } + + @action + toggleApiKeySecret() { + this.apiKeySecret = !this.apiKeySecret; + } + + @action + async save() { + this.isSaving = true; + const isNew = this.args.model.isNew; + + try { + await this.args.model.save(); + + if (isNew) { + this.args.llms.addObject(this.args.model); + this.router.transitionTo( + "adminPlugins.show.discourse-ai-embeddings.index" + ); + } else { + this.toasts.success({ + data: { message: I18n.t("discourse_ai.embeddings.saved") }, + duration: 2000, + }); + } + } catch (e) { + popupAjaxError(e); + } finally { + later(() => { + this.isSaving = false; + }, 1000); + } + } + + @action + async test() { + this.testRunning = true; + + try { + const configTestResult = await this.args.model.testConfig(); + this.testResult = configTestResult.success; + + if (this.testResult) { + this.testError = null; + } else { + this.testError = configTestResult.error; + } + } catch (e) { + popupAjaxError(e); + } finally { + later(() => { + this.testRunning = false; + }, 1000); + } + } + + @action + delete() { + return this.dialog.confirm({ + message: I18n.t("discourse_ai.embeddings.confirm_delete"), + didConfirm: () => { + return this.args.model + .destroyRecord() + .then(() => { + this.args.llms.removeObject(this.args.model); + this.router.transitionTo( + "adminPlugins.show.discourse-ai-embeddings.index" + ); + }) + .catch(popupAjaxError); + }, + }); + } + + +} diff --git a/assets/javascripts/discourse/components/ai-embeddings-list-editor.gjs b/assets/javascripts/discourse/components/ai-embeddings-list-editor.gjs new file mode 100644 index 000000000..a34c29b74 --- /dev/null +++ b/assets/javascripts/discourse/components/ai-embeddings-list-editor.gjs @@ -0,0 +1,95 @@ +import Component from "@glimmer/component"; +import { service } from "@ember/service"; +import DBreadcrumbsItem from "discourse/components/d-breadcrumbs-item"; +import i18n from "discourse-common/helpers/i18n"; +import AiEmbeddingEditor from "./ai-embedding-editor"; +import DPageSubheader from "discourse/components/d-page-subheader"; +import DButton from "discourse/components/d-button"; +import { concat } from "@ember/helper"; + + + + +export default class AiEmbeddingsListEditor extends Component { + @service adminPluginNavManager; + + get hasEmbeddingElements() { + return this.args.embeddings.length !== 0; + } + + +} diff --git a/assets/javascripts/initializers/admin-plugin-configuration-nav.js b/assets/javascripts/initializers/admin-plugin-configuration-nav.js index 119744a1e..54499728f 100644 --- a/assets/javascripts/initializers/admin-plugin-configuration-nav.js +++ b/assets/javascripts/initializers/admin-plugin-configuration-nav.js @@ -12,6 +12,10 @@ export default { withPluginApi("1.1.0", (api) => { api.addAdminPluginConfigurationNav("discourse-ai", PLUGIN_NAV_MODE_TOP, [ + { + label: "discourse_ai.embeddings.short_title", + route: "adminPlugins.show.discourse-ai-embeddings", + }, { label: "discourse_ai.llms.short_title", route: "adminPlugins.show.discourse-ai-llms", diff --git a/assets/stylesheets/modules/embeddings/common/ai-embedding-editor.scss b/assets/stylesheets/modules/embeddings/common/ai-embedding-editor.scss new file mode 100644 index 000000000..9efe64945 --- /dev/null +++ b/assets/stylesheets/modules/embeddings/common/ai-embedding-editor.scss @@ -0,0 +1,26 @@ +.ai-embedding-editor { + padding-left: 0.5em; + + .ai-embedding-editor-input { + width: 350px; + } + + .ai-embedding-editor-tests { + &__failure { + color: var(--danger); + } + + &__success { + color: var(--success); + } + } + + &__api-key { + margin-right: 0.5em; + } + + &__secret-api-key-group { + display: flex; + align-items: center; + } +} \ No newline at end of file diff --git a/config/locales/client.en.yml b/config/locales/client.en.yml index bd5a2c9e2..2b8777e7a 100644 --- a/config/locales/client.en.yml +++ b/config/locales/client.en.yml @@ -478,6 +478,35 @@ en: accuracy: "Accuracy:" embeddings: + short_title: "Embeddings" + description: "Embeddings are a crucial component of the Discourse AI plugin, enabling features like related topics and semantic search." + new: "New embedding" + back: "Back" + save: "Save" + saved: "Embedding configuration saved" + delete: "Delete" + confirm_delete: Are you sure you want to remove this embedding configuration? + tests: + title: "Run test" + running: "Running test..." + success: "Success!" + failure: "Attempting to generate an embedding resulted in: %{error}" + + display_name: "Name" + provider: "Provider" + url: "Embeddings service URL" + api_key: "Embeddings service API Key" + tokenizer: "Tokenizer" + dimensions: "Embedding dimensions" + max_sequence_length: "Sequence length" + distance_function: "Distance function" + providers: + hugging_face: "Hugging Face" + open_ai: "OpenAI" + google: "Google" + cloudflare: "Cloudflare" + + semantic_search: "Topics (Semantic)" semantic_search_loading: "Searching for more results using AI" semantic_search_results: diff --git a/lib/inference/discourse_classifier.rb b/lib/inference/discourse_classifier.rb deleted file mode 100644 index 43a2462da..000000000 --- a/lib/inference/discourse_classifier.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -module ::DiscourseAi - module Inference - class DiscourseClassifier - def initialize(endpoint, api_key, model, referer = Discourse.base_url) - @endpoint = endpoint - @api_key = api_key - @model = model - @referer = referer - end - - attr_reader :endpoint, :api_key, :model, :referer - - def perform!(content) - headers = { "Referer" => referer, "Content-Type" => "application/json" } - headers["X-API-KEY"] = api_key if api_key.present? - - conn = Faraday.new { |f| f.adapter FinalDestination::FaradayAdapter } - response = conn.post(endpoint, { model: model, content: content }.to_json, headers) - - if ![200, 415].include?(response.status) - raise raise Net::HTTPBadResponse.new(response.body.to_s) - end - - JSON.parse(response.body, symbolize_names: true) - end - end - end -end diff --git a/plugin.rb b/plugin.rb index ea3c719ad..1dec2cd4b 100644 --- a/plugin.rb +++ b/plugin.rb @@ -35,6 +35,7 @@ register_asset "stylesheets/modules/sentiment/common/dashboard.scss" register_asset "stylesheets/modules/llms/common/ai-llms-editor.scss" +register_asset "stylesheets/modules/embeddings/common/ai-embedding-editor.scss" register_asset "stylesheets/modules/llms/common/usage.scss" register_asset "stylesheets/modules/llms/common/spam.scss" diff --git a/spec/fabricators/embedding_definition_fabricator.rb b/spec/fabricators/embedding_definition_fabricator.rb index f240e4657..5daee9777 100644 --- a/spec/fabricators/embedding_definition_fabricator.rb +++ b/spec/fabricators/embedding_definition_fabricator.rb @@ -2,25 +2,16 @@ Fabricator(:embedding_definition) do display_name "Multilingual E5 Large" - provider "discourse" + provider "hugging_face" tokenizer_class "DiscourseAi::Tokenizer::MultilingualE5LargeTokenizer" api_key "123" url "https://test.com/embeddings" - provider_params { { model_name: "multilingual-e5-large" } } + provider_params nil pg_function "<=>" max_sequence_length 512 dimensions 1024 end -Fabricator(:hugging_face_embedding_def, from: :embedding_definition) do - display_name "BGE M3" - provider "hugging_face" - tokenizer_class "DiscourseAi::Tokenizer::BgeM3Tokenizer" - max_sequence_length 8192 - pg_function "<#>" - provider_params nil -end - Fabricator(:cloudflare_embedding_def, from: :embedding_definition) do display_name "BGE Large EN" provider "cloudflare" @@ -41,7 +32,7 @@ Fabricator(:gemini_embedding_def, from: :embedding_definition) do display_name "Gemini's embedding-001" - provider "gemini" + provider "google" dimensions 768 max_sequence_length 1536 tokenizer_class "DiscourseAi::Tokenizer::OpenAiTokenizer" diff --git a/spec/lib/modules/embeddings/vector_spec.rb b/spec/lib/modules/embeddings/vector_spec.rb index 9279624d2..1b6939a1a 100644 --- a/spec/lib/modules/embeddings/vector_spec.rb +++ b/spec/lib/modules/embeddings/vector_spec.rb @@ -106,21 +106,7 @@ def stub_vector_mapping(text, expected_embedding) context "with hugging_face as the provider" do end - context "with discourse as the provider" do - fab!(:vdef) { Fabricate(:embedding_definition) } - - def stub_vector_mapping(text, expected_embedding) - EmbeddingsGenerationStubs.discourse_service( - vdef.lookup_custom_param("model_name"), - text, - expected_embedding, - ) - end - - it_behaves_like "generates and store embeddings using a vector definition" - end - - context "with gemini as the provider" do + context "with google as the provider" do fab!(:vdef) { Fabricate(:gemini_embedding_def) } def stub_vector_mapping(text, expected_embedding)