feat: similarity check in test suites

xavidop · Jan 24, 2025 · 3d24a03 · 3d24a03
1 parent 3a3ba4b
commit 3d24a03
Show file tree

Hide file tree

Showing 37 changed files with 321 additions and 34 deletions.
diff --git a/cmd/root.go b/cmd/root.go
@@ -65,6 +65,7 @@ func init() {
 	// Add the subcommands
 	rootCmd.PersistentFlags().BoolVarP(&global.Verbose, "verbose", "v", false, "verbose error output (with stack trace) (optional)")
 	rootCmd.PersistentFlags().StringVarP(&global.VoiceflowAPIKey, "voiceflow-api-key", "x", "", "Voiceflow API Key (optional)")
+	rootCmd.PersistentFlags().StringVarP(&global.OpenAIAPIKey, "open-api-key", "z", "", "Open API Key (optional)")
 	rootCmd.PersistentFlags().StringVarP(&global.VoiceflowSubdomain, "voiceflow-subdomain", "b", "", "Voiceflow Base URL (optional). Default: empty")
 	rootCmd.PersistentFlags().BoolVarP(&global.SkipUpdate, "skip-update-check", "u", false, "Skip the check for updates check run before every command (optional)")
 	rootCmd.PersistentFlags().StringVarP(&global.Output, "output-format", "o", "text", "Output Format. Options: text, json. Default: text (optional)")

diff --git a/cmd/test/execute.go b/cmd/test/execute.go
@@ -6,6 +6,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/xavidop/voiceflow-cli/cmd/cmdutils"
 	"github.com/xavidop/voiceflow-cli/internal/global"
+	"github.com/xavidop/voiceflow-cli/internal/openai"
 	"github.com/xavidop/voiceflow-cli/internal/voiceflow"
 	"github.com/xavidop/voiceflow-cli/pkg/test"
 )
@@ -20,6 +21,7 @@ var executeCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		suite := args[0]
 		voiceflow.SetVoiceflowAPIKey()
+		openai.SetOpenAIAPIKey()
 		if err := test.ExecuteSuite(suite); err != nil {
 			global.Log.Errorf("%s", err.Error())
 			os.Exit(1)

diff --git a/docs/docs/cmd/voiceflow.md b/docs/docs/cmd/voiceflow.md
@@ -21,6 +21,7 @@ voiceflow [flags]
 
 ```
   -h, --help                         help for voiceflow
+  -z, --open-api-key string          Open API Key (optional)
   -o, --output-format string         Output Format. Options: text, json. Default: text (optional) (default "text")
   -u, --skip-update-check            Skip the check for updates check run before every command (optional)
   -v, --verbose                      verbose error output (with stack trace) (optional)

diff --git a/docs/docs/cmd/voiceflow_agent.md b/docs/docs/cmd/voiceflow_agent.md
@@ -15,6 +15,7 @@ voiceflow agent [flags]
 ## Options inherited from parent commands
 
 ```
+  -z, --open-api-key string          Open API Key (optional)
   -o, --output-format string         Output Format. Options: text, json. Default: text (optional) (default "text")
   -u, --skip-update-check            Skip the check for updates check run before every command (optional)
   -v, --verbose                      verbose error output (with stack trace) (optional)

diff --git a/docs/docs/cmd/voiceflow_agent_export.md b/docs/docs/cmd/voiceflow_agent_export.md
@@ -18,6 +18,7 @@ voiceflow agent export [flags]
 ## Options inherited from parent commands
 
 ```
+  -z, --open-api-key string          Open API Key (optional)
   -o, --output-format string         Output Format. Options: text, json. Default: text (optional) (default "text")
   -u, --skip-update-check            Skip the check for updates check run before every command (optional)
   -v, --verbose                      verbose error output (with stack trace) (optional)

diff --git a/docs/docs/cmd/voiceflow_analytics.md b/docs/docs/cmd/voiceflow_analytics.md
@@ -15,6 +15,7 @@ voiceflow analytics [flags]
 ## Options inherited from parent commands
 
 ```
+  -z, --open-api-key string          Open API Key (optional)
   -o, --output-format string         Output Format. Options: text, json. Default: text (optional) (default "text")
   -u, --skip-update-check            Skip the check for updates check run before every command (optional)
   -v, --verbose                      verbose error output (with stack trace) (optional)

diff --git a/docs/docs/cmd/voiceflow_analytics_fetch.md b/docs/docs/cmd/voiceflow_analytics_fetch.md
@@ -21,6 +21,7 @@ voiceflow analytics fetch [flags]
 ## Options inherited from parent commands
 
 ```
+  -z, --open-api-key string          Open API Key (optional)
   -o, --output-format string         Output Format. Options: text, json. Default: text (optional) (default "text")
   -u, --skip-update-check            Skip the check for updates check run before every command (optional)
   -v, --verbose                      verbose error output (with stack trace) (optional)

diff --git a/docs/docs/cmd/voiceflow_completion.md b/docs/docs/cmd/voiceflow_completion.md
@@ -17,6 +17,7 @@ See each sub-command's help for details on how to use the generated script.
 ## Options inherited from parent commands
 
 ```
+  -z, --open-api-key string          Open API Key (optional)
   -o, --output-format string         Output Format. Options: text, json. Default: text (optional) (default "text")
   -u, --skip-update-check            Skip the check for updates check run before every command (optional)
   -v, --verbose                      verbose error output (with stack trace) (optional)

diff --git a/docs/docs/cmd/voiceflow_completion_bash.md b/docs/docs/cmd/voiceflow_completion_bash.md
@@ -40,6 +40,7 @@ voiceflow completion bash
 ## Options inherited from parent commands
 
 ```
+  -z, --open-api-key string          Open API Key (optional)
   -o, --output-format string         Output Format. Options: text, json. Default: text (optional) (default "text")
   -u, --skip-update-check            Skip the check for updates check run before every command (optional)
   -v, --verbose                      verbose error output (with stack trace) (optional)

diff --git a/docs/docs/cmd/voiceflow_completion_fish.md b/docs/docs/cmd/voiceflow_completion_fish.md
@@ -31,6 +31,7 @@ voiceflow completion fish [flags]
 ## Options inherited from parent commands
 
 ```
+  -z, --open-api-key string          Open API Key (optional)
   -o, --output-format string         Output Format. Options: text, json. Default: text (optional) (default "text")
   -u, --skip-update-check            Skip the check for updates check run before every command (optional)
   -v, --verbose                      verbose error output (with stack trace) (optional)

diff --git a/docs/docs/cmd/voiceflow_completion_powershell.md b/docs/docs/cmd/voiceflow_completion_powershell.md
@@ -28,6 +28,7 @@ voiceflow completion powershell [flags]
 ## Options inherited from parent commands
 
 ```
+  -z, --open-api-key string          Open API Key (optional)
   -o, --output-format string         Output Format. Options: text, json. Default: text (optional) (default "text")
   -u, --skip-update-check            Skip the check for updates check run before every command (optional)
   -v, --verbose                      verbose error output (with stack trace) (optional)

diff --git a/docs/docs/cmd/voiceflow_completion_zsh.md b/docs/docs/cmd/voiceflow_completion_zsh.md
@@ -42,6 +42,7 @@ voiceflow completion zsh [flags]
 ## Options inherited from parent commands
 
 ```
+  -z, --open-api-key string          Open API Key (optional)
   -o, --output-format string         Output Format. Options: text, json. Default: text (optional) (default "text")
   -u, --skip-update-check            Skip the check for updates check run before every command (optional)
   -v, --verbose                      verbose error output (with stack trace) (optional)

diff --git a/docs/docs/cmd/voiceflow_document.md b/docs/docs/cmd/voiceflow_document.md
@@ -15,6 +15,7 @@ voiceflow document [flags]
 ## Options inherited from parent commands
 
 ```
+  -z, --open-api-key string          Open API Key (optional)
   -o, --output-format string         Output Format. Options: text, json. Default: text (optional) (default "text")
   -u, --skip-update-check            Skip the check for updates check run before every command (optional)
   -v, --verbose                      verbose error output (with stack trace) (optional)

diff --git a/docs/docs/cmd/voiceflow_document_upload-file.md b/docs/docs/cmd/voiceflow_document_upload-file.md
@@ -24,6 +24,7 @@ voiceflow document upload-file [flags]
 ## Options inherited from parent commands
 
 ```
+  -z, --open-api-key string          Open API Key (optional)
   -o, --output-format string         Output Format. Options: text, json. Default: text (optional) (default "text")
   -u, --skip-update-check            Skip the check for updates check run before every command (optional)
   -v, --verbose                      verbose error output (with stack trace) (optional)

diff --git a/docs/docs/cmd/voiceflow_document_upload-url.md b/docs/docs/cmd/voiceflow_document_upload-url.md
@@ -25,6 +25,7 @@ voiceflow document upload-url [flags]
 ## Options inherited from parent commands
 
 ```
+  -z, --open-api-key string          Open API Key (optional)
   -o, --output-format string         Output Format. Options: text, json. Default: text (optional) (default "text")
   -u, --skip-update-check            Skip the check for updates check run before every command (optional)
   -v, --verbose                      verbose error output (with stack trace) (optional)

diff --git a/docs/docs/cmd/voiceflow_jsonschema.md b/docs/docs/cmd/voiceflow_jsonschema.md
@@ -16,6 +16,7 @@ voiceflow jsonschema [flags]
 ## Options inherited from parent commands
 
 ```
+  -z, --open-api-key string          Open API Key (optional)
   -o, --output-format string         Output Format. Options: text, json. Default: text (optional) (default "text")
   -u, --skip-update-check            Skip the check for updates check run before every command (optional)
   -v, --verbose                      verbose error output (with stack trace) (optional)

diff --git a/docs/docs/cmd/voiceflow_kb.md b/docs/docs/cmd/voiceflow_kb.md
@@ -15,6 +15,7 @@ voiceflow kb [flags]
 ## Options inherited from parent commands
 
 ```
+  -z, --open-api-key string          Open API Key (optional)
   -o, --output-format string         Output Format. Options: text, json. Default: text (optional) (default "text")
   -u, --skip-update-check            Skip the check for updates check run before every command (optional)
   -v, --verbose                      verbose error output (with stack trace) (optional)

diff --git a/docs/docs/cmd/voiceflow_kb_query.md b/docs/docs/cmd/voiceflow_kb_query.md
@@ -28,6 +28,7 @@ voiceflow kb query [flags]
 ## Options inherited from parent commands
 
 ```
+  -z, --open-api-key string          Open API Key (optional)
   -o, --output-format string         Output Format. Options: text, json. Default: text (optional) (default "text")
   -u, --skip-update-check            Skip the check for updates check run before every command (optional)
   -v, --verbose                      verbose error output (with stack trace) (optional)

diff --git a/docs/docs/cmd/voiceflow_test.md b/docs/docs/cmd/voiceflow_test.md
@@ -15,6 +15,7 @@ voiceflow test [flags]
 ## Options inherited from parent commands
 
 ```
+  -z, --open-api-key string          Open API Key (optional)
   -o, --output-format string         Output Format. Options: text, json. Default: text (optional) (default "text")
   -u, --skip-update-check            Skip the check for updates check run before every command (optional)
   -v, --verbose                      verbose error output (with stack trace) (optional)

diff --git a/docs/docs/cmd/voiceflow_test_execute.md b/docs/docs/cmd/voiceflow_test_execute.md
@@ -15,6 +15,7 @@ voiceflow test execute [suite-path] [flags]
 ## Options inherited from parent commands
 
 ```
+  -z, --open-api-key string          Open API Key (optional)
   -o, --output-format string         Output Format. Options: text, json. Default: text (optional) (default "text")
   -u, --skip-update-check            Skip the check for updates check run before every command (optional)
   -v, --verbose                      verbose error output (with stack trace) (optional)

diff --git a/docs/docs/cmd/voiceflow_transcript.md b/docs/docs/cmd/voiceflow_transcript.md
@@ -15,6 +15,7 @@ voiceflow transcript [flags]
 ## Options inherited from parent commands
 
 ```
+  -z, --open-api-key string          Open API Key (optional)
   -o, --output-format string         Output Format. Options: text, json. Default: text (optional) (default "text")
   -u, --skip-update-check            Skip the check for updates check run before every command (optional)
   -v, --verbose                      verbose error output (with stack trace) (optional)

diff --git a/docs/docs/cmd/voiceflow_transcript_fetch-all.md b/docs/docs/cmd/voiceflow_transcript_fetch-all.md
@@ -21,6 +21,7 @@ voiceflow transcript fetch-all [flags]
 ## Options inherited from parent commands
 
 ```
+  -z, --open-api-key string          Open API Key (optional)
   -o, --output-format string         Output Format. Options: text, json. Default: text (optional) (default "text")
   -u, --skip-update-check            Skip the check for updates check run before every command (optional)
   -v, --verbose                      verbose error output (with stack trace) (optional)

diff --git a/docs/docs/cmd/voiceflow_transcript_fetch.md b/docs/docs/cmd/voiceflow_transcript_fetch.md
@@ -18,6 +18,7 @@ voiceflow transcript fetch [flags]
 ## Options inherited from parent commands
 
 ```
+  -z, --open-api-key string          Open API Key (optional)
   -o, --output-format string         Output Format. Options: text, json. Default: text (optional) (default "text")
   -u, --skip-update-check            Skip the check for updates check run before every command (optional)
   -v, --verbose                      verbose error output (with stack trace) (optional)

diff --git a/docs/docs/cmd/voiceflow_transcript_to-test.md b/docs/docs/cmd/voiceflow_transcript_to-test.md
@@ -20,6 +20,7 @@ voiceflow transcript to-test [flags]
 ## Options inherited from parent commands
 
 ```
+  -z, --open-api-key string          Open API Key (optional)
   -o, --output-format string         Output Format. Options: text, json. Default: text (optional) (default "text")
   -u, --skip-update-check            Skip the check for updates check run before every command (optional)
   -v, --verbose                      verbose error output (with stack trace) (optional)

diff --git a/docs/docs/cmd/voiceflow_version.md b/docs/docs/cmd/voiceflow_version.md
@@ -15,6 +15,7 @@ voiceflow version [flags]
 ## Options inherited from parent commands
 
 ```
+  -z, --open-api-key string          Open API Key (optional)
   -o, --output-format string         Output Format. Options: text, json. Default: text (optional) (default "text")
   -u, --skip-update-check            Skip the check for updates check run before every command (optional)
   -v, --verbose                      verbose error output (with stack trace) (optional)

diff --git a/docs/docs/overview/authentication.md b/docs/docs/overview/authentication.md
@@ -1,9 +1,15 @@
 # Authentication
 
+## Voiceflow API Key
+
 `voiceflow-cli` uses Voiceflow APIs. To interact with your Vocieflow projects you will need a [Voiceflow API Key](https://docs.voiceflow.com/reference/authentication). You can get your API Key in your Voiceflow project > Integration page. You can pass the API Key to the CLI using the `--voiceflow-api-key` flag or by setting the `VF_API_KEY` environment variable. `voiceflow-cli` also works with `.env` files. You can create a `.env` file in the root of your project and add the `VF_API_KEY` variable to it.
 
 The `voiceflow-cli` source code is open source, you can check it out [here](https://github.com/xavidop/voiceflow-cli) to learn more about the actions the tool performs.
 
 ## Base URL
 
-The base URL for the Voiceflow API is `https://<api>.<subdomain>.voiceflow.com`. The default value is without subdomain: `https://<api>.voiceflow.com`. If you are using a different Voiceflow environment, you can pass the subdomain using the `--voiceflow-subdomain` flag.
+The base URL for the Voiceflow API is `https://<api>.<subdomain>.voiceflow.com`. The default value is without subdomain: `https://<api>.voiceflow.com`. If you are using a different Voiceflow environment, you can pass the subdomain using the `--voiceflow-subdomain` flag.
+
+## Open AI PI Key
+
+`voiceflow-cli` uses Open AI APIs. To interact with Open AI you will need an API Key. You can get your API Key in your Open AI account. You can pass the API Key to the CLI using the `--openai-api-key` flag or by setting the `OPENAI_API_KEY` environment variable. `voiceflow-cli` also works with `.env` files. You can create a `.env` file in the root of your project and add the `OPENAI_API_KEY` variable to it.
diff --git a/docs/docs/static/conversationsuite.json b/docs/docs/static/conversationsuite.json
@@ -84,6 +84,38 @@
 							"agent"
 						]
 					},
+					"SimilarityConfig": {
+						"properties": {
+							"provider": {
+								"type": "string"
+							},
+							"model": {
+								"type": "string"
+							},
+							"temperature": {
+								"type": "number"
+							},
+							"similarityThreshold": {
+								"type": "number"
+							},
+							"top_k": {
+								"type": "integer"
+							},
+							"top_p": {
+								"type": "number"
+							}
+						},
+						"additionalProperties": false,
+						"type": "object",
+						"required": [
+							"provider",
+							"model",
+							"temperature",
+							"similarityThreshold",
+							"top_k",
+							"top_p"
+						]
+					},
 					"Test": {
 						"properties": {
 							"name": {
@@ -132,6 +164,15 @@
 							},
 							"value": {
 								"type": "string"
+							},
+							"values": {
+								"items": {
+									"type": "string"
+								},
+								"type": "array"
+							},
+							"similarityConfig": {
+								"$ref": "#/$defs/SimilarityConfig"
 							}
 						},
 						"additionalProperties": false,

diff --git a/docs/docs/static/conversationtest.json b/docs/docs/static/conversationtest.json
@@ -38,6 +38,38 @@
 					"agent"
 				]
 			},
+			"SimilarityConfig": {
+				"properties": {
+					"provider": {
+						"type": "string"
+					},
+					"model": {
+						"type": "string"
+					},
+					"temperature": {
+						"type": "number"
+					},
+					"similarityThreshold": {
+						"type": "number"
+					},
+					"top_k": {
+						"type": "integer"
+					},
+					"top_p": {
+						"type": "number"
+					}
+				},
+				"additionalProperties": false,
+				"type": "object",
+				"required": [
+					"provider",
+					"model",
+					"temperature",
+					"similarityThreshold",
+					"top_k",
+					"top_p"
+				]
+			},
 			"Test": {
 				"properties": {
 					"name": {
@@ -86,6 +118,15 @@
 					},
 					"value": {
 						"type": "string"
+					},
+					"values": {
+						"items": {
+							"type": "string"
+						},
+						"type": "array"
+					},
+					"similarityConfig": {
+						"$ref": "#/$defs/SimilarityConfig"
 					}
 				},
 				"additionalProperties": false,