diff --git a/cmd/root.go b/cmd/root.go index b240f5f..fd55a0a 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -65,6 +65,7 @@ func init() { // Add the subcommands rootCmd.PersistentFlags().BoolVarP(&global.Verbose, "verbose", "v", false, "verbose error output (with stack trace) (optional)") rootCmd.PersistentFlags().StringVarP(&global.VoiceflowAPIKey, "voiceflow-api-key", "x", "", "Voiceflow API Key (optional)") + rootCmd.PersistentFlags().StringVarP(&global.OpenAIAPIKey, "open-api-key", "z", "", "Open API Key (optional)") rootCmd.PersistentFlags().StringVarP(&global.VoiceflowSubdomain, "voiceflow-subdomain", "b", "", "Voiceflow Base URL (optional). Default: empty") rootCmd.PersistentFlags().BoolVarP(&global.SkipUpdate, "skip-update-check", "u", false, "Skip the check for updates check run before every command (optional)") rootCmd.PersistentFlags().StringVarP(&global.Output, "output-format", "o", "text", "Output Format. Options: text, json. Default: text (optional)") diff --git a/cmd/test/execute.go b/cmd/test/execute.go index 0075b81..9adcc4a 100644 --- a/cmd/test/execute.go +++ b/cmd/test/execute.go @@ -6,6 +6,7 @@ import ( "github.com/spf13/cobra" "github.com/xavidop/voiceflow-cli/cmd/cmdutils" "github.com/xavidop/voiceflow-cli/internal/global" + "github.com/xavidop/voiceflow-cli/internal/openai" "github.com/xavidop/voiceflow-cli/internal/voiceflow" "github.com/xavidop/voiceflow-cli/pkg/test" ) @@ -20,6 +21,7 @@ var executeCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { suite := args[0] voiceflow.SetVoiceflowAPIKey() + openai.SetOpenAIAPIKey() if err := test.ExecuteSuite(suite); err != nil { global.Log.Errorf("%s", err.Error()) os.Exit(1) diff --git a/docs/docs/cmd/voiceflow.md b/docs/docs/cmd/voiceflow.md index 1db546b..28859be 100644 --- a/docs/docs/cmd/voiceflow.md +++ b/docs/docs/cmd/voiceflow.md @@ -21,6 +21,7 @@ voiceflow [flags] ``` -h, --help help for voiceflow + -z, --open-api-key string Open API Key (optional) -o, --output-format string Output Format. Options: text, json. Default: text (optional) (default "text") -u, --skip-update-check Skip the check for updates check run before every command (optional) -v, --verbose verbose error output (with stack trace) (optional) diff --git a/docs/docs/cmd/voiceflow_agent.md b/docs/docs/cmd/voiceflow_agent.md index 9a9e1e0..0b6043d 100644 --- a/docs/docs/cmd/voiceflow_agent.md +++ b/docs/docs/cmd/voiceflow_agent.md @@ -15,6 +15,7 @@ voiceflow agent [flags] ## Options inherited from parent commands ``` + -z, --open-api-key string Open API Key (optional) -o, --output-format string Output Format. Options: text, json. Default: text (optional) (default "text") -u, --skip-update-check Skip the check for updates check run before every command (optional) -v, --verbose verbose error output (with stack trace) (optional) diff --git a/docs/docs/cmd/voiceflow_agent_export.md b/docs/docs/cmd/voiceflow_agent_export.md index 5ae9493..b6d9e5f 100644 --- a/docs/docs/cmd/voiceflow_agent_export.md +++ b/docs/docs/cmd/voiceflow_agent_export.md @@ -18,6 +18,7 @@ voiceflow agent export [flags] ## Options inherited from parent commands ``` + -z, --open-api-key string Open API Key (optional) -o, --output-format string Output Format. Options: text, json. Default: text (optional) (default "text") -u, --skip-update-check Skip the check for updates check run before every command (optional) -v, --verbose verbose error output (with stack trace) (optional) diff --git a/docs/docs/cmd/voiceflow_analytics.md b/docs/docs/cmd/voiceflow_analytics.md index 4f2e02c..cbf85a8 100644 --- a/docs/docs/cmd/voiceflow_analytics.md +++ b/docs/docs/cmd/voiceflow_analytics.md @@ -15,6 +15,7 @@ voiceflow analytics [flags] ## Options inherited from parent commands ``` + -z, --open-api-key string Open API Key (optional) -o, --output-format string Output Format. Options: text, json. Default: text (optional) (default "text") -u, --skip-update-check Skip the check for updates check run before every command (optional) -v, --verbose verbose error output (with stack trace) (optional) diff --git a/docs/docs/cmd/voiceflow_analytics_fetch.md b/docs/docs/cmd/voiceflow_analytics_fetch.md index 2cbfd0e..beffe3e 100644 --- a/docs/docs/cmd/voiceflow_analytics_fetch.md +++ b/docs/docs/cmd/voiceflow_analytics_fetch.md @@ -21,6 +21,7 @@ voiceflow analytics fetch [flags] ## Options inherited from parent commands ``` + -z, --open-api-key string Open API Key (optional) -o, --output-format string Output Format. Options: text, json. Default: text (optional) (default "text") -u, --skip-update-check Skip the check for updates check run before every command (optional) -v, --verbose verbose error output (with stack trace) (optional) diff --git a/docs/docs/cmd/voiceflow_completion.md b/docs/docs/cmd/voiceflow_completion.md index 10fd1c6..8b1e949 100644 --- a/docs/docs/cmd/voiceflow_completion.md +++ b/docs/docs/cmd/voiceflow_completion.md @@ -17,6 +17,7 @@ See each sub-command's help for details on how to use the generated script. ## Options inherited from parent commands ``` + -z, --open-api-key string Open API Key (optional) -o, --output-format string Output Format. Options: text, json. Default: text (optional) (default "text") -u, --skip-update-check Skip the check for updates check run before every command (optional) -v, --verbose verbose error output (with stack trace) (optional) diff --git a/docs/docs/cmd/voiceflow_completion_bash.md b/docs/docs/cmd/voiceflow_completion_bash.md index 20fdf14..cdb6f0e 100644 --- a/docs/docs/cmd/voiceflow_completion_bash.md +++ b/docs/docs/cmd/voiceflow_completion_bash.md @@ -40,6 +40,7 @@ voiceflow completion bash ## Options inherited from parent commands ``` + -z, --open-api-key string Open API Key (optional) -o, --output-format string Output Format. Options: text, json. Default: text (optional) (default "text") -u, --skip-update-check Skip the check for updates check run before every command (optional) -v, --verbose verbose error output (with stack trace) (optional) diff --git a/docs/docs/cmd/voiceflow_completion_fish.md b/docs/docs/cmd/voiceflow_completion_fish.md index c70a9fb..8dfde19 100644 --- a/docs/docs/cmd/voiceflow_completion_fish.md +++ b/docs/docs/cmd/voiceflow_completion_fish.md @@ -31,6 +31,7 @@ voiceflow completion fish [flags] ## Options inherited from parent commands ``` + -z, --open-api-key string Open API Key (optional) -o, --output-format string Output Format. Options: text, json. Default: text (optional) (default "text") -u, --skip-update-check Skip the check for updates check run before every command (optional) -v, --verbose verbose error output (with stack trace) (optional) diff --git a/docs/docs/cmd/voiceflow_completion_powershell.md b/docs/docs/cmd/voiceflow_completion_powershell.md index 55c35f7..b6cbc4f 100644 --- a/docs/docs/cmd/voiceflow_completion_powershell.md +++ b/docs/docs/cmd/voiceflow_completion_powershell.md @@ -28,6 +28,7 @@ voiceflow completion powershell [flags] ## Options inherited from parent commands ``` + -z, --open-api-key string Open API Key (optional) -o, --output-format string Output Format. Options: text, json. Default: text (optional) (default "text") -u, --skip-update-check Skip the check for updates check run before every command (optional) -v, --verbose verbose error output (with stack trace) (optional) diff --git a/docs/docs/cmd/voiceflow_completion_zsh.md b/docs/docs/cmd/voiceflow_completion_zsh.md index a8b89b3..f5bc1ed 100644 --- a/docs/docs/cmd/voiceflow_completion_zsh.md +++ b/docs/docs/cmd/voiceflow_completion_zsh.md @@ -42,6 +42,7 @@ voiceflow completion zsh [flags] ## Options inherited from parent commands ``` + -z, --open-api-key string Open API Key (optional) -o, --output-format string Output Format. Options: text, json. Default: text (optional) (default "text") -u, --skip-update-check Skip the check for updates check run before every command (optional) -v, --verbose verbose error output (with stack trace) (optional) diff --git a/docs/docs/cmd/voiceflow_document.md b/docs/docs/cmd/voiceflow_document.md index ac1e9f0..f887dbc 100644 --- a/docs/docs/cmd/voiceflow_document.md +++ b/docs/docs/cmd/voiceflow_document.md @@ -15,6 +15,7 @@ voiceflow document [flags] ## Options inherited from parent commands ``` + -z, --open-api-key string Open API Key (optional) -o, --output-format string Output Format. Options: text, json. Default: text (optional) (default "text") -u, --skip-update-check Skip the check for updates check run before every command (optional) -v, --verbose verbose error output (with stack trace) (optional) diff --git a/docs/docs/cmd/voiceflow_document_upload-file.md b/docs/docs/cmd/voiceflow_document_upload-file.md index cd3d5e6..501f9ca 100644 --- a/docs/docs/cmd/voiceflow_document_upload-file.md +++ b/docs/docs/cmd/voiceflow_document_upload-file.md @@ -24,6 +24,7 @@ voiceflow document upload-file [flags] ## Options inherited from parent commands ``` + -z, --open-api-key string Open API Key (optional) -o, --output-format string Output Format. Options: text, json. Default: text (optional) (default "text") -u, --skip-update-check Skip the check for updates check run before every command (optional) -v, --verbose verbose error output (with stack trace) (optional) diff --git a/docs/docs/cmd/voiceflow_document_upload-url.md b/docs/docs/cmd/voiceflow_document_upload-url.md index 93e0215..6eca4b6 100644 --- a/docs/docs/cmd/voiceflow_document_upload-url.md +++ b/docs/docs/cmd/voiceflow_document_upload-url.md @@ -25,6 +25,7 @@ voiceflow document upload-url [flags] ## Options inherited from parent commands ``` + -z, --open-api-key string Open API Key (optional) -o, --output-format string Output Format. Options: text, json. Default: text (optional) (default "text") -u, --skip-update-check Skip the check for updates check run before every command (optional) -v, --verbose verbose error output (with stack trace) (optional) diff --git a/docs/docs/cmd/voiceflow_jsonschema.md b/docs/docs/cmd/voiceflow_jsonschema.md index ff026b6..cfd865c 100644 --- a/docs/docs/cmd/voiceflow_jsonschema.md +++ b/docs/docs/cmd/voiceflow_jsonschema.md @@ -16,6 +16,7 @@ voiceflow jsonschema [flags] ## Options inherited from parent commands ``` + -z, --open-api-key string Open API Key (optional) -o, --output-format string Output Format. Options: text, json. Default: text (optional) (default "text") -u, --skip-update-check Skip the check for updates check run before every command (optional) -v, --verbose verbose error output (with stack trace) (optional) diff --git a/docs/docs/cmd/voiceflow_kb.md b/docs/docs/cmd/voiceflow_kb.md index a0048da..6fb3405 100644 --- a/docs/docs/cmd/voiceflow_kb.md +++ b/docs/docs/cmd/voiceflow_kb.md @@ -15,6 +15,7 @@ voiceflow kb [flags] ## Options inherited from parent commands ``` + -z, --open-api-key string Open API Key (optional) -o, --output-format string Output Format. Options: text, json. Default: text (optional) (default "text") -u, --skip-update-check Skip the check for updates check run before every command (optional) -v, --verbose verbose error output (with stack trace) (optional) diff --git a/docs/docs/cmd/voiceflow_kb_query.md b/docs/docs/cmd/voiceflow_kb_query.md index 6846ff2..bd13fda 100644 --- a/docs/docs/cmd/voiceflow_kb_query.md +++ b/docs/docs/cmd/voiceflow_kb_query.md @@ -28,6 +28,7 @@ voiceflow kb query [flags] ## Options inherited from parent commands ``` + -z, --open-api-key string Open API Key (optional) -o, --output-format string Output Format. Options: text, json. Default: text (optional) (default "text") -u, --skip-update-check Skip the check for updates check run before every command (optional) -v, --verbose verbose error output (with stack trace) (optional) diff --git a/docs/docs/cmd/voiceflow_test.md b/docs/docs/cmd/voiceflow_test.md index 8eb9663..f1b76f5 100644 --- a/docs/docs/cmd/voiceflow_test.md +++ b/docs/docs/cmd/voiceflow_test.md @@ -15,6 +15,7 @@ voiceflow test [flags] ## Options inherited from parent commands ``` + -z, --open-api-key string Open API Key (optional) -o, --output-format string Output Format. Options: text, json. Default: text (optional) (default "text") -u, --skip-update-check Skip the check for updates check run before every command (optional) -v, --verbose verbose error output (with stack trace) (optional) diff --git a/docs/docs/cmd/voiceflow_test_execute.md b/docs/docs/cmd/voiceflow_test_execute.md index 42ba05d..ac84e51 100644 --- a/docs/docs/cmd/voiceflow_test_execute.md +++ b/docs/docs/cmd/voiceflow_test_execute.md @@ -15,6 +15,7 @@ voiceflow test execute [suite-path] [flags] ## Options inherited from parent commands ``` + -z, --open-api-key string Open API Key (optional) -o, --output-format string Output Format. Options: text, json. Default: text (optional) (default "text") -u, --skip-update-check Skip the check for updates check run before every command (optional) -v, --verbose verbose error output (with stack trace) (optional) diff --git a/docs/docs/cmd/voiceflow_transcript.md b/docs/docs/cmd/voiceflow_transcript.md index 2fdf096..d72ec3c 100644 --- a/docs/docs/cmd/voiceflow_transcript.md +++ b/docs/docs/cmd/voiceflow_transcript.md @@ -15,6 +15,7 @@ voiceflow transcript [flags] ## Options inherited from parent commands ``` + -z, --open-api-key string Open API Key (optional) -o, --output-format string Output Format. Options: text, json. Default: text (optional) (default "text") -u, --skip-update-check Skip the check for updates check run before every command (optional) -v, --verbose verbose error output (with stack trace) (optional) diff --git a/docs/docs/cmd/voiceflow_transcript_fetch-all.md b/docs/docs/cmd/voiceflow_transcript_fetch-all.md index 7521d54..6f6ed2b 100644 --- a/docs/docs/cmd/voiceflow_transcript_fetch-all.md +++ b/docs/docs/cmd/voiceflow_transcript_fetch-all.md @@ -21,6 +21,7 @@ voiceflow transcript fetch-all [flags] ## Options inherited from parent commands ``` + -z, --open-api-key string Open API Key (optional) -o, --output-format string Output Format. Options: text, json. Default: text (optional) (default "text") -u, --skip-update-check Skip the check for updates check run before every command (optional) -v, --verbose verbose error output (with stack trace) (optional) diff --git a/docs/docs/cmd/voiceflow_transcript_fetch.md b/docs/docs/cmd/voiceflow_transcript_fetch.md index e33d6af..e0507bd 100644 --- a/docs/docs/cmd/voiceflow_transcript_fetch.md +++ b/docs/docs/cmd/voiceflow_transcript_fetch.md @@ -18,6 +18,7 @@ voiceflow transcript fetch [flags] ## Options inherited from parent commands ``` + -z, --open-api-key string Open API Key (optional) -o, --output-format string Output Format. Options: text, json. Default: text (optional) (default "text") -u, --skip-update-check Skip the check for updates check run before every command (optional) -v, --verbose verbose error output (with stack trace) (optional) diff --git a/docs/docs/cmd/voiceflow_transcript_to-test.md b/docs/docs/cmd/voiceflow_transcript_to-test.md index 8b3f3c8..e28c493 100644 --- a/docs/docs/cmd/voiceflow_transcript_to-test.md +++ b/docs/docs/cmd/voiceflow_transcript_to-test.md @@ -20,6 +20,7 @@ voiceflow transcript to-test [flags] ## Options inherited from parent commands ``` + -z, --open-api-key string Open API Key (optional) -o, --output-format string Output Format. Options: text, json. Default: text (optional) (default "text") -u, --skip-update-check Skip the check for updates check run before every command (optional) -v, --verbose verbose error output (with stack trace) (optional) diff --git a/docs/docs/cmd/voiceflow_version.md b/docs/docs/cmd/voiceflow_version.md index 4e8c81a..f0489cd 100644 --- a/docs/docs/cmd/voiceflow_version.md +++ b/docs/docs/cmd/voiceflow_version.md @@ -15,6 +15,7 @@ voiceflow version [flags] ## Options inherited from parent commands ``` + -z, --open-api-key string Open API Key (optional) -o, --output-format string Output Format. Options: text, json. Default: text (optional) (default "text") -u, --skip-update-check Skip the check for updates check run before every command (optional) -v, --verbose verbose error output (with stack trace) (optional) diff --git a/docs/docs/overview/authentication.md b/docs/docs/overview/authentication.md index bdd5dad..25a10ee 100644 --- a/docs/docs/overview/authentication.md +++ b/docs/docs/overview/authentication.md @@ -1,9 +1,15 @@ # Authentication +## Voiceflow API Key + `voiceflow-cli` uses Voiceflow APIs. To interact with your Vocieflow projects you will need a [Voiceflow API Key](https://docs.voiceflow.com/reference/authentication). You can get your API Key in your Voiceflow project > Integration page. You can pass the API Key to the CLI using the `--voiceflow-api-key` flag or by setting the `VF_API_KEY` environment variable. `voiceflow-cli` also works with `.env` files. You can create a `.env` file in the root of your project and add the `VF_API_KEY` variable to it. The `voiceflow-cli` source code is open source, you can check it out [here](https://github.com/xavidop/voiceflow-cli) to learn more about the actions the tool performs. ## Base URL -The base URL for the Voiceflow API is `https://..voiceflow.com`. The default value is without subdomain: `https://.voiceflow.com`. If you are using a different Voiceflow environment, you can pass the subdomain using the `--voiceflow-subdomain` flag. \ No newline at end of file +The base URL for the Voiceflow API is `https://..voiceflow.com`. The default value is without subdomain: `https://.voiceflow.com`. If you are using a different Voiceflow environment, you can pass the subdomain using the `--voiceflow-subdomain` flag. + +## Open AI PI Key + +`voiceflow-cli` uses Open AI APIs. To interact with Open AI you will need an API Key. You can get your API Key in your Open AI account. You can pass the API Key to the CLI using the `--openai-api-key` flag or by setting the `OPENAI_API_KEY` environment variable. `voiceflow-cli` also works with `.env` files. You can create a `.env` file in the root of your project and add the `OPENAI_API_KEY` variable to it. \ No newline at end of file diff --git a/docs/docs/static/conversationsuite.json b/docs/docs/static/conversationsuite.json index 774b4c4..0a1b28e 100644 --- a/docs/docs/static/conversationsuite.json +++ b/docs/docs/static/conversationsuite.json @@ -84,6 +84,38 @@ "agent" ] }, + "SimilarityConfig": { + "properties": { + "provider": { + "type": "string" + }, + "model": { + "type": "string" + }, + "temperature": { + "type": "number" + }, + "similarityThreshold": { + "type": "number" + }, + "top_k": { + "type": "integer" + }, + "top_p": { + "type": "number" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "provider", + "model", + "temperature", + "similarityThreshold", + "top_k", + "top_p" + ] + }, "Test": { "properties": { "name": { @@ -132,6 +164,15 @@ }, "value": { "type": "string" + }, + "values": { + "items": { + "type": "string" + }, + "type": "array" + }, + "similarityConfig": { + "$ref": "#/$defs/SimilarityConfig" } }, "additionalProperties": false, diff --git a/docs/docs/static/conversationtest.json b/docs/docs/static/conversationtest.json index 0d951fe..3b93e16 100644 --- a/docs/docs/static/conversationtest.json +++ b/docs/docs/static/conversationtest.json @@ -38,6 +38,38 @@ "agent" ] }, + "SimilarityConfig": { + "properties": { + "provider": { + "type": "string" + }, + "model": { + "type": "string" + }, + "temperature": { + "type": "number" + }, + "similarityThreshold": { + "type": "number" + }, + "top_k": { + "type": "integer" + }, + "top_p": { + "type": "number" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "provider", + "model", + "temperature", + "similarityThreshold", + "top_k", + "top_p" + ] + }, "Test": { "properties": { "name": { @@ -86,6 +118,15 @@ }, "value": { "type": "string" + }, + "values": { + "items": { + "type": "string" + }, + "type": "array" + }, + "similarityConfig": { + "$ref": "#/$defs/SimilarityConfig" } }, "additionalProperties": false, diff --git a/docs/docs/tests/tests.md b/docs/docs/tests/tests.md index dfa5bf4..3c4438e 100644 --- a/docs/docs/tests/tests.md +++ b/docs/docs/tests/tests.md @@ -48,6 +48,25 @@ interactions: validate: - type: regexp value: '/my-regex/' + + - id: test_4 + user: + type: text + audio: hello + agent: + # example with a similarity validation + validate: + - type: similarity + similarityConfig: + provider: 'openai' + model: 'gpt-4o' + temperature: 0.8 + top_k: 5 + top_p: 0.9 + similarityThreshold: 0.5 + values: + - 'hi' + - 'Hello' ``` ## Input types @@ -71,17 +90,6 @@ validate: value: pizza ``` -The `contains` validation has its own options: - -```yaml -validate: - # String validation to check if the response returned by Voiceflow is correct - - type: contains - value: pizza -``` - -If you set the `casesensitive` field to `true`, the validation will be case sensitive. By default, it is set to `false`. - ### Equals The equals validation type is a little bit more complex. It checks if the response returned by the Voiceflow agent is equal to the value specified in the `value` field. To use this type you have to set the `type` field to `equals` and the `value` field to the value you want to check: @@ -93,38 +101,52 @@ validate: value: Here you have 3 pizzas ``` -The `equals` validation has its own options: +### Regexp + +The regexp validation type is the most complex one. It checks if the response returned by the Voiceflow agent matches the regexp specified in the `value` field. To use this type you have to set the `type` field to `regexp` and the `value` field to the regular expression you want to check: ```yaml validate: # String validation to check if the response returned by Voiceflow is correct - - type: equals - value: Here you have 3 pizzas + - type: regexp + value: '/Here you have \d pizzas/' ``` -If you set the `casesensitive` field to `true`, the validation will be case sensitive. By default, it is set to `false`. - -### Regexp - -The regexp validation type is the most complex one. It checks if the response returned by the Voiceflow agent matches the regexp specified in the `value` field. To use this type you have to set the `type` field to `regexp` and the `value` field to the regular expression you want to check: +### TraceType +The traceType validation type checks if the response returned by the Voiceflow agent has the trace type specified in the `value` field. To use this type you have to set the `type` field to `traceType` and the `value` field to the trace type you want to check: ```yaml validate: # String validation to check if the response returned by Voiceflow is correct - - type: regexp - value: '/Here you have \d pizzas/' + - type: traceType + value: speak ``` -The `regexp` validation has its own options: +### Similarity +The similarity validation type checks if the response returned by the Voiceflow agent is similar to the values specified in the `values` field. To use this type you have to set the `type` field to `similarity` and the `values` field to the values you want to check: ```yaml validate: # String validation to check if the response returned by Voiceflow is correct - - type: regexp - value: '/Here you have \d pizzas/' + - type: similarity + similarityConfig: + provider: 'openai' + model: 'gpt-4o' + temperature: 0.8 + top_k: 5 + top_p: 0.9 + similarityThreshold: 0.5 + values: + - 'hi' + - 'Hello' ``` -If you set the `findinsubmatches` field to `true`, the validation will check if the regexp matches any of the submatches. By default, it is set to `false`. +You can also use the `similarityConfig` field to specify the similarity configuration. The `provider` field specifies the similarity provider you want to use. The `model` field specifies the model you want to use. The `temperature` field specifies the temperature you want to use. The `top_k` field specifies the top k you want to use. The `top_p` field specifies the top p you want to use. The `similarityThreshold` field specifies the similarity threshold you want to use. + +The only provider available for now is `openai`. + +For LLM Providers authentication please check the [Authentication](/overview/authentication) page. + ## JSON Schema diff --git a/examples/suite.yaml b/examples/suite.yaml index 0808f1b..b8a6311 100644 --- a/examples/suite.yaml +++ b/examples/suite.yaml @@ -1,6 +1,6 @@ name: Example Conversation Profiler Suite description: Suite used as an example -environmentName: development +environmentName: production tests: - id: test_1 file: ./tests/test_1.yaml diff --git a/examples/tests/test_1.yaml b/examples/tests/test_1.yaml index 4c8d4c4..af8ee6f 100644 --- a/examples/tests/test_1.yaml +++ b/examples/tests/test_1.yaml @@ -29,5 +29,16 @@ interactions: validate: - type: regexp value: 'classic' + - type: similarity + similarityConfig: + provider: 'openai' + model: 'gpt-4o' + temperature: 0.8 + top_k: 5 + top_p: 0.9 + similarityThreshold: 0.1 + values: + - 'work hard' + - 'expressing hard work' # yaml-language-server: $schema=https://voiceflow.xavidop.me/static/conversationtest.json \ No newline at end of file diff --git a/internal/global/vars.go b/internal/global/vars.go index d99e008..1eaec21 100644 --- a/internal/global/vars.go +++ b/internal/global/vars.go @@ -5,6 +5,7 @@ import "github.com/sirupsen/logrus" var Log logrus.Logger var VoiceflowAPIKey string +var OpenAIAPIKey string var VoiceflowSubdomain string var VersionString string var Output string diff --git a/internal/openai/authentication.go b/internal/openai/authentication.go new file mode 100644 index 0000000..5e1c704 --- /dev/null +++ b/internal/openai/authentication.go @@ -0,0 +1,14 @@ +package openai + +import ( + "os" + + "github.com/xavidop/voiceflow-cli/internal/global" +) + +func SetOpenAIAPIKey() { + + if global.OpenAIAPIKey == "" { + global.OpenAIAPIKey = os.Getenv("OPENAI_API_KEY") + } +} diff --git a/internal/types/tests/testTypes.go b/internal/types/tests/testTypes.go index c506f16..8b07d2c 100644 --- a/internal/types/tests/testTypes.go +++ b/internal/types/tests/testTypes.go @@ -22,7 +22,18 @@ type Agent struct { } type Validation struct { - ID string `yaml:"id" json:"id,omitempty"` - Type string `yaml:"type" json:"type"` - Value string `yaml:"value" json:"value"` + ID string `yaml:"id" json:"id,omitempty"` + Type string `yaml:"type" json:"type"` + Value string `yaml:"value" json:"value"` + Values []string `yaml:"values,omitempty" json:"values,omitempty"` + SimilarityConfig *SimilarityConfig `yaml:"similarityConfig,omitempty" json:"similarityConfig,omitempty"` +} + +type SimilarityConfig struct { + Provider string `yaml:"provider" json:"provider"` + Model string `yaml:"model" json:"model"` + Temperature float64 `yaml:"temperature" json:"temperature"` + SimilarityThreshold float64 `yaml:"similarityThreshold" json:"similarityThreshold"` + TopK int `yaml:"top_k" json:"top_k"` + TopP float64 `yaml:"top_p" json:"top_p"` } diff --git a/internal/types/voiceflow/interact/interactRequestTypes.go b/internal/types/voiceflow/interact/interactRequestTypes.go index 2271d7d..fadd63e 100644 --- a/internal/types/voiceflow/interact/interactRequestTypes.go +++ b/internal/types/voiceflow/interact/interactRequestTypes.go @@ -16,7 +16,7 @@ type State struct { } type InteratctionRequest struct { - Action Action `json:"action"` - Config Config `json:"config,omitempty"` - State State `json:"state,omitempty"` + Action Action `json:"action"` + Config *Config `json:"config,omitempty"` + State *State `json:"state,omitempty"` } diff --git a/pkg/openai/similarity.go b/pkg/openai/similarity.go new file mode 100644 index 0000000..57fae8d --- /dev/null +++ b/pkg/openai/similarity.go @@ -0,0 +1,91 @@ +package openai + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + + "github.com/xavidop/voiceflow-cli/internal/global" + "github.com/xavidop/voiceflow-cli/internal/types/tests" +) + +func OpenAICheckSimilarity(message string, s []string, similarityConfig tests.SimilarityConfig) (float64, error) { + if similarityConfig.Provider != "openai" { + return 0.0, fmt.Errorf("unsupported provider: %s", similarityConfig.Provider) + } + + // OpenAI API endpoint + apiURL := "https://api.openai.com/v1/chat/completions" + + // Prepare the prompt for similarity comparison + prompt := fmt.Sprintf( + "Compare the input message: \"%s\" to the following strings: %v. Calculate the average similarity score (0 to 1) across all comparisons. Return only the numeric score without any additional text or explanation.", + message, s, + ) + + // Create the request payload + payload := map[string]interface{}{ + "model": similarityConfig.Model, + "temperature": similarityConfig.Temperature, + "top_p": similarityConfig.TopP, + "messages": []map[string]string{ + {"role": "system", "content": "You are a helpful assistant that calculates similarity scores and returns only numeric results."}, + {"role": "user", "content": prompt}, + }, + } + + // Serialize the payload to JSON + jsonPayload, err := json.Marshal(payload) + if err != nil { + return 0.0, fmt.Errorf("failed to serialize payload: %w", err) + } + + // Make the HTTP POST request + req, err := http.NewRequest("POST", apiURL, bytes.NewBuffer(jsonPayload)) + if err != nil { + return 0.0, fmt.Errorf("failed to create request: %w", err) + } + req.Header.Set("Authorization", "Bearer "+global.OpenAIAPIKey) + req.Header.Set("Content-Type", "application/json") + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return 0.0, fmt.Errorf("request failed: %w", err) + } + defer resp.Body.Close() + + // Read and parse the response + body, err := io.ReadAll(resp.Body) + if err != nil { + return 0.0, fmt.Errorf("failed to read response: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return 0.0, fmt.Errorf("API returned status %d: %s", resp.StatusCode, string(body)) + } + + var response map[string]interface{} + if err := json.Unmarshal(body, &response); err != nil { + return 0.0, fmt.Errorf("failed to parse response: %w", err) + } + + // Extract similarity scores from the response + choices := response["choices"].([]interface{}) + if len(choices) == 0 { + return 0.0, fmt.Errorf("no choices returned in the response") + } + + // Assume the assistant returns a list of similarity scores in the response text + // Parse the similarity score from the response text + responseText := choices[0].(map[string]interface{})["message"].(map[string]interface{})["content"].(string) + var avgSimilarity float64 + if _, err := fmt.Sscanf(responseText, "%f", &avgSimilarity); err != nil { + return 0, fmt.Errorf("failed to parse similarity score: %w", err) + } + + return avgSimilarity, nil + +} diff --git a/pkg/test/runner.go b/pkg/test/runner.go index b38c9ab..0d63713 100644 --- a/pkg/test/runner.go +++ b/pkg/test/runner.go @@ -9,6 +9,7 @@ import ( "github.com/xavidop/voiceflow-cli/internal/global" "github.com/xavidop/voiceflow-cli/internal/types/tests" "github.com/xavidop/voiceflow-cli/internal/types/voiceflow/interact" + "github.com/xavidop/voiceflow-cli/pkg/openai" "github.com/xavidop/voiceflow-cli/pkg/voiceflow" ) @@ -41,7 +42,7 @@ func runTest(EnvironmentName, userID string, test tests.Test) error { if len(validations) == 0 { global.Log.Infof("All validations passed for Interaction ID: %s", interaction.ID) } else { - return fmt.Errorf("validation failed for Interaction ID: %s, not all validations were executed", validations) + return fmt.Errorf("validation failed for Interaction ID: %s, not all validations were executed: %v", interaction.ID, validations) } } // No errors, test passed @@ -95,11 +96,33 @@ func validateResponse(interactionResponse interact.InteractionResponse, validati continue } } + if validation.Type == "similarity" { + if checkSimilarity(message, validation.Values, *validation.SimilarityConfig) { + validations = removeById(validations, validation.ID) + continue + } + } } } return validations, nil } +func checkSimilarity(message string, stringsToEvaluate []string, similarityConfig tests.SimilarityConfig) bool { + switch similarityConfig.Provider { + case "openai": + similarity, err := openai.OpenAICheckSimilarity(message, stringsToEvaluate, similarityConfig) + if err != nil { + global.Log.Errorf("Error checking similarity: %s", err.Error()) + return false + } + return similarity >= similarityConfig.SimilarityThreshold + + default: + global.Log.Errorf("Unsupported provider: %s", similarityConfig.Provider) + return false + } +} + func removeById(slice []tests.Validation, ID string) []tests.Validation { for index, validation := range slice { if validation.ID == ID {