diff --git a/CMakeLists.txt b/CMakeLists.txt index 4db8827..84ad50c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,6 +13,10 @@ if(NOT DEFINED ENV{OPENAI_API_KEY}) message(FATAL_ERROR "Env variable OPENAI_API_KEY must be set") endif() +if(DEFINED ENV{LOG_DATACHANNEL_MESSAGES}) + add_compile_definitions(LOG_DATACHANNEL_MESSAGES="1") +endif() + add_compile_definitions(OPENAI_API_KEY="$ENV{OPENAI_API_KEY}") add_compile_definitions(OPENAI_REALTIMEAPI="https://api.openai.com/v1/realtime?model=gpt-4o-mini-realtime-preview-2024-12-17") diff --git a/src/webrtc.cpp b/src/webrtc.cpp index e41b284..bc1782d 100644 --- a/src/webrtc.cpp +++ b/src/webrtc.cpp @@ -10,6 +10,9 @@ #include "main.h" #define TICK_INTERVAL 15 +#define GREETING \ + "{\"type\": \"response.create\", \"response\": {\"modalities\": " \ + "[\"audio\", \"text\"], \"instructions\": \"Say 'How can I help?.'\"}}" PeerConnection *peer_connection = NULL; @@ -25,6 +28,25 @@ void oai_send_audio_task(void *user_data) { } #endif +static void oai_ondatachannel_onmessage_task(char *msg, size_t len, + void *userdata, uint16_t sid) { +#ifdef LOG_DATACHANNEL_MESSAGES + ESP_LOGI(LOG_TAG, "DataChannel Message: %s", msg); +#endif +} + +static void oai_ondatachannel_onopen_task(void *userdata) { + if (peer_connection_create_datachannel(peer_connection, DATA_CHANNEL_RELIABLE, + 0, 0, (char *)"oai-events", + (char *)"") != -1) { + ESP_LOGI(LOG_TAG, "DataChannel created"); + peer_connection_datachannel_send(peer_connection, (char *)GREETING, + strlen(GREETING)); + } else { + ESP_LOGE(LOG_TAG, "Failed to create DataChannel"); + } +} + static void oai_onconnectionstatechange_task(PeerConnectionState state, void *user_data) { ESP_LOGI(LOG_TAG, "PeerConnectionState: %s", @@ -56,7 +78,7 @@ void oai_webrtc() { .ice_servers = {}, .audio_codec = CODEC_OPUS, .video_codec = CODEC_NONE, - .datachannel = DATA_CHANNEL_NONE, + .datachannel = DATA_CHANNEL_STRING, .onaudiotrack = [](uint8_t *data, size_t size, void *userdata) -> void { #ifndef LINUX_BUILD oai_audio_decode(data, size); @@ -78,6 +100,10 @@ void oai_webrtc() { peer_connection_oniceconnectionstatechange(peer_connection, oai_onconnectionstatechange_task); peer_connection_onicecandidate(peer_connection, oai_on_icecandidate_task); + peer_connection_ondatachannel(peer_connection, + oai_ondatachannel_onmessage_task, + oai_ondatachannel_onopen_task, NULL); + peer_connection_create_offer(peer_connection); while (1) {