openai realtime python port

x/README.md

@@ -0,0 +1 @@
+This folder contains experimental clients for various LLM providers that may at some point be integrated into ell.

x/openai_realtime/.gitignore

@@ -0,0 +1 @@
+reference/

x/openai_realtime/README.md

@@ -0,0 +1,81 @@
+# OpenAI Realtime Python Client
+
+This is a Python port of the OpenAI Realtime Client, designed to interact with OpenAI's realtime API for advanced AI interactions.
+
+**Note:** This is a port of OpenAI's realtime client to Python by William Guss.
+
+## Features
+
+- Realtime communication with OpenAI's API
+- Support for text and audio modalities
+- Tool integration for extended functionality
+- Conversation management and event handling
+- Asynchronous operations for improved performance
+
+## Installation
+```bash
+git clone https://github.com/MadcowD/ell.git
+cd x/openai_realtime
+pip install -e .
+```
+## Quick Start
+```python
+from openai_realtime import RealtimeClient
+
+async def main():
+    client = RealtimeClient(api_key="your-api-key")
+    await client.connect()
+
+    # Send a text message
+    client.send_user_message_content([{"type": "text", "text": "Hello, AI!"}])
+
+    # Wait for the AI's response
+    response = await client.wait_for_next_completed_item()
+    print(response['item']['formatted']['text'])
+
+    client.disconnect()
+
+if __name__ == "__main__":
+    import asyncio
+    asyncio.run(main())
+```
+
+## Key Components
+- **RealtimeClient**: The main client for interacting with the OpenAI Realtime API.
+- **RealtimeAPI**: Handles the WebSocket connection and low-level communication.
+- **RealtimeConversation**: Manages the conversation state and message processing.
+- **RealtimeEventHandler**: Provides event handling capabilities for the client.
+- **RealtimeUtils**: Utility functions for data conversion and manipulation.
+
+## Advanced Usage
+### Adding Custom Tools
+```python3
+def my_tool_handler(args):
+    # Implement your tool logic here
+    return {"result": "Tool output"}
+
+client.add_tool(
+    {"name": "my_tool", "description": "A custom tool"},
+    my_tool_handler
+)
+```
+
+### Handling Audio
+```
+import numpy as np
+
+# Append audio data
+audio_data = np.array([...], dtype=np.int16)
+client.append_input_audio(audio_data)
+
+# Create a response (including audio if available)
+client.create_response()
+```
+## Documentation
+For more detailed documentation, please refer to the [API Reference](#).
+
+## Contributing
+Contributions are welcome! Please feel free to submit a Pull Request.
+
+## License
+This project is licensed under the MIT License - see the LICENSE file for details.

x/openai_realtime/pyproject.toml

@@ -0,0 +1,36 @@
+[tool.poetry]
+name = "openai-realtime"
+version = "0.0.1"
+description = "A real-time API client for OpenAI using websockets"
+authors = ["William Guss"]
+license = "MIT"
+readme = "README.md"
+packages = [{include = "openai_realtime", from = "src"}]
+
+[tool.poetry.dependencies]
+python = "^3.12"  # Updated to match your Python version
+websockets = "^10.4"
+aiohttp = "^3.8.4"
+asyncio = "^3.4.3"
+pydub = "^0.25.1"  # Note: This package has a deprecation warning for Python 3.13+
+
+[tool.poetry.dev-dependencies]
+pytest = "^7.3.0"
+pytest-asyncio = "^0.21.0"  # Added pytest-asyncio
+black = "^23.3.0"
+isort = "^5.12.0"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.black]
+line-length = 88
+target-version = ['py312']  # Updated to Python 3.12
+
+[tool.isort]
+profile = "black"
+line_length = 88
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"  # Added configuration for pytest-asyncio

x/openai_realtime/src/openai_realtime/__init__.py

@@ -0,0 +1,13 @@
+from .client import RealtimeClient
+from .api import RealtimeAPI
+from .conversation import RealtimeConversation
+from .event_handler import RealtimeEventHandler
+from .utils import RealtimeUtils
+
+__all__ = [
+    "RealtimeClient",
+    "RealtimeAPI",
+    "RealtimeConversation",
+    "RealtimeEventHandler",
+    "RealtimeUtils"
+]

x/openai_realtime/src/openai_realtime/api.py

@@ -0,0 +1,81 @@
+import asyncio
+import json
+import websockets
+from .event_handler import RealtimeEventHandler
+from .utils import RealtimeUtils
+
+class RealtimeAPI(RealtimeEventHandler):
+    def __init__(self, url=None, api_key=None, dangerously_allow_api_key_in_browser=False, debug=False):
+        super().__init__()
+        self.default_url = 'wss://api.openai.com/v1/realtime'
+        self.url = url or self.default_url
+        self.api_key = api_key
+        self.debug = debug
+        self.ws = None
+
+    def is_connected(self):
+        return self.ws is not None and self.ws.open
+
+    def log(self, *args):
+        if self.debug:
+            print(*args)
+        return True
+
+    async def connect(self, model='gpt-4o-realtime-preview-2024-10-01'):
+        if self.is_connected():
+            raise Exception("Already connected")
+
+        headers = {
+            'Authorization': f'Bearer {self.api_key}',
+            'OpenAI-Beta': 'realtime=v1'
+        }
+
+        self.ws = await websockets.connect(f"{self.url}?model={model}", extra_headers=headers)
+
+        self.log(f"Connected to {self.url}")
+
+        asyncio.create_task(self._message_handler())
+
+        return True
+
+    async def _message_handler(self):
+        try:
+            async for message in self.ws:
+                data = json.loads(message)
+                self.receive(data['type'], data)
+        except websockets.exceptions.ConnectionClosed:
+            self.disconnect()
+            self.dispatch('close', {'error': True})
+
+    def disconnect(self):
+        if self.ws:
+            asyncio.create_task(self.ws.close())
+            self.ws = None
+        return True
+
+    def receive(self, event_name, event):
+        self.log("received:", event_name, event)
+        self.dispatch(f"server.{event_name}", event)
+        self.dispatch("server.*", event)
+        return True
+
+    def send(self, event_name, data=None):
+        if not self.is_connected():
+            raise Exception("RealtimeAPI is not connected")
+
+        data = data or {}
+        if not isinstance(data, dict):
+            raise ValueError("data must be a dictionary")
+
+        event = {
+            "event_id": RealtimeUtils.generate_id("evt_"),
+            "type": event_name,
+            **data
+        }
+
+        self.dispatch(f"client.{event_name}", event)
+        self.dispatch("client.*", event)
+        self.log("sent:", event_name, event)
+
+        asyncio.create_task(self.ws.send(json.dumps(event)))
+        return True