From e7e7b3d105c1b4bb517bcf1efeab3ac5e83283f3 Mon Sep 17 00:00:00 2001 From: William Guss Date: Sat, 12 Oct 2024 16:58:53 -0700 Subject: [PATCH] v0.0.14 --- examples/bedrock_hello.py | 57 ----------------------------- examples/future/limbo.py | 4 +-- examples/openai_audio.py | 19 +++++----- examples/openai_prompt_caching.py | 3 ++ examples/providers/xai_ex.py | 1 + pyproject.toml | 3 +- src/ell/models/xai.py | 7 +++- tests/.exampleignore | 9 ++++- tests/run_all_examples.py | 60 +++++++++++++++++++++---------- 9 files changed, 72 insertions(+), 91 deletions(-) delete mode 100644 examples/bedrock_hello.py diff --git a/examples/bedrock_hello.py b/examples/bedrock_hello.py deleted file mode 100644 index b2608237d..000000000 --- a/examples/bedrock_hello.py +++ /dev/null @@ -1,57 +0,0 @@ -import ell -from PIL import Image -from ell.providers.bedrock import BedrockProvider -from pydantic import BaseModel, Field -import anthropic -from typing import List -import ell -from ell.types import Message -import boto3 - -@ell.simple(model="anthropic.claude-3-haiku-20240307-v1:0", api_params={"stream":True}, client=boto3.client('bedrock-runtime', region_name='us-east-1')) -def hello_from_bedrock_streaming(): - """You are an AI assistant. Your task is to respond to the user's message with a friendly greeting.""" - return "Say hello to the world!!!" - -@ell.simple(model="anthropic.claude-3-haiku-20240307-v1:0", client=boto3.client('bedrock-runtime', region_name='us-east-1')) -def hello_from_bedrock(): - """You are an AI assistant. Your task is to respond to the user's message with a friendly greeting.""" - return "Say hello to the world!!!" - -@ell.simple(model="anthropic.claude-3-haiku-20240307-v1:0", client=boto3.client('bedrock-runtime', region_name='us-east-1')) -def describe_activity(image: Image.Image): - return [ - ell.system("You are VisionGPT. Answer <5 words all lower case."), - ell.user(["Describe what the person in the image is doing:", image]) - ] - - -@ell.tool() -def approve_claim(claim_id : str): - """Approve a claim""" - return f"approved {claim_id}" - -@ell.complex(model="anthropic.claude-3-haiku-20240307-v1:0", tools=[approve_claim], client=boto3.client('bedrock-runtime', region_name='us-east-1')) -def insurance_claim_chatbot(message_history: str) -> str: - """You are an insurance claim approver. You can use a tool to approve a claim if an id is given to you.""" - return message_history - - -if __name__ == "__main__": - ell.init(verbose=True, store="./logdir", autocommit=True) - hello_from_bedrock_streaming() - hello_from_bedrock() - # Capture an image from the webcam - describe_activity(Image.open(r"./examples/future/catmeme.jpg")) # "they are holding a book" - - message_history = [] - - # Run through messages automatically! - user_messages = "Please approve the claim 12344534" - - response_message = insurance_claim_chatbot(user_messages) - print(response_message) - if response_message.tool_calls: - tool_results = response_message.call_tools_and_collect_as_message() - print(tool_results) - print('THE END') \ No newline at end of file diff --git a/examples/future/limbo.py b/examples/future/limbo.py index 06a76ab1a..9e6e42e86 100644 --- a/examples/future/limbo.py +++ b/examples/future/limbo.py @@ -7,8 +7,8 @@ ell.init(verbose=True, store='./logdir', autocommit=True) -@ell.tool(autogenerate=True) -def order_t_shirt(size, color, address): +@ell.tool() +def order_t_shirt(size : str, color : str, address : str): # ....\ pass diff --git a/examples/openai_audio.py b/examples/openai_audio.py index 469775c5f..117b1bbe1 100644 --- a/examples/openai_audio.py +++ b/examples/openai_audio.py @@ -1,15 +1,14 @@ -import ell +# import ell -ell.init(verbose=True) +# # ell.init(verbose=True) -@ell.complex("gpt-4o-audio-preview") -def test(): - return [ell.user("Hey! Could you talk to me in spanish? I'd like to hear how you say 'ell'.")] +# # @ell.complex("gpt-4o-audio-preview") +# # def test(): +# # return [ell.user("Hey! Could you talk to me in spanish? I'd like to hear how you say 'ell'.")] -response = test() -print(response.audios[0]) - -if __name__ == "__main__": - test() +if __name__ == "__main__": +# # response = test() +# # print(response.audios[0]) + pass diff --git a/examples/openai_prompt_caching.py b/examples/openai_prompt_caching.py index 170fda831..b7bf662ec 100644 --- a/examples/openai_prompt_caching.py +++ b/examples/openai_prompt_caching.py @@ -13,6 +13,9 @@ def cached_chat(history : List[str], new_message : str) -> str: +if __name__ == "__main__": + pass + diff --git a/examples/providers/xai_ex.py b/examples/providers/xai_ex.py index aa9e7e09e..bee67cc93 100644 --- a/examples/providers/xai_ex.py +++ b/examples/providers/xai_ex.py @@ -5,6 +5,7 @@ ell.init(verbose=True) # Models are automatically registered, so we can use them without specifying the client +# set XAI_API_KEY=your_api_key in your environment to run this example @ell.simple(model='grok-2-mini') def use_default_xai_client(prompt: str) -> str: return prompt diff --git a/pyproject.toml b/pyproject.toml index 762fe02b4..48e14a495 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "ell-ai" -version = "0.0.13" +version = "0.0.14" description = "ell - the language model programming library" authors = ["William Guss "] license = "MIT" @@ -41,6 +41,7 @@ requests = "^2.32.3" typing-extensions = "^4.12.2" black = "^24.8.0" pillow = "^10.4.0" +psutil = "^5.9.0" [tool.poetry.group.dev.dependencies] pytest = "^8.3.2" diff --git a/src/ell/models/xai.py b/src/ell/models/xai.py index dad543791..6f3d891a4 100644 --- a/src/ell/models/xai.py +++ b/src/ell/models/xai.py @@ -56,9 +56,14 @@ def register(client: openai.Client): for model_id in standard_models: config.register_model(model_id, client) + default_client = None try: - default_client = openai.Client(base_url="https://api.x.ai/v1", api_key=os.environ.get("XAI_API_KEY")) + + xai_api_key = os.environ.get("XAI_API_KEY") + if not xai_api_key: + raise openai.OpenAIError("XAI_API_KEY not found in environment variables") + default_client = openai.Client(base_url="https://api.x.ai/v1", api_key=xai_api_key) except openai.OpenAIError as e: pass diff --git a/tests/.exampleignore b/tests/.exampleignore index 10d790d44..f77107cbb 100644 --- a/tests/.exampleignore +++ b/tests/.exampleignore @@ -6,4 +6,11 @@ webcam_activity_describer.py openaigym.py o1.py chord* -server_example.py \ No newline at end of file +server_example.py +limbo.py +providers/bedrock_ex.py +azure_ex.py +openrouter_ex.py +vllm_ex.py +*_ex.py +bedrock_hello.py \ No newline at end of file diff --git a/tests/run_all_examples.py b/tests/run_all_examples.py index 200659f89..b87e03314 100644 --- a/tests/run_all_examples.py +++ b/tests/run_all_examples.py @@ -2,14 +2,15 @@ import subprocess import sys import time -from concurrent.futures import ThreadPoolExecutor, as_completed +from concurrent.futures import ThreadPoolExecutor, as_completed, TimeoutError import logging -from colorama import Fore, Style, init +from colorama import Fore, Style, init # type: ignore import argparse import json import hashlib import fnmatch import threading +import psutil # Initialize colorama for cross-platform colored output init() @@ -63,7 +64,6 @@ def run_example(example_path, verbose=False, cache=None): # Prepare simulated input based on the example file simulated_input = get_simulated_input(filename) - # Run the example with simulated input process = subprocess.Popen( [sys.executable, example_path], stdin=subprocess.PIPE, @@ -71,12 +71,28 @@ def run_example(example_path, verbose=False, cache=None): stderr=subprocess.PIPE, text=True ) - - stdout, stderr = process.communicate(input=simulated_input) - + + def kill_proc_tree(pid): + parent = psutil.Process(pid) + for child in parent.children(recursive=True): + child.kill() + parent.kill() + + try: + stdout, stderr = process.communicate(input=simulated_input, timeout=60) + except subprocess.TimeoutExpired: + kill_proc_tree(process.pid) + return filename, "TIMEOUT", 60, "Example execution timed out after 60 seconds", "" + end_time = time.time() runtime = end_time - start_time + if process.returncode != 0: + error_message = f"Process exited with non-zero status: {process.returncode}\nStderr: {stderr}" + if cache is not None: + update_cache(cache, file_hash, "ERROR", runtime) + return filename, "ERROR", runtime, error_message, stdout + if cache is not None: update_cache(cache, file_hash, "SUCCESS", runtime) return filename, "SUCCESS", runtime, None, stdout @@ -146,18 +162,21 @@ def run_all_examples(args): print(f"{Fore.GREEN}{futures[future]} . (Runtime: {runtime:.2f}s){Style.RESET_ALL}") elif status == "CACHED": print(f"{Fore.BLUE}{futures[future]} C (Cached Runtime: {runtime:.2f}s){Style.RESET_ALL}") + elif status == "TIMEOUT": + print(f"{Fore.YELLOW}{futures[future]} T (Timeout: {runtime:.2f}s){Style.RESET_ALL}") + print(f" Error: {error}") else: print(f"{Fore.RED}{futures[future]} F (Runtime: {runtime:.2f}s){Style.RESET_ALL}") print(f" Error: {error}") print(f" Full output:") print(output) - if not args.continue_on_error: - print(f"\n{Fore.RED}Stopping execution due to failure.{Style.RESET_ALL}") - for running_future in futures: - if not running_future.done(): - print(f"{Fore.YELLOW}Cancelling: {futures[running_future]}{Style.RESET_ALL}") - executor.shutdown(wait=False, cancel_futures=True) - break + if status in ["ERROR", "TIMEOUT"] and not args.continue_on_error: + print(f"\n{Fore.RED}Stopping execution due to failure.{Style.RESET_ALL}") + for running_future in futures: + if not running_future.done(): + print(f"{Fore.YELLOW}Cancelling: {futures[running_future]}{Style.RESET_ALL}") + executor.shutdown(wait=False, cancel_futures=True) + break if args.cache: save_cache(cache) @@ -166,18 +185,21 @@ def run_all_examples(args): total_examples = len(results) successful = sum(1 for _, status, _, _ in results if status in {"SUCCESS", "CACHED"}) failed = sum(1 for _, status, _, _ in results if status == "ERROR") - skipped = total_examples - successful - failed + timed_out = sum(1 for _, status, _, _ in results if status == "TIMEOUT") + skipped = total_examples - successful - failed - timed_out print(f"Total examples: {total_examples}") print(f"{Fore.GREEN}Successful: {successful}{Style.RESET_ALL}") print(f"{Fore.RED}Failed: {failed}{Style.RESET_ALL}") + print(f"{Fore.YELLOW}Timed out: {timed_out}{Style.RESET_ALL}") print(f"{Fore.YELLOW}Skipped: {skipped}{Style.RESET_ALL}") - if failed > 0: - print("\nFailed examples:") + if failed > 0 or timed_out > 0: + print("\nFailed or timed out examples:") for example, status, runtime, error in results: - if status == "ERROR": - print(f"{Fore.RED}{example} (Runtime: {runtime:.2f}s){Style.RESET_ALL}") + if status in ["ERROR", "TIMEOUT"]: + color = Fore.RED if status == "ERROR" else Fore.YELLOW + print(f"{color}{example} ({status}, Runtime: {runtime:.2f}s){Style.RESET_ALL}") print(f" Error: {error}") average_runtime = sum(runtime for _, _, runtime, _ in results) / len(results) @@ -186,7 +208,7 @@ def run_all_examples(args): if all(status in {"SUCCESS", "CACHED"} for _, status, _, _ in results): print(f"\n{Fore.GREEN}All examples were successful.{Style.RESET_ALL}") else: - print(f"\n{Fore.YELLOW}Some examples did not run successfully. Please review the output above.{Style.RESET_ALL}") + print(f"\n{Fore.YELLOW}Some examples did not run successfully or timed out. Please review the output above.{Style.RESET_ALL}") if __name__ == "__main__": args = parse_arguments()