From 6c83368dcda613b8a99757ee8cd425b27d08f563 Mon Sep 17 00:00:00 2001 From: Bill Welense Date: Wed, 26 Feb 2025 10:07:45 -0600 Subject: [PATCH] Separate initial setup from running the app --- max-serve-anythingllm/README.md | 71 +++++++++++++++---------- max-serve-anythingllm/main.py | 77 ++++++++++++---------------- max-serve-anythingllm/pyproject.toml | 9 +++- max-serve-anythingllm/setup.py | 39 ++++++++++++++ 4 files changed, 124 insertions(+), 72 deletions(-) create mode 100644 max-serve-anythingllm/setup.py diff --git a/max-serve-anythingllm/README.md b/max-serve-anythingllm/README.md index c28b991..e140523 100644 --- a/max-serve-anythingllm/README.md +++ b/max-serve-anythingllm/README.md @@ -6,6 +6,7 @@ In this recipe you will: - Use MAX Serve to provide an OpenAI-compatible endpoint for [DeepSeek R1](https://api-docs.deepseek.com/news/news250120) - Set up [AnythingLLM](https://github.com/Mintplex-Labs/anything-llm) to provide a robust chat interface +- Learn how to orchestrate multiple services in pure Python, without tools like Kubernetes or docker-compose ## About AnythingLLM @@ -35,7 +36,7 @@ A valid [Hugging Face token](https://huggingface.co/settings/tokens) ensures acc ### Docker -We’ll use Docker to run the AnythingLLM container. Follow the instructions in the [Docker documentation](https://docs.docker.com/desktop/) if you need to install it. +We'll use Docker to run the AnythingLLM container. Follow the instructions in the [Docker documentation](https://docs.docker.com/desktop/) if you need to install it. ## Get the code @@ -46,10 +47,10 @@ git clone https://github.com/modular/max-recipes.git cd max-recipes/max-serve-anythingllm ``` -Next, include your Hugging Face token in a `.env` file by running: +Next, include your Hugging Face token in a `.env.max` file by running: ```bash -echo "HUGGING_FACE_HUB_TOKEN=your_token_here" >> .env +echo "HUGGING_FACE_HUB_TOKEN=your_token_here" >> .env.max ``` ## Quick start: Run the app @@ -74,6 +75,8 @@ AnythingLLM is ready once you see a line like the following in the log output: Primary server in HTTP mode listening on port 3001 ``` +Note: the port that AnythingLLM reports can differ from the port you configure in `pyroject.toml` as the `UI_PORT`. + Once both servers are ready, launch AnythingLLM in your browser at [http://localhost:3001](http://localhost:3001) ## Using AnythingLLM @@ -84,33 +87,34 @@ The first time you [launch AnythingLLM in your browser](http://localhost:3001), 1. Select *Generic OpenAI* as the LLM provider, then enter: - Base URL = `http://host.docker.internal:3002/v1` - - API Key = `local` + - API Key = `local` (MAX doesn't require an API key, but this field can't be blank) - Chat Model Name = `deepseek-ai/DeepSeek-R1-Distill-Llama-8B` - - Token Context Window = `16384` + - Token Context Window = `16384` (Must match `MAX_CONTEXT_LENTH` from `pyproject.toml`) - Max Tokens = `1024` -2. Next, for User Setup, choose *Just me* or *My team*, and set an admin password +2. Next, for User Setup, choose *Just me* or *My team*, and set an admin password. 3. If asked to fill in a survey, you may participate or skip this step. (The survey data goes to the AnythingLLM project, not Modular.) -4. Finally, enter a workspace name +4. Finally, enter a workspace name. ## Understand the project -Let's explore how the key components of this project work together. +Let's explore how the key components of this recipe work together. ### Configuration with `pyproject.toml` -The project is configured in the `pyproject.toml` file, which defines: - -1. **Environment variables** to control the ports and storage location: +The recipe is configured in the `pyproject.toml` file, which defines: - ```bash - MAX_LLM_PORT = "3002" # Port for MAX Serve - UI_PORT = "3001" # Port for AnythingLLM - UI_STORAGE_LOCATION = "./data" # Persistent storage for AnythingLLM - UI_CONTAINER_NAME = "anythingllm-max" # Docker container name - ``` +1. **Environment variables** to control the ports, storage locations, and additional settings: + - `MAX_SECRETS_LOCATION = ".env.max"`: Location of file containing your Hugging Face token + - `MAX_CONTEXT_LENGTH = "16384"`: LLM context window size + - `MAX_BATCH_SIZE = "1"`: LLM batch size (use 1 when running on CPU) + - `MAX_LLM_PORT = "3002"`: Port for MAX Serve + - `UI_PORT = "3001"`: Port for AnythingLLM + - `UI_STORAGE_LOCATION = "./data"`: Persistent storage for AnythingLLM + - `UI_CONTAINER_NAME = "anythingllm-max"`: Name for referencing the container with Docker 2. **Tasks** you can run with the `magic run` command: - `app`: Runs the main Python script that coordinates both services + - `setup`: Sets up persistent storage for AnythingLLM - `ui`: Launches the AnythingLLM Docker container - `llm`: Starts MAX Serve with DeepSeek R1 - `clean`: Cleans up network resources for both services @@ -120,22 +124,37 @@ The project is configured in the `pyproject.toml` file, which defines: - AnythingLLM runs in a Docker container, keeping its dependencies isolated - Additional dependencies to orchestrate both services +### Setup with `setup.py` + +The `setup.py` script handles the initial setup for AnythingLLM: + +- Reads the `UI_STORAGE_LOCATION` from `pyproject.toml` +- Creates the storage directory if it doesn't exist +- Ensures an empty `.env` file is present for AnythingLLM settings +- This script is automatically run as a pre-task when you execute `magic run app` + ### Orchestration with `main.py` When you run `magic run app`, the `main.py` script coordinates everything necessary to start and shutdown both services: -1. **initial_setup()** - - Reads the configuration from `pyproject.toml` - - Creates the persistent storage location for AnythingLLM if it doesn't exist - - Ensures an empty `.env` file is present for AnythingLLM settings +1. **Command-line Interface** + - Uses [Click](https://click.palletsprojects.com/en/stable/) to provide a simple CLI + - Supports main tasks to run concurently, pre-tasks, and post-tasks + - Default command from pyproject.toml runs: `python main.py llm ui --pre setup --post clean` -2. **run_tasks()** - - Uses Honcho to run multiple `magic run` tasks concurrently +2. **run_app()** + - Loads environment variables, including those from the configured secrets location + - Uses [Honcho](https://honcho.readthedocs.io/en/latest/) to run multiple `magic run` tasks concurrently - Starts both the MAX Serve LLM backend and AnythingLLM UI -3. **cleanup()** - - `cleanup()` fnction is registered with `atexit` to ensure it runs when the script exits - - Runs the `clean` task to terminate all processes and remove containers +3. **run_task()** + - Executes individual tasks like `setup` or `clean` + - Reads task definitions from `pyproject.toml` + - Pre-tasks run before main tasks, post-tasks are registered with `atexit` to ensure cleanup + +4. **Cleanup Process** + - Post-tasks (like `clean`) are automatically run when the application exits + - The `clean` task terminates all processes and removes Docker containers ## What's next? diff --git a/max-serve-anythingllm/main.py b/max-serve-anythingllm/main.py index 86974d4..c79ed67 100644 --- a/max-serve-anythingllm/main.py +++ b/max-serve-anythingllm/main.py @@ -4,24 +4,46 @@ import atexit import os import tomli +import click from dotenv import load_dotenv -TASKS = [ "llm", "ui" ] +@click.command() +@click.argument("tasks", nargs=-1) +@click.option("--pre", multiple=True, help="Tasks to run before app tasks (can be used multiple times)") +@click.option("--post", multiple=True, help="Tasks to run after app tasks complete (can be used multiple times)") +def main(tasks, pre, post): + """Run Magic tasks. One or more task names must be specified.""" + if not tasks: + click.echo("Error: At least one task must be specified.", err=True) + click.echo("Example: python main.py llm ui --pre setup --post clean", err=True) + sys.exit(1) + + # Run pre-tasks + for task in pre: + run_task(task) + + # Register post-tasks to run at exit + for task in post: + atexit.register(run_task, task) + + # Run app tasks + run_app(tasks) -def run_tasks(): +def run_app(tasks: list[str]): """ - Runs the tasks specified in the TASKS list. This includes loading environment + Runs the tasks specified in the tasks list. This includes loading environment variables, setting up the task manager, and starting the tasks. """ try: - load_dotenv(".env.max") + if secrets_location := os.getenv("MAX_SECRETS_LOCATION"): + load_dotenv(secrets_location) env = os.environ.copy() manager = honcho.manager.Manager() - for task in TASKS: + for task in tasks: manager.add_process(task, f"magic run {task}", env=env) manager.loop() @@ -34,50 +56,17 @@ def run_tasks(): sys.exit(1) -def initial_setup(): - """ - Initializes persistent storage for AnythingLLM. Reads storage location from - the pyproject.toml file. If the directory and/or .env file don't already exist, - it creates the directory and ensures an empty .env file is present within it. - """ - - with open("pyproject.toml", "rb") as f: - pyproject_data = tomli.load(f) - data_dir = ( - pyproject_data.get("tool", {}) - .get("pixi", {}) - .get("activation", {}) - .get("env", {}) - .get("UI_STORAGE_LOCATION") - ) - if data_dir is None: - raise ValueError("UI_STORAGE_LOCATION not found in pyproject.toml") - - if not os.path.exists(data_dir): - os.makedirs(data_dir) - print(f"Created directory: {data_dir}") - - env_file = os.path.join(data_dir, ".env") - if not os.path.exists(env_file): - open(env_file, "w").close() # Create empty file - print(f"Created empty file: {env_file}") - - -def cleanup(): - """Checks if the `clean` task exists in pyproject.toml and runs it.""" +def run_task(task: str): + """Runs a Magic task.""" with open("pyproject.toml", "rb") as f: pyproject_data = tomli.load(f) tasks = pyproject_data.get("tool", {}).get("pixi", {}).get("tasks", {}) - if "clean" in tasks: - print("Running cleanup task...") - subprocess.run(["magic", "run", "clean"]) - else: - print("Cleanup task not found in pyproject.toml") + if task in tasks: + print(f"Running {task} task...") + subprocess.run(["magic", "run", task]) if __name__ == "__main__": - atexit.register(cleanup) - initial_setup() - run_tasks() + main() diff --git a/max-serve-anythingllm/pyproject.toml b/max-serve-anythingllm/pyproject.toml index 9ea60cc..91a91bb 100644 --- a/max-serve-anythingllm/pyproject.toml +++ b/max-serve-anythingllm/pyproject.toml @@ -25,15 +25,19 @@ platforms = ["linux-64", "linux-aarch64", "osx-arm64"] max_serve_anythingllm = { path = ".", editable = true } [tool.pixi.activation.env] +MAX_SECRETS_LOCATION = ".env.max" +MAX_CONTEXT_LENGTH = "16384" +MAX_BATCH_SIZE = "1" MAX_LLM_PORT = "3002" UI_PORT = "3001" UI_STORAGE_LOCATION = "./data" UI_CONTAINER_NAME = "anythingllm-max" [tool.pixi.tasks] -app = "python main.py" +app = "python main.py llm ui --pre setup --post clean" +setup = "python setup.py" +llm = "export MAX_SERVE_PORT=$MAX_LLM_PORT && max-pipelines serve --max-length=$MAX_CONTEXT_LENGTH --max-batch-size=$MAX_BATCH_SIZE --model-path=deepseek-ai/DeepSeek-R1-Distill-Llama-8B --weight-path=lmstudio-community/DeepSeek-R1-Distill-Llama-8B-GGUF/DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf" ui = "docker run -p $UI_PORT:3001 --name $UI_CONTAINER_NAME --cap-add SYS_ADMIN -v $UI_STORAGE_LOCATION:/app/server/storage -v $UI_STORAGE_LOCATION/.env:/app/server/.env -e STORAGE_DIR=\"/app/server/storage\" mintplexlabs/anythingllm" -llm = "export MAX_SERVE_PORT=$MAX_LLM_PORT && max-pipelines serve --max-length=16384 --max-batch-size=1 --model-path=deepseek-ai/DeepSeek-R1-Distill-Llama-8B --weight-path=lmstudio-community/DeepSeek-R1-Distill-Llama-8B-GGUF/DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf" clean = "pkill -f \"max-pipelines serve\" || true && lsof -ti:$MAX_LLM_PORT,$UI_PORT | xargs -r kill -9 2>/dev/null || true && docker rm -f $UI_CONTAINER_NAME 2>/dev/null || true" [tool.pixi.dependencies] @@ -41,3 +45,4 @@ max-pipelines = ">=25.2.0.dev2025022405,<26" honcho = ">=2.0.0,<3" tomli = ">=2.2.1,<3" python-dotenv = ">=1.0.1,<2" +click = ">=8.1.8,<9" diff --git a/max-serve-anythingllm/setup.py b/max-serve-anythingllm/setup.py new file mode 100644 index 0000000..180c6f7 --- /dev/null +++ b/max-serve-anythingllm/setup.py @@ -0,0 +1,39 @@ +import os +import tomli + + +def setup_anythingllm_storage(): + """ + Initializes persistent storage for AnythingLLM. Reads storage location from + the pyproject.toml file. If the required directory and/or .env file don't already + exist, it creates the directory and ensures an empty .env file is present within it. + """ + + with open("pyproject.toml", "rb") as f: + pyproject_data = tomli.load(f) + data_dir = ( + pyproject_data.get("tool", {}) + .get("pixi", {}) + .get("activation", {}) + .get("env", {}) + .get("UI_STORAGE_LOCATION") + ) + if data_dir is None: + raise ValueError("UI_STORAGE_LOCATION not found in pyproject.toml") + + if not os.path.exists(data_dir): + os.makedirs(data_dir) + print(f"Created directory: {data_dir}") + else: + print(f"Directory already exists: {data_dir}") + + env_file = os.path.join(data_dir, ".env") + if not os.path.exists(env_file): + open(env_file, "w").close() # Create empty file + print(f"Created empty file: {env_file}") + else: + print(f"File already exists: {env_file}") + + +if __name__ == "__main__": + setup_anythingllm_storage() \ No newline at end of file