From 3e5d59134b758b383f2f4c8277826d05aab81402 Mon Sep 17 00:00:00 2001 From: Cedar Date: Wed, 19 Feb 2025 12:42:07 -0800 Subject: [PATCH] add a note under the generate route to point to the lifecycle manager --- shortfin/python/shortfin_apps/llm/routes/generate.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/shortfin/python/shortfin_apps/llm/routes/generate.py b/shortfin/python/shortfin_apps/llm/routes/generate.py index 664fb78d4..1e16002d3 100644 --- a/shortfin/python/shortfin_apps/llm/routes/generate.py +++ b/shortfin/python/shortfin_apps/llm/routes/generate.py @@ -10,6 +10,7 @@ from ..components.generate import ClientGenerateBatchProcess from ..components.io_struct import GenerateReqInput +from ..components.service import GenerateService generation_router = APIRouter() @@ -17,7 +18,9 @@ @generation_router.post("/generate") @generation_router.put("/generate") async def generate_request(gen_req: GenerateReqInput, request: Request): - service = request.app.state.services["default"] + # app.state.services is populated by the ShortfinLlmLifecycleManager + # see shortfin/python/shortfin_apps/llm/components/lifecycle.py + service: GenerateService = request.app.state.services["default"] gen_req.post_init() responder = FastAPIResponder(request) ClientGenerateBatchProcess(service, gen_req, responder).launch()