add a note under the generate route to point to the lifecycle manager

nod-ai · Feb 19, 2025 · 3e5d591 · 3e5d591
1 parent 9b0f1e6
commit 3e5d591
Showing 1 changed file with 4 additions and 1 deletion.
diff --git a/shortfin/python/shortfin_apps/llm/routes/generate.py b/shortfin/python/shortfin_apps/llm/routes/generate.py
@@ -10,14 +10,17 @@
 
 from ..components.generate import ClientGenerateBatchProcess
 from ..components.io_struct import GenerateReqInput
+from ..components.service import GenerateService
 
 generation_router = APIRouter()
 
 
 @generation_router.post("/generate")
 @generation_router.put("/generate")
 async def generate_request(gen_req: GenerateReqInput, request: Request):
-    service = request.app.state.services["default"]
+    # app.state.services is populated by the ShortfinLlmLifecycleManager
+    # see shortfin/python/shortfin_apps/llm/components/lifecycle.py
+    service: GenerateService = request.app.state.services["default"]
     gen_req.post_init()
     responder = FastAPIResponder(request)
     ClientGenerateBatchProcess(service, gen_req, responder).launch()