Build a Python microservice for image recognition with TensorFlow Lite. Keep the model…
- User
Build a Python microservice for image recognition with TensorFlow Lite. Keep the model in memory as a singleton and implement batching middleware that collects incoming requests for 50ms and feeds them to the GPU/CPU as a single tensor batch.
- Assistant
Designing Python microservice with batching
The design details a a FastAPI microservice using a singleton TensorFlow Lite interpreter with optional GPU delegate, an asyncio-based batcher aggregating requests every 50ms for batched inference, and returning JSON predictions. The plan includes thread-safe inference on one background task, dynamic input resizing, image preprocessing with Pillow, and concurrency handling via asyncio futures. Below is a minimal FastAPI microservice that:
- Loads a TensorFlow Lite model once (singleton).
- Uses an async batching layer that accumulates requests for 50 ms and runs them as a single batch.
- Returns the model outputs (e.g., class probabilities) per request.
Requirements:
fastapi,uvicorn,Pillow,numpy,tensorflowortflite-runtime
Optional (GPU delegate):tflite_runtimewith GPU delegate installed.# app.py import asyncio import io import time from typing import List, Tuple import numpy as np from fastapi import FastAPI, File, UploadFile from pydantic import BaseModel from PIL import Image try: # Use tflite_runtime if installed (faster, smaller), else fallback to TF import tflite_runtime.interpreter as tflite except ImportError: # pragma: no cover import tensorflow as tf tflite = tf.lite app = FastAPI() # --------------------------- # Model Singleton # --------------------------- class ModelSingleton: _instance = None def __new__(cls, model_path: str, use_gpu: bool = False): if cls._instance is None: cls._instance = super(ModelSingleton, cls).__new__(cls) cls._instance._load_model(model_path, use_gpu) return cls._instance def _load_model(self, model_path: str, use_gpu: bool): delegates = [] if use_gpu: try: # GPU delegate (if available in tflite_runtime) from tflite_runtime.interpreter import load_delegate delegates.append(load_delegate("libedgetpu.so.1")) except Exception: try: delegates.append( tflite.load_delegate( "libtensorflowlite_gpu_delegate.so", {"allow_precision_loss": True}, ) ) except Exception: print("GPU delegate not available, falling back to CPU.") self.interpreter = tflite.Interpreter( model_path=model_path, experimental_delegates=delegates if delegates else None, ) self.interpreter.allocate_tensors() self.input_details = self.interpreter.get_input_details() self.output_details = self.interpreter.get_output_details() # Assume a single input of shape [1, H, W, C] _, self.in_h, self.in_w, self.in_c = self.input_details[0]["shape"] # For variable batch size, we'll dynamically resize per batch def predict_batch(self, batch_array: np.ndarray) -> np.ndarray: """ batch_array: numpy array of shape [B, H, W, C] returns: numpy array of shape [B, ...] as model output """ # Dynamically resize input tensor to match batch size self.interpreter.resize_tensor_input( self.input_details[0]["index"], batch_array.shape, strict=False ) self.interpreter.allocate_tensors() self.interpreter.set_tensor(self.input_details[0]["index"], batch_array) self.interpreter.invoke() outputs = [] for out_detail in self.output_details: outputs.append(self.interpreter.get_tensor(out_detail["index"])) # If single output, return [B, ...], else list of outputs return outputs[0] if len(outputs) == 1 else outputs # --------------------------- # Request/Response Models # --------------------------- class PredictionResponse(BaseModel): probabilities: List[float] # --------------------------- # Batching Middleware # --------------------------- class Batcher: def __init__(self, model_path: str, batch_window_ms: int = 50, use_gpu: bool = False): self.model = ModelSingleton(model_path, use_gpu) self.batch_window = batch_window_ms / 1000.0 self.queue: asyncio.Queue[ Tuple[np.ndarray, asyncio.Future] ] = asyncio.Queue() self._task = asyncio.create_task(self._batch_loop()) async def enqueue(self, img: np.ndarray) -> np.ndarray: loop = asyncio.get_event_loop() future: asyncio.Future = loop.create_future() await self.queue.put((img, future)) return await future async def _batch_loop(self): while True: start = time.perf_counter() # At least one item required to proceed img, fut = await self.queue.get() imgs = [img] futures = [fut] # Collect more within the time window while True: elapsed = time.perf_counter() - start if elapsed >= self.batch_window: break try: nxt_img, nxt_fut = self.queue.get_nowait() imgs.append(nxt_img) futures.append(nxt_fut) except asyncio.QueueEmpty: await asyncio.sleep(self.batch_window - elapsed) break batch_array = np.stack(imgs, axis=0) # Run inference (CPU/GPU) try: preds = self.model.predict_batch(batch_array) # preds shape [B, num_classes] (typical for classifiers) for i, fut_ in enumerate(futures): if not fut_.done(): fut_.set_result(preds[i].tolist()) except Exception as e: for fut_ in futures: if not fut_.done(): fut_.set_exception(e) # --------------------------- # Utils # --------------------------- def preprocess_image(file_bytes: bytes, target_size: Tuple[int, int]) -> np.ndarray: """ Converts raw image bytes to a normalized float32 tensor of shape [H, W, C]. Adjust normalization to match your model's requirements. """ image = Image.open(io.BytesIO(file_bytes)).convert("RGB") image = image.resize(target_size) arr = np.asarray(image).astype(np.float32) # Example normalization: 0..255 -> 0..1 arr = arr / 255.0 return arr # --------------------------- # Instantiate batching with model # --------------------------- # Change "model.tflite" to your actual model path batcher = Batcher(model_path="model.tflite", batch_window_ms=50, use_gpu=False) # --------------------------- # API Routes # --------------------------- @app.post("/predict", response_model=PredictionResponse) async def predict(file: UploadFile = File(...)): raw = await file.read() # preprocess to model input img = preprocess_image(raw, (batcher.model.in_w, batcher.model.in_h)) # enqueue to batcher probs = await batcher.enqueue(img) return {"probabilities": probs} @app.get("/health") def health(): return {"status": "ok"} # --------------------------- # Entrypoint # --------------------------- # Run: uvicorn app:app --host 0.0.0.0 --port 8000 if __name__ == "__main__": # pragma: no cover import uvicorn uvicorn.run("app:app", host="0.0.0.0", port=8000, reload=False)Notes & Customization
- Model Input/Output:
- The sample assumes a single RGB input (H, W, C) and outputs class probabilities. Adjust
preprocess_imageand the response shape for your model.
- The sample assumes a single RGB input (H, W, C) and outputs class probabilities. Adjust
- GPU/CPU:
- Set
use_gpu=TrueinBatcherif you have a GPU delegate available. You may need to provide the correct delegate library name for your environment (e.g.,libtensorflowlite_gpu_delegate.so).
- Set
- Batch Size & Window:
- The batch collects requests for up to 50 ms. Adjust
batch_window_msas desired. You can add a maximum batch size to prevent very large batches.
- The batch collects requests for up to 50 ms. Adjust
- Thread Safety:
- TFLite interpreters are not thread-safe. The batching loop runs in a single asyncio task, ensuring only one inference happens at a time.