Skip to content
TheChatArchiveThe Chat Archive
Grok 4.1 Fast· xaiHugging Face Frontier Reasoning Traces

Build a tenancy-aware cache: keys, invalidation, and fairness.

  1. User

    Build a tenancy-aware cache: keys, invalidation, and fairness.

  2. Assistant
    import threading
    from collections import OrderedDict
    from typing import Optional, Dict, Any
    
    
    class TenancyAwareCache:
        """
        A thread-safe, tenancy-aware LRU cache with the following features:
        
        - **Keys**: Namespaced by tenant_id. Use `get(tenant_id, key)` and `set(tenant_id, key, value)`.
          Keys within a tenant are isolated.
        
        - **Invalidation**:
          - `invalidate_key(tenant_id, key)`: Removes a specific key for a tenant.
          - `invalidate_tenant(tenant_id)`: Clears all keys for a tenant.
          - `invalidate_all()`: Clears everything.
        
        - **Fairness**:
          - Per-tenant LRU eviction (max_entries_per_tenant).
          - Tenant-level LRU eviction when max_tenants is reached (evicts least recently used tenants).
          - Tenants are "touched" (promoted in LRU) on cache hits (`get`) and sets.
          - Lazy tenant creation (only on `set`).
        
        Example usage:
            cache = TenancyAwareCache(max_tenants=10, max_entries_per_tenant=1000)
            cache.set("tenant1", "user:123", {"name": "Alice"})
            value = cache.get("tenant1", "user:123")
            cache.invalidate_tenant("tenant1")
        """
    
        def __init__(self, max_tenants: Optional[int] = 100, max_entries_per_tenant: int = 1000):
            self.caches: Dict[str, OrderedDict] = {}  # tenant_id -> LRU cache (OrderedDict)
            self.tenant_order: OrderedDict[str, None] = OrderedDict()  # LRU order for tenants
            self.max_tenants = max_tenants
            self.max_entries_per_tenant = max_entries_per_tenant
            self._lock = threading.RLock()  # Reentrant lock for thread-safety
    
        def get(self, tenant_id: str, key: str) -> Optional[Any]:
            """Retrieve a value for a tenant-specific key. Touches tenant on hit."""
            with self._lock:
                if tenant_id not in self.caches:
                    return None
                cache = self.caches[tenant_id]
                if key in cache:
                    # LRU: move key and tenant to most recent
                    cache.move_to_end(key)
                    self.tenant_order.move_to_end(tenant_id)
                    return cache[key]
                return None
    
        def set(self, tenant_id: str, key: str, value: Any) -> None:
            """Set a value for a tenant-specific key. Creates tenant if needed, evicts if quotas exceeded.
            Touches tenant LRU."""
            with self._lock:
                # Lazy-create tenant if needed, evict LRU tenant if at quota
                if tenant_id not in self.caches:
                    if self.max_tenants and len(self.caches) >= self.max_tenants:
                        # Evict least recently used tenant
                        lru_tenant, _ = self.tenant_order.popitem(last=False)
                        del self.caches[lru_tenant]
                    self.caches[tenant_id] = OrderedDict()
                    self.tenant_order[tenant_id] = None
                cache = self.caches[tenant_id]
                # Set and touch LRU
                cache[key] = value
                cache.move_to_end(key)
                self.tenant_order.move_to_end(tenant_id)
                # Per-tenant eviction if exceeded
                if len(cache) > self.max_entries_per_tenant:
                    cache.popitem(last=False)
    
        def invalidate_key(self, tenant_id: str, key: str) -> None:
            """Remove a specific key from a tenant's cache. Removes empty tenants."""
            with self._lock:
                if tenant_id in self.caches:
                    self.caches[tenant_id].pop(key, None)
                    self._cleanup_empty_tenant(tenant_id)
    
        def invalidate_tenant(self, tenant_id: str) -> None:
            """Clear all keys for a tenant."""
            with self._lock:
                self.caches.pop(tenant_id, None)
                self.tenant_order.pop(tenant_id, None)
    
        def invalidate_all(self) -> None:
            """Clear all tenants and caches."""
            with self._lock:
                self.caches.clear()
                self.tenant_order.clear()
    
        def _cleanup_empty_tenant(self, tenant_id: str) -> None:
            """Helper: Remove tenant if its cache is empty."""
            if tenant_id in self.caches and len(self.caches[tenant_id]) == 0:
                del self.caches[tenant_id]
                self.tenant_order.pop(tenant_id, None)
    
        def get_stats(self) -> Dict[str, int]:
            """Get cache sizes per tenant."""
            with self._lock:
                return {tenant: len(cache) for tenant, cache in self.caches.items()}
    
        def get_total_size(self) -> int:
            """Total number of entries across all tenants."""
            with self._lock:
                return sum(len(cache) for cache in self.caches.values())
    
        def get_tenant_count(self) -> int:
            """Number of active tenants."""
            with self._lock:
                return len(self.caches)
    

    Key Design Decisions

    • Thread-Safety: Uses threading.RLock() for concurrent access.
    • Memory Efficiency: OrderedDict for O(1) LRU operations. No value serialization.
    • Fairness Granularity:
      • Per-Tenant: Fixed max entries with LRU eviction.
      • Cross-Tenant: Tenant LRU eviction prevents any single tenant from dominating.
    • Edge Cases Handled:
      • Non-existent tenants/keys return None (no error).
      • Empty caches after invalidation are cleaned up.
      • Unlimited tenants if max_tenants=None.
    • Extensibility: Easy to add TTL (e.g., store (value, expiry) tuples) or weighted eviction.

    This implementation balances simplicity, performance, and the requested features. Test it in your multi-tenant environment! If you need extensions (e.g., persistence, metrics, compression), let me know.

Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.