┌─────────────────────────────────────────────────────────────────────────────┐
│                    LLM SERVING AT SCALE ARCHITECTURE                        │
├─────────────────────────────────────────────────────────────────────────────┤
│                                                                             │
│  ┌─────────────────────────────────────────────────────────────────────┐   │
│  │ REQUEST INGRESS                                                     │   │
│  │   Load Balancer ──► Rate Limiter ──► Authentication                 │   │
│  └─────────────────────────────────────────────────────────────────────┘   │
│                           │                                                 │
│                           ▼                                                 │
│  ┌─────────────────────────────────────────────────────────────────────┐   │
│  │ INTELLIGENT ROUTING                                                 │   │
│  │   Request Classifier ──► Semantic Cache                             │   │
│  │                              │                                      │   │
│  │                    ┌─────────┴─────────┐                            │   │
│  │                    ▼                   ▼                            │   │
│  │               [Cache Hit]        [Cache Miss]                       │   │
│  │                    │                   │                            │   │
│  │                    │                   ▼                            │   │
│  │                    │            Model Router                        │   │
│  └────────────────────┼───────────────────┼────────────────────────────┘   │
│                       │                   │                                 │
│                       │                   ▼                                 │
│  ┌────────────────────┼───────────────────────────────────────────────┐    │
│  │ MODEL POOL         │                                               │    │
│  │   ┌─────────┐ ┌─────────┐ ┌─────────┐ ┌─────────┐                 │    │
│  │   │ GPT-4o  │ │GPT-3.5  │ │Claude   │ │ Local   │                 │    │
│  │   │         │ │Turbo    │ │3.5      │ │ Models  │                 │    │
│  │   └────┬────┘ └────┬────┘ └────┬────┘ └────┬────┘                 │    │
│  └────────┴───────────┴───────────┴───────────┴───────────────────────┘    │
│                           │                                                 │
│                           ▼                                                 │
│  ┌─────────────────────────────────────────────────────────────────────┐   │
│  │ RELIABILITY LAYER                                                   │   │
│  │   Circuit Breaker ──► Fallback Handler ──► Retry Logic              │   │
│  └─────────────────────────────────────────────────────────────────────┘   │
│                           │                                                 │
│                           ▼                                                 │
│  ┌─────────────────────────────────────────────────────────────────────┐   │
│  │ MONITORING                                                          │   │
│  │   ┌───────────┐  ┌───────────┐  ┌───────────┐  ┌───────────┐       │   │
│  │   │Prometheus │  │  Jaeger   │  │Structured │  │ Alerting  │       │   │
│  │   │(Metrics)  │──►│ (Traces) │  │   Logs    │  │           │       │   │
│  │   └───────────┘  └───────────┘  └───────────┘  └───────────┘       │   │
│  └─────────────────────────────────────────────────────────────────────┘   │
│                                                                             │
└─────────────────────────────────────────────────────────────────────────────┘

llm-serving/
├── src/
│   ├── __init__.py
│   ├── config.py
│   ├── gateway/
│   │   ├── __init__.py
│   │   ├── router.py           # Request routing
│   │   ├── rate_limiter.py     # Rate limiting
│   │   └── auth.py             # Authentication
│   ├── models/
│   │   ├── __init__.py
│   │   ├── base.py             # Model interface
│   │   ├── openai_provider.py  # OpenAI models
│   │   ├── anthropic_provider.py
│   │   └── local_provider.py   # Local models
│   ├── routing/
│   │   ├── __init__.py
│   │   ├── classifier.py       # Request classification
│   │   ├── selector.py         # Model selection
│   │   └── balancer.py         # Load balancing
│   ├── caching/
│   │   ├── __init__.py
│   │   └── semantic_cache.py   # Semantic caching
│   ├── resilience/
│   │   ├── __init__.py
│   │   ├── circuit_breaker.py
│   │   ├── fallback.py
│   │   └── retry.py
│   ├── observability/
│   │   ├── __init__.py
│   │   ├── metrics.py          # Prometheus metrics
│   │   ├── tracing.py          # Distributed tracing
│   │   └── logging.py          # Structured logging
│   └── api/
│       ├── __init__.py
│       └── main.py             # FastAPI application
├── kubernetes/
│   ├── deployment.yaml
│   ├── service.yaml
│   └── hpa.yaml
├── tests/
└── requirements.txt

# src/config.py
from pydantic_settings import BaseSettings
from typing import Dict, List, Optional

class ModelConfig(BaseSettings):
    name: str
    provider: str
    model_id: str
    max_tokens: int = 4096
    cost_per_1k_input: float
    cost_per_1k_output: float
    latency_p50_ms: int
    quality_score: float  # 0-1

class Settings(BaseSettings):
    # API Keys
    openai_api_key: str
    anthropic_api_key: str

    # Redis
    redis_url: str = "redis://localhost:6379"

    # Rate Limiting
    rate_limit_requests: int = 100
    rate_limit_window: int = 60

    # Caching
    cache_ttl: int = 3600
    cache_similarity_threshold: float = 0.95

    # Circuit Breaker
    failure_threshold: int = 5
    recovery_timeout: int = 30

    # Model Pool
    models: Dict[str, ModelConfig] = {
        "gpt-4o": ModelConfig(
            name="gpt-4o",
            provider="openai",
            model_id="gpt-4o",
            cost_per_1k_input=0.005,
            cost_per_1k_output=0.015,
            latency_p50_ms=800,
            quality_score=0.95
        ),
        "gpt-3.5-turbo": ModelConfig(
            name="gpt-3.5-turbo",
            provider="openai",
            model_id="gpt-3.5-turbo",
            cost_per_1k_input=0.0005,
            cost_per_1k_output=0.0015,
            latency_p50_ms=300,
            quality_score=0.80
        ),
        "claude-3-5-sonnet": ModelConfig(
            name="claude-3-5-sonnet",
            provider="anthropic",
            model_id="claude-3-5-sonnet-20241022",
            cost_per_1k_input=0.003,
            cost_per_1k_output=0.015,
            latency_p50_ms=600,
            quality_score=0.93
        )
    }

    class Config:
        env_file = ".env"

settings = Settings()

# src/models/base.py
from abc import ABC, abstractmethod
from typing import List, Dict, Optional, AsyncIterator
from pydantic import BaseModel
from dataclasses import dataclass

class Message(BaseModel):
    role: str
    content: str

class CompletionRequest(BaseModel):
    messages: List[Message]
    max_tokens: int = 1024
    temperature: float = 0.7
    stream: bool = False

@dataclass
class CompletionResponse:
    content: str
    model: str
    input_tokens: int
    output_tokens: int
    latency_ms: float

class ModelProvider(ABC):
    """Base class for model providers."""

    @abstractmethod
    async def complete(
        self,
        request: CompletionRequest
    ) -> CompletionResponse:
        pass

    @abstractmethod
    async def stream(
        self,
        request: CompletionRequest
    ) -> AsyncIterator[str]:
        pass

    @abstractmethod
    def health_check(self) -> bool:
        pass

# src/models/openai_provider.py
import time
from typing import AsyncIterator
from openai import AsyncOpenAI
from .base import ModelProvider, CompletionRequest, CompletionResponse
from ..config import settings

class OpenAIProvider(ModelProvider):
    """OpenAI model provider."""

    def __init__(self, model_id: str):
        self.client = AsyncOpenAI(api_key=settings.openai_api_key)
        self.model_id = model_id

    async def complete(
        self,
        request: CompletionRequest
    ) -> CompletionResponse:
        start = time.time()

        messages = [
            {"role": m.role, "content": m.content}
            for m in request.messages
        ]

        response = await self.client.chat.completions.create(
            model=self.model_id,
            messages=messages,
            max_tokens=request.max_tokens,
            temperature=request.temperature
        )

        latency_ms = (time.time() - start) * 1000

        return CompletionResponse(
            content=response.choices[0].message.content,
            model=self.model_id,
            input_tokens=response.usage.prompt_tokens,
            output_tokens=response.usage.completion_tokens,
            latency_ms=latency_ms
        )

    async def stream(
        self,
        request: CompletionRequest
    ) -> AsyncIterator[str]:
        messages = [
            {"role": m.role, "content": m.content}
            for m in request.messages
        ]

        stream = await self.client.chat.completions.create(
            model=self.model_id,
            messages=messages,
            max_tokens=request.max_tokens,
            temperature=request.temperature,
            stream=True
        )

        async for chunk in stream:
            if chunk.choices[0].delta.content:
                yield chunk.choices[0].delta.content

    def health_check(self) -> bool:
        try:
            # Quick model list call to verify API is working
            return True
        except Exception:
            return False

# src/routing/classifier.py
from typing import Tuple
from enum import Enum
from langchain_openai import ChatOpenAI
from langchain_core.prompts import ChatPromptTemplate
from pydantic import BaseModel, Field
from ..config import settings

class RequestComplexity(str, Enum):
    SIMPLE = "simple"      # Factual, short answers
    MODERATE = "moderate"  # Some reasoning required
    COMPLEX = "complex"    # Multi-step reasoning, analysis

class ClassificationResult(BaseModel):
    complexity: RequestComplexity
    requires_creativity: bool = False
    requires_accuracy: bool = True
    estimated_tokens: int
    reasoning: str

class RequestClassifier:
    """Classifies incoming requests for routing."""

    def __init__(self):
        self.llm = ChatOpenAI(
            model="gpt-3.5-turbo",
            api_key=settings.openai_api_key,
            temperature=0
        ).with_structured_output(ClassificationResult)

        self.prompt = ChatPromptTemplate.from_messages([
            ("system", """Classify this LLM request for optimal routing.

Simple: Factual questions, translations, formatting
Moderate: Explanations, summaries, basic analysis
Complex: Multi-step reasoning, code generation, creative writing

Be concise in reasoning."""),
            ("human", "{request}")
        ])

    async def classify(self, messages: list) -> ClassificationResult:
        """Classify a request."""
        # Use last user message for classification
        user_messages = [m for m in messages if m.role == "user"]
        if not user_messages:
            return ClassificationResult(
                complexity=RequestComplexity.SIMPLE,
                estimated_tokens=100,
                reasoning="No user message"
            )

        request_text = user_messages[-1].content[:500]  # Limit for classification

        chain = self.prompt | self.llm
        result = await chain.ainvoke({"request": request_text})
        return result

# src/routing/selector.py
from typing import List, Optional
from dataclasses import dataclass
from .classifier import ClassificationResult, RequestComplexity
from ..config import settings, ModelConfig

@dataclass
class ModelSelection:
    primary: str
    fallback: Optional[str]
    reason: str

class ModelSelector:
    """Selects optimal model based on request characteristics."""

    def __init__(self):
        self.models = settings.models

    def select(
        self,
        classification: ClassificationResult,
        constraints: dict = None
    ) -> ModelSelection:
        """Select optimal model for the request."""
        constraints = constraints or {}

        # Filter models by constraints
        available_models = list(self.models.values())

        if constraints.get("max_cost"):
            available_models = [
                m for m in available_models
                if m.cost_per_1k_input <= constraints["max_cost"]
            ]

        if constraints.get("max_latency_ms"):
            available_models = [
                m for m in available_models
                if m.latency_p50_ms <= constraints["max_latency_ms"]
            ]

        if not available_models:
            # Fallback to cheapest
            return ModelSelection(
                primary="gpt-3.5-turbo",
                fallback=None,
                reason="No models meet constraints, using fallback"
            )

        # Score models based on request
        scored = []
        for model in available_models:
            score = self._score_model(model, classification)
            scored.append((model.name, score))

        scored.sort(key=lambda x: x[1], reverse=True)

        primary = scored[0][0]
        fallback = scored[1][0] if len(scored) > 1 else None

        return ModelSelection(
            primary=primary,
            fallback=fallback,
            reason=f"Selected based on {classification.complexity} complexity"
        )

    def _score_model(
        self,
        model: ModelConfig,
        classification: ClassificationResult
    ) -> float:
        """Score a model for the given request."""
        score = 0

        # Quality weight based on complexity
        if classification.complexity == RequestComplexity.COMPLEX:
            score += model.quality_score * 0.6
            score -= (model.cost_per_1k_input / 0.01) * 0.1  # Less cost sensitive
        elif classification.complexity == RequestComplexity.MODERATE:
            score += model.quality_score * 0.4
            score -= (model.cost_per_1k_input / 0.01) * 0.3
        else:  # SIMPLE
            score += model.quality_score * 0.2
            score -= (model.cost_per_1k_input / 0.01) * 0.5  # Cost sensitive

        # Latency consideration
        if classification.requires_accuracy:
            score += 0.1  # Prefer accuracy over speed
        else:
            score -= (model.latency_p50_ms / 1000) * 0.2

        return score

┌─────────────────────────────────────────────────────────────┐
│ REQUEST CLASSIFICATION → MODEL SELECTION                    │
├─────────────────────────────────────────────────────────────┤
│                                                             │
│  Request: "What is 2+2?"                                    │
│        │                                                    │
│        ▼                                                    │
│  ┌─────────────────────────────────────────────────────┐    │
│  │ RequestClassifier (GPT-3.5, fast)                   │    │
│  │ • complexity: SIMPLE                                │    │
│  │ • requires_creativity: false                        │    │
│  │ • estimated_tokens: 50                              │    │
│  └─────────────────────────────────────────────────────┘    │
│        │                                                     │
│        ▼                                                     │
│  ┌─────────────────────────────────────────────────────┐    │
│  │ ModelSelector scoring:                              │    │
│  │                                                     │    │
│  │ GPT-4o:      quality×0.2 - cost×0.5 = 0.19 - 0.25   │    │
│  │ GPT-3.5:     quality×0.2 - cost×0.5 = 0.16 - 0.025  │    │
│  │ Claude 3.5:  quality×0.2 - cost×0.5 = 0.19 - 0.15   │    │
│  │                                                     │    │
│  │ For SIMPLE: cost matters most → GPT-3.5 wins        │    │
│  └─────────────────────────────────────────────────────┘    │
│        │                                                    │
│        ▼                                                    │
│  Selected: primary=gpt-3.5-turbo, fallback=gpt-4o           │
│                                                             │
└─────────────────────────────────────────────────────────────┘

# src/caching/semantic_cache.py
from typing import Optional, Tuple
import hashlib
import json
import numpy as np
from openai import OpenAI
import redis.asyncio as redis
from ..config import settings

class SemanticCache:
    """Semantic caching for LLM responses."""

    def __init__(self):
        self.redis = redis.from_url(settings.redis_url)
        self.client = OpenAI(api_key=settings.openai_api_key)
        self.threshold = settings.cache_similarity_threshold
        self.ttl = settings.cache_ttl

    def _get_embedding(self, text: str) -> list:
        """Get embedding for text."""
        response = self.client.embeddings.create(
            input=text,
            model="text-embedding-3-small"
        )
        return response.data[0].embedding

    def _cosine_similarity(self, a: list, b: list) -> float:
        """Calculate cosine similarity."""
        a = np.array(a)
        b = np.array(b)
        return np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b))

    async def get(
        self,
        messages: list
    ) -> Optional[Tuple[str, float]]:
        """Get cached response if similar query exists."""
        # Create cache key from messages
        query = " ".join([m.content for m in messages if m.role == "user"])
        query_embedding = self._get_embedding(query)

        # Check recent cache entries
        keys = await self.redis.keys("cache:*")

        for key in keys[:100]:  # Check last 100 entries
            data = await self.redis.get(key)
            if not data:
                continue

            entry = json.loads(data)
            cached_embedding = entry.get("embedding")

            if cached_embedding:
                similarity = self._cosine_similarity(
                    query_embedding,
                    cached_embedding
                )

                if similarity >= self.threshold:
                    return entry["response"], similarity

        return None

    async def set(
        self,
        messages: list,
        response: str
    ):
        """Cache a response."""
        query = " ".join([m.content for m in messages if m.role == "user"])
        query_embedding = self._get_embedding(query)

        # Create unique key
        key_content = json.dumps([m.dict() for m in messages])
        cache_key = f"cache:{hashlib.sha256(key_content.encode()).hexdigest()[:16]}"

        entry = {
            "response": response,
            "embedding": query_embedding,
            "messages": [m.dict() for m in messages]
        }

        await self.redis.setex(
            cache_key,
            self.ttl,
            json.dumps(entry)
        )

┌─────────────────────────────────────────────────────────────┐
│ SEMANTIC CACHE FLOW                                         │
├─────────────────────────────────────────────────────────────┤
│                                                             │
│  New query: "What's the capital of France?"                 │
│        │                                                    │
│        ▼                                                    │
│  ┌─────────────────────────────────────────────────────┐    │
│  │ 1. Embed query with text-embedding-3-small          │    │
│  │    → [0.12, -0.34, 0.56, ...]                       │    │
│  └─────────────────────────────────────────────────────┘    │
│        │                                                     │
│        ▼                                                     │
│  ┌─────────────────────────────────────────────────────┐    │
│  │ 2. Check Redis for similar queries                  │    │
│  │                                                     │    │
│  │ Cached: "What is the capital city of France?"       │    │
│  │ Embedding: [0.13, -0.33, 0.55, ...]                 │    │
│  │ Cosine similarity: 0.97 > threshold (0.95)          │    │
│  │                                                     │    │
│  │ ✓ CACHE HIT                                         │    │
│  └─────────────────────────────────────────────────────┘    │
│        │                                                    │
│        ▼                                                    │
│  Return cached response: "The capital of France is Paris."  │
│  Cost: $0 (embedding only: ~$0.00001)                       │
│                                                             │
└─────────────────────────────────────────────────────────────┘

# src/resilience/circuit_breaker.py
import asyncio
from typing import Callable, Any
from enum import Enum
from dataclasses import dataclass
from datetime import datetime, timedelta
from ..config import settings

class CircuitState(str, Enum):
    CLOSED = "closed"
    OPEN = "open"
    HALF_OPEN = "half_open"

@dataclass
class CircuitBreaker:
    """Circuit breaker for model providers."""

    name: str
    failure_threshold: int = settings.failure_threshold
    recovery_timeout: int = settings.recovery_timeout
    state: CircuitState = CircuitState.CLOSED
    failures: int = 0
    last_failure: datetime = None
    successes_in_half_open: int = 0

    async def call(
        self,
        func: Callable,
        *args,
        **kwargs
    ) -> Any:
        """Execute function with circuit breaker protection."""
        if self.state == CircuitState.OPEN:
            if self._should_try_reset():
                self.state = CircuitState.HALF_OPEN
                self.successes_in_half_open = 0
            else:
                raise CircuitOpenError(f"Circuit {self.name} is open")

        try:
            result = await func(*args, **kwargs)
            self._on_success()
            return result
        except Exception as e:
            self._on_failure()
            raise

    def _on_success(self):
        """Handle successful call."""
        if self.state == CircuitState.HALF_OPEN:
            self.successes_in_half_open += 1
            if self.successes_in_half_open >= 3:
                self.state = CircuitState.CLOSED
                self.failures = 0
        else:
            self.failures = 0

    def _on_failure(self):
        """Handle failed call."""
        self.failures += 1
        self.last_failure = datetime.now()

        if self.failures >= self.failure_threshold:
            self.state = CircuitState.OPEN

    def _should_try_reset(self) -> bool:
        """Check if we should try resetting the circuit."""
        if not self.last_failure:
            return True
        return datetime.now() - self.last_failure > timedelta(
            seconds=self.recovery_timeout
        )

class CircuitOpenError(Exception):
    pass


class CircuitBreakerManager:
    """Manages circuit breakers for all providers."""

    def __init__(self):
        self.breakers: dict[str, CircuitBreaker] = {}

    def get_or_create(self, name: str) -> CircuitBreaker:
        """Get or create a circuit breaker."""
        if name not in self.breakers:
            self.breakers[name] = CircuitBreaker(name=name)
        return self.breakers[name]

    def get_status(self) -> dict:
        """Get status of all circuit breakers."""
        return {
            name: {
                "state": cb.state.value,
                "failures": cb.failures
            }
            for name, cb in self.breakers.items()
        }

┌─────────────────────────────────────────────────────────────┐
│ CIRCUIT BREAKER STATES                                      │
├─────────────────────────────────────────────────────────────┤
│                                                             │
│                    ┌───────────────┐                        │
│                    │    CLOSED     │                        │
│                    │ (Normal ops)  │                        │
│                    └───────┬───────┘                        │
│                            │                                │
│                            │ 5 failures                     │
│                            ▼                                │
│                    ┌───────────────┐                        │
│                    │     OPEN      │                        │
│                    │ (Fail fast)   │◄────────┐              │
│                    └───────┬───────┘         │              │
│                            │                 │              │
│                            │ 30s timeout     │ failure      │
│                            ▼                 │              │
│                    ┌───────────────┐         │              │
│                    │  HALF-OPEN    │─────────┘              │
│                    │ (Test traffic)│                        │
│                    └───────┬───────┘                        │
│                            │                                │
│                            │ 3 successes                    │
│                            ▼                                │
│                    ┌───────────────┐                        │
│                    │    CLOSED     │                        │
│                    └───────────────┘                        │
│                                                             │
└─────────────────────────────────────────────────────────────┘

# src/observability/metrics.py
from prometheus_client import Counter, Histogram, Gauge
import time
from functools import wraps

# Request metrics
REQUEST_COUNT = Counter(
    "llm_requests_total",
    "Total LLM requests",
    ["model", "status"]
)

REQUEST_LATENCY = Histogram(
    "llm_request_latency_seconds",
    "LLM request latency",
    ["model"],
    buckets=[0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0]
)

# Token metrics
INPUT_TOKENS = Counter(
    "llm_input_tokens_total",
    "Total input tokens",
    ["model"]
)

OUTPUT_TOKENS = Counter(
    "llm_output_tokens_total",
    "Total output tokens",
    ["model"]
)

# Cost tracking
ESTIMATED_COST = Counter(
    "llm_estimated_cost_dollars",
    "Estimated cost in dollars",
    ["model"]
)

# Cache metrics
CACHE_HITS = Counter(
    "llm_cache_hits_total",
    "Total cache hits"
)

CACHE_MISSES = Counter(
    "llm_cache_misses_total",
    "Total cache misses"
)

# Circuit breaker metrics
CIRCUIT_STATE = Gauge(
    "llm_circuit_breaker_state",
    "Circuit breaker state (0=closed, 1=open, 2=half_open)",
    ["provider"]
)


class MetricsCollector:
    """Collects and records metrics."""

    def record_request(
        self,
        model: str,
        status: str,
        latency_ms: float,
        input_tokens: int,
        output_tokens: int,
        cost: float
    ):
        """Record request metrics."""
        REQUEST_COUNT.labels(model=model, status=status).inc()
        REQUEST_LATENCY.labels(model=model).observe(latency_ms / 1000)
        INPUT_TOKENS.labels(model=model).inc(input_tokens)
        OUTPUT_TOKENS.labels(model=model).inc(output_tokens)
        ESTIMATED_COST.labels(model=model).inc(cost)

    def record_cache_hit(self):
        CACHE_HITS.inc()

    def record_cache_miss(self):
        CACHE_MISSES.inc()

    def update_circuit_state(self, provider: str, state: int):
        CIRCUIT_STATE.labels(provider=provider).set(state)

metrics = MetricsCollector()

┌─────────────────────────────────────────────────────────────┐
│ KEY METRICS FOR LLM SERVING                                 │
├─────────────────────────────────────────────────────────────┤
│                                                             │
│  REQUEST METRICS                                            │
│  ┌─────────────────────────────────────────────────────┐    │
│  │ llm_requests_total{model="gpt-4o", status="success"}│    │
│  │ llm_request_latency_seconds{model="gpt-3.5-turbo"}  │    │
│  │ (histogram with buckets: 0.1, 0.25, 0.5, 1, 2.5, 5) │    │
│  └─────────────────────────────────────────────────────┘    │
│                                                              │
│  COST TRACKING                                               │
│  ┌─────────────────────────────────────────────────────┐    │
│  │ llm_input_tokens_total{model="gpt-4o"}              │    │
│  │ llm_output_tokens_total{model="gpt-4o"}             │    │
│  │ llm_estimated_cost_dollars{model="gpt-4o"}          │    │
│  │                                                     │    │
│  │ Dashboard: sum(rate(llm_estimated_cost_dollars[1h]))│    │
│  │            → $28.50/hour across all models          │    │
│  └─────────────────────────────────────────────────────┘    │
│                                                              │
│  CACHE EFFICIENCY                                            │
│  ┌─────────────────────────────────────────────────────┐    │
│  │ llm_cache_hits_total / (hits + misses) = hit rate   │    │
│  │ Target: > 30% for significant cost savings          │    │
│  └─────────────────────────────────────────────────────┘    │
│                                                              │
│  RELIABILITY                                                 │
│  ┌─────────────────────────────────────────────────────┐    │
│  │ llm_circuit_breaker_state{provider="openai"}        │    │
│  │ 0=closed, 1=open, 2=half_open                       │    │
│  │ Alert: state != 0 for > 5 minutes                   │    │
│  └─────────────────────────────────────────────────────┘    │
│                                                             │
└─────────────────────────────────────────────────────────────┘

# src/api/main.py
from fastapi import FastAPI, HTTPException, Request, Depends
from fastapi.responses import StreamingResponse
from prometheus_client import make_asgi_app
from typing import Optional
import time

from ..models.base import CompletionRequest, Message
from ..models.openai_provider import OpenAIProvider
from ..routing.classifier import RequestClassifier
from ..routing.selector import ModelSelector
from ..caching.semantic_cache import SemanticCache
from ..resilience.circuit_breaker import CircuitBreakerManager, CircuitOpenError
from ..observability.metrics import metrics
from ..config import settings

app = FastAPI(title="LLM Gateway", description="Production LLM serving")

# Mount Prometheus metrics
metrics_app = make_asgi_app()
app.mount("/metrics", metrics_app)

# Initialize components
classifier = RequestClassifier()
selector = ModelSelector()
cache = SemanticCache()
circuit_manager = CircuitBreakerManager()

# Model providers
providers = {
    "gpt-4o": OpenAIProvider("gpt-4o"),
    "gpt-3.5-turbo": OpenAIProvider("gpt-3.5-turbo"),
}

@app.post("/v1/chat/completions")
async def chat_completions(request: CompletionRequest):
    """OpenAI-compatible chat completions endpoint."""
    start_time = time.time()

    # Check cache
    cached = await cache.get(request.messages)
    if cached:
        response, similarity = cached
        metrics.record_cache_hit()
        return {
            "choices": [{"message": {"role": "assistant", "content": response}}],
            "model": "cached",
            "cached": True,
            "similarity": similarity
        }

    metrics.record_cache_miss()

    # Classify request
    classification = await classifier.classify(request.messages)

    # Select model
    selection = selector.select(classification)

    # Get provider and circuit breaker
    provider = providers.get(selection.primary)
    if not provider:
        raise HTTPException(500, f"Provider {selection.primary} not found")

    circuit = circuit_manager.get_or_create(selection.primary)

    try:
        # Execute with circuit breaker
        response = await circuit.call(provider.complete, request)

        # Calculate cost
        model_config = settings.models[selection.primary]
        cost = (
            response.input_tokens * model_config.cost_per_1k_input / 1000 +
            response.output_tokens * model_config.cost_per_1k_output / 1000
        )

        # Record metrics
        metrics.record_request(
            model=selection.primary,
            status="success",
            latency_ms=response.latency_ms,
            input_tokens=response.input_tokens,
            output_tokens=response.output_tokens,
            cost=cost
        )

        # Cache response
        await cache.set(request.messages, response.content)

        return {
            "choices": [{"message": {"role": "assistant", "content": response.content}}],
            "model": response.model,
            "usage": {
                "prompt_tokens": response.input_tokens,
                "completion_tokens": response.output_tokens,
                "total_tokens": response.input_tokens + response.output_tokens
            },
            "latency_ms": response.latency_ms,
            "cost": cost
        }

    except CircuitOpenError:
        # Try fallback
        if selection.fallback:
            fallback_provider = providers.get(selection.fallback)
            if fallback_provider:
                response = await fallback_provider.complete(request)
                return {
                    "choices": [{"message": {"role": "assistant", "content": response.content}}],
                    "model": response.model,
                    "fallback": True
                }

        raise HTTPException(503, "Service temporarily unavailable")

    except Exception as e:
        metrics.record_request(
            model=selection.primary,
            status="error",
            latency_ms=(time.time() - start_time) * 1000,
            input_tokens=0,
            output_tokens=0,
            cost=0
        )
        raise HTTPException(500, str(e))

@app.get("/v1/models")
async def list_models():
    """List available models."""
    return {
        "models": [
            {
                "id": name,
                "provider": config.provider,
                "quality_score": config.quality_score,
                "cost_per_1k_input": config.cost_per_1k_input
            }
            for name, config in settings.models.items()
        ]
    }

@app.get("/health")
async def health():
    """Health check endpoint."""
    circuit_status = circuit_manager.get_status()
    all_closed = all(
        s["state"] == "closed"
        for s in circuit_status.values()
    )

    return {
        "status": "healthy" if all_closed else "degraded",
        "circuits": circuit_status
    }

@app.get("/ready")
async def ready():
    """Readiness check."""
    return {"ready": True}

# kubernetes/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: llm-gateway
spec:
  replicas: 3
  selector:
    matchLabels:
      app: llm-gateway
  template:
    metadata:
      labels:
        app: llm-gateway
    spec:
      containers:
      - name: llm-gateway
        image: llm-gateway:latest
        ports:
        - containerPort: 8000
        env:
        - name: OPENAI_API_KEY
          valueFrom:
            secretKeyRef:
              name: llm-secrets
              key: openai-api-key
        - name: REDIS_URL
          value: redis://redis-service:6379
        resources:
          requests:
            cpu: 500m
            memory: 512Mi
          limits:
            cpu: 2000m
            memory: 2Gi
        readinessProbe:
          httpGet:
            path: /ready
            port: 8000
          initialDelaySeconds: 5
          periodSeconds: 10
        livenessProbe:
          httpGet:
            path: /health
            port: 8000
          initialDelaySeconds: 15
          periodSeconds: 20
---
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: llm-gateway-hpa
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: llm-gateway
  minReplicas: 3
  maxReplicas: 20
  metrics:
  - type: Resource
    resource:
      name: cpu
      target:
        type: Utilization
        averageUtilization: 70