Circuit Breaker & Resilience

import time
from enum import Enum

class State(Enum):
    CLOSED = "closed"
    OPEN = "open"
    HALF_OPEN = "half_open"

class CircuitBreaker:
    def __init__(self, failure_threshold=5, recovery_timeout=30):
        self.failure_threshold = failure_threshold
        self.recovery_timeout = recovery_timeout
        self.state = State.CLOSED
        self.failure_count = 0
        self.success_count = 0
        self.last_failure_time = None

    def call(self, func, *args, fallback=None, **kwargs):
        # OPEN state check
        if self.state == State.OPEN:
            if time.time() - self.last_failure_time > self.recovery_timeout:
                self.state = State.HALF_OPEN
                print("🟡 HALF-OPEN: Testing service...")
            else:
                print("🔴 OPEN: Fast fail!")
                return fallback() if fallback else None

        try:
            result = func(*args, **kwargs)
            self._handle_success()
            return result
        except Exception as e:
            self._handle_failure()
            return fallback() if fallback else None

    def _handle_success(self):
        if self.state == State.HALF_OPEN:
            self.success_count += 1
            if self.success_count >= 2:
                self.state = State.CLOSED
                self.failure_count = self.success_count = 0
                print("🟢 CLOSED: Service recovered!")

    def _handle_failure(self):
        self.failure_count += 1
        self.last_failure_time = time.time()
        if self.failure_count >= self.failure_threshold:
            self.state = State.OPEN
            print(f"🔴 OPEN: {self.failure_count} failures!")

# Usage
cb = CircuitBreaker(failure_threshold=3)

def payment_call():
    raise Exception("Payment down!")

def fallback():
    return {"status": "pending", "msg": "Processing soon"}

for i in range(5):
    result = cb.call(payment_call, fallback=fallback)
    print(f"Attempt {i+1}:", result)

async function retryWithBackoff(fn, maxRetries = 3, baseDelay = 100) {
    for (let attempt = 1; attempt <= maxRetries; attempt++) {
        try {
            return await fn();
        } catch (err) {
            if (attempt === maxRetries) throw err;
            // Exponential Backoff: 100 → 200 → 400ms
            const delay = baseDelay * Math.pow(2, attempt - 1);
            // Jitter: random 0-100ms extra (prevent thundering herd)
            const jitter = Math.random() * 100;
            console.log(`❌ Attempt ${attempt} failed. Wait ${(delay+jitter).toFixed(0)}ms...`);
            await new Promise(r => setTimeout(r, delay + jitter));
        }
    }
}

// Usage: wrap any async service call
const result = await retryWithBackoff(async () => {
    const res = await fetch('http://payment-service/charge', {
        method: 'POST',
        signal: AbortSignal.timeout(2000)  // 2s timeout
    });
    if (!res.ok) throw new Error(`HTTP ${res.status}`);
    return res.json();
}, 3, 200);

apiVersion: networking.istio.io/v1alpha3
kind: DestinationRule
metadata:
  name: payment-circuit-breaker
spec:
  host: payment-service
  trafficPolicy:
    connectionPool:
      tcp:
        maxConnections: 100        # Max TCP connections
      http:
        http1MaxPendingRequests: 10  # Bulkhead: max pending
        maxRequestsPerConnection: 2
    outlierDetection:
      # Circuit Breaker: 50% error rate → eject
      consecutiveGatewayErrors: 5   # 5 consecutive errors → OPEN
      interval: 10s                 # Check every 10 seconds
      baseEjectionTime: 30s         # OPEN for 30 seconds
      maxEjectionPercent: 100       # Eject all bad instances
      minHealthPercent: 0