Observability vs Monitoring: Complete Guide for Modern Systems

from prometheus_client import Counter, Histogram, Gauge, CollectorRegistry
from functools import wraps
import time


# Define metrics
class MetricsCollector:
    """Collect application metrics."""
    
    def __init__(self, registry=None):
        self.registry = registry or CollectorRegistry()
        
        # Counter - for things that only increase
        self.request_count = Counter(
            'http_requests_total',
            'Total HTTP requests',
            ['method', 'endpoint', 'status'],
            registry=self.registry
        )
        
        # Histogram - for distributions
        self.request_duration = Histogram(
            'http_request_duration_seconds',
            'HTTP request duration',
            ['method', 'endpoint'],
            buckets=(0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0),
            registry=self.registry
        )
        
        # Gauge - for current values
        self.active_users = Gauge(
            'active_users',
            'Number of currently active users',
            registry=self.registry
        )
    
    def track_request(self, method: str, endpoint: str, status: int, duration: float):
        """Track an HTTP request."""
        self.request_count.labels(
            method=method,
            endpoint=endpoint,
            status=str(status)
        ).inc()
        
        self.request_duration.labels(
            method=method,
            endpoint=endpoint
        ).observe(duration)
    
    def set_active_users(self, count: int):
        """Set active user count."""
        self.active_users.set(count)


# Decorator for automatic metrics
def track_metrics(metrics: MetricsCollector, endpoint: str):
    """Decorator to automatically track metrics."""
    def decorator(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            start = time.time()
            status = 200
            
            try:
                result = func(*args, **kwargs)
                return result
            except Exception as e:
                status = 500
                raise
            finally:
                duration = time.time() - start
                method = 'GET'  # Extract from request in real usage
                metrics.track_request(method, endpoint, status, duration)
        
        return wrapper
    return decorator


# Usage
metrics = MetricsCollector()

@track_metrics(metrics, '/api/users')
def get_users():
    return User.query.all()

import logging
import json
from datetime import datetime
from typing import Any, Dict
import sys


class StructuredLogger:
    """Structured logging for observability."""
    
    def __init__(self, name: str, level: int = logging.INFO):
        self.logger = logging.getLogger(name)
        self.logger.setLevel(level)
        
        # Console handler with JSON formatter
        handler = logging.StreamHandler(sys.stdout)
        handler.setFormatter(JsonFormatter())
        self.logger.addHandler(handler)
    
    def log(self, level: int, event: str, **kwargs):
        """Log structured event."""
        self.logger.log(
            level,
            event,
            extra={
                'timestamp': datetime.utcnow().isoformat(),
                'event': event,
                **kwargs
            }
        )
    
    def debug(self, event: str, **kwargs):
        self.log(logging.DEBUG, event, **kwargs)
    
    def info(self, event: str, **kwargs):
        self.log(logging.INFO, event, **kwargs)
    
    def warning(self, event: str, **kwargs):
        self.log(logging.WARNING, event, **kwargs)
    
    def error(self, event: str, **kwargs):
        self.log(logging.ERROR, event, **kwargs)


class JsonFormatter(logging.Formatter):
    """Format logs as JSON."""
    
    def format(self, record: logging.LogRecord) -> str:
        log_data = {
            'timestamp': datetime.utcnow().isoformat(),
            'level': record.levelname,
            'logger': record.name,
            'message': record.getMessage(),
            'module': record.module,
            'function': record.funcName,
            'line': record.lineno
        }
        
        # Add exception info if present
        if record.exc_info:
            log_data['exception'] = self.formatException(record.exc_info)
        
        # Add extra fields
        if hasattr(record, 'event'):
            log_data['event'] = record.event
        
        return json.dumps(log_data)


# Usage
logger = StructuredLogger('myapp')

# Log with context
logger.info(
    'user_action',
    user_id='123',
    action='purchase',
    product_id='456',
    amount=99.99
)

from opentelemetry import trace
from opentelemetry.exporter.jaeger.thrift import JaegerExporter
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.instrumentation.flask import FlaskInstrumentor
from opentelemetry.trace import Status, StatusCode


# Initialize tracing
def init_tracing(service_name: str):
    """Initialize OpenTelemetry tracing."""
    
    # Create tracer provider
    provider = TracerProvider()
    
    # Jaeger exporter
    jaeger_exporter = JaegerExporter(
        agent_host_name="jaeger",
        agent_port=6831,
    )
    
    # Add batch processor
    provider.add_span_processor(
        BatchSpanProcessor(jaeger_exporter)
    )
    
    # Set global tracer provider
    trace.set_tracer_provider(provider)
    
    return trace.get_tracer(service_name)


# Instrument Flask
def instrument_flask(app):
    """Instrument Flask application."""
    FlaskInstrumentor().instrument_app(app)


# Usage
tracer = init_tracing('my-service')

def get_tracer():
    return tracer


# Decorator for tracing
def traced(span_name: str = None):
    """Decorator to add tracing to functions."""
    def decorator(func):
        def wrapper(*args, **kwargs):
            with tracer.start_as_current_span(
                span_name or func.__name__,
                attributes={
                    'function.name': func.__name__,
                    'function.module': func.__module__
                }
            ) as span:
                try:
                    result = func(*args, **kwargs)
                    span.set_status(Status(StatusCode.OK))
                    return result
                except Exception as e:
                    span.set_status(Status(StatusCode.ERROR, str(e)))
                    span.record_exception(e)
                    raise
        
        return wrapper
    return decorator


# Example: Trace a request through services
@traced('process-order')
def process_order(order_id: str, user_id: str):
    """Process an order with full tracing."""
    
    with tracer.start_as_current_span('fetch-user') as span:
        span.set_attribute('user_id', user_id)
        user = fetch_user(user_id)
    
    with tracer.start_as_current_span('validate-order') as span:
        span.set_attribute('order_id', order_id)
        validate_order(order_id)
    
    with tracer.start_as_current_span('charge-payment') as span:
        charge = process_payment(user, order_id)
    
    with tracer.start_as_current_span('send-notification') as span:
        send_order_notification(user, order_id)
    
    return {'order_id': order_id, 'status': 'complete'}

from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.resources import Resource, SERVICE_NAME
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.trace.export import BatchSpanProcessor


def setup_opentelemetry(service_name: str, otlp_endpoint: str = None):
    """Setup OpenTelemetry for a service."""
    
    # Create resource with service info
    resource = Resource.create({
        SERVICE_NAME: service_name,
        'service.version': '1.0.0',
        'deployment.environment': 'production'
    })
    
    # Create tracer provider
    provider = TracerProvider(resource=resource)
    
    # Add OTLP exporter if endpoint provided
    if otlp_endpoint:
        exporter = OTLPSpanExporter(
            endpoint=otlp_endpoint,
            insecure=True
        )
        provider.add_span_processor(
            BatchSpanProcessor(exporter)
        )
    
    # Set provider
    trace.set_tracer_provider(provider)
    
    return trace.get_tracer(service_name)


# Instrument libraries automatically
from opentelemetry.instrumentation import auto_instrumentation_init

auto_instrumentation_init()

import dash
from dash import dcc, html
from dash.dependencies import Input, Output
import plotly.graph_objs as go
from prometheus_client import CollectorRegistry, generate_latest
import requests


# Prometheus query function
def query_prometheus(query: str) -> list:
    """Query Prometheus for metrics."""
    response = requests.get(
        'http://localhost:9090/api/v1/query',
        params={'query': query}
    )
    return response.json()['data']['result']


# Create dashboard
app = dash.Dash(__name__)

app.layout = html.Div([
    html.H1('Observability Dashboard'),
    
    # Metrics section
    html.Div([
        html.H2('Key Metrics'),
        
        # Request rate
        dcc.Graph(id='request-rate'),
        
        # Error rate
        dcc.Graph(id='error-rate'),
        
        # Latency
        dcc.Graph(id='latency'),
        
    ]),
    
    # Refresh interval
    dcc.Interval(
        id='interval-component',
        interval=5*1000,  # 5 seconds
        n_intervals=0
    )
])


@app.callback(
    Output('request-rate', 'figure'),
    Input('interval-component', 'n_intervals')
)
def update_request_rate(n):
    """Update request rate chart."""
    results = query_prometheus('rate(http_requests_total[5m])')
    
    # Extract data
    timestamps = []
    values = []
    
    for result in results:
        timestamps.append(result['value'][0])
        values.append(float(result['value'][1]))
    
    return {
        'data': [go.Scatter(
            x=timestamps,
            y=values,
            mode='lines',
            name='Request Rate'
        )],
        'layout': go.Layout(
            title='Request Rate (req/s)',
            xaxis={'title': 'Time'},
            yaxis={'title': 'Requests/sec'}
        )
    }


if __name__ == '__main__':
    app.run_server(debug=True)