OpenSearch in Production: Logging, Analytics, and Security

Introduction

OpenSearch powers production systems for logging, analytics, and security. This article explores real-world use cases with practical implementation patterns.

Log Analytics

Application Logging

# Create logs index
PUT /app-logs
{
  "settings": {
    "number_of_shards": 3,
    "number_of_replicas": 1
  },
  "mappings": {
    "properties": {
      "timestamp": { "type": "date" },
      "level": { "type": "keyword" },
      "service": { "type": "keyword" },
      "message": { "type": "text" },
      "trace_id": { "type": "keyword" }
    }
  }
}

Log Ingest Pipeline

# Parse JSON logs
PUT /_ingest/pipeline/json-logs
{
  "processors": [
    {
      "json": {
        "field": "message",
        "target_field": "parsed"
      }
    }
  ]
}

Log Analysis Queries

# Error count by service
GET /app-logs/_search
{
  "size": 0,
  "query": { "term": { "level": "ERROR" } },
  "aggs": {
    "by_service": {
      "terms": { "field": "service" }
    }
  }
}

# Recent errors
GET /app-logs/_search
{
  "query": {
    "bool": {
      "must": [
        { "term": { "level": "ERROR" } },
        { "range": { "timestamp": { "gte": "now-1h" } } }
      ]
    }
  }
}

Application Search

E-Commerce Search

# Products index
PUT /products
{
  "mappings": {
    "properties": {
      "name": { "type": "text", "analyzer": "standard" },
      "description": { "type": "text" },
      "category": { "type": "keyword" },
      "price": { "type": "float" },
      "brand": { "type": "keyword" },
      "rating": { "type": "float" }
    }
  }
}

Autocomplete

# Completion suggester
PUT /products
{
  "mappings": {
    "properties": {
      "name_suggest": {
        "type": "completion"
      }
    }
  }
}

# Search suggestions
POST /products/_search
{
  "suggest": {
    "product-suggest": {
      "prefix": "wire",
      "completion": {
        "field": "name_suggest",
        "size": 10
      }
    }
  }
}

Security Analytics

Security Events

# Security logs index
PUT /security-events
{
  "mappings": {
    "properties": {
      "timestamp": { "type": "date" },
      "event_type": { "type": "keyword" },
      "source_ip": { "type": "ip" },
      "destination_ip": { "type": "ip" },
      "user": { "type": "keyword" },
      "action": { "type": "keyword" },
      "severity": { "type": "keyword" }
    }
  }
}

Threat Detection

# Failed logins
GET /security-events/_search
{
  "query": {
    "bool": {
      "must": [
        { "term": { "event_type": "login" } },
        { "term": { "action": "failed" } },
        { "range": { "timestamp": { "gte": "now-15m" } } }
      ]
    }
  }
}

# Group by source IP
GET /security-events/_search
{
  "size": 0,
  "query": { "term": { "action": "failed" } },
  "aggs": {
    "by_ip": {
      "terms": { "field": "source_ip", "size": 10 }
    }
  }
}

Business Intelligence

Sales Analytics

# Sales data
PUT /sales
{
  "mappings": {
    "properties": {
      "date": { "type": "date" },
      "product": { "type": "keyword" },
      "category": { "type": "keyword" },
      "amount": { "type": "float" },
      "region": { "type": "keyword" }
    }
  }
}

# Daily revenue
GET /sales/_search
{
  "size": 0,
  "aggs": {
    "daily_revenue": {
      "date_histogram": {
        "field": "date",
        "calendar_interval": "day"
      },
      "aggs": {
        "revenue": { "sum": { "field": "amount" } }
      }
    }
  }
}

Top Products

GET /sales/_search
{
  "size": 0,
  "aggs": {
    "top_products": {
      "terms": { "field": "product", "size": 10 },
      "aggs": {
        "revenue": { "sum": { "field": "amount" } }
      }
    }
  }
}

Observability

Metrics Collection

# System metrics
PUT /system-metrics
{
  "mappings": {
    "properties": {
      "timestamp": { "type": "date" },
      "host": { "type": "keyword" },
      "cpu": { "type": "float" },
      "memory": { "type": "float" },
      "disk": { "type": "float" }
    }
  }
}

Dashboards

# OpenSearch Dashboards
- CPU/Memory over time
- Error rates
- Request latency
- Active users

Best Practices

Index Management

# Use index templates
PUT /_index_template/logs-template
{
  "index_patterns": ["logs-*"],
  "template": {
    "settings": {
      "number_of_shards": 3
    }
  }
}

# ILM policy
PUT /_ilm/policy/logs-policy
{
  "policy": {
    "phases": {
      "hot": { "min_age": "0ms", "actions": {} },
      "warm": { "min_age": "30d", "actions": {"shrink": {"number_of_shards": 1}} },
      "delete": { "min_age": "90d", "actions": {"delete": {}} }
    }
  }
}

Conclusion

OpenSearch excels in production environments requiring log analytics, search, and observability. From application search to security monitoring, OpenSearch provides powerful real-time capabilities.

With proper index management and ILM policies, OpenSearch can handle massive data volumes while maintaining performance.