Secrets Management at Scale: Vault, AWS Secrets Manager

# vault-values.yaml
global:
  enabled: true
  tlsDisable: false
  
injector:
  enabled: true
  annotations:
    vault.hashicorp.com/agent-inject: "true"
    vault.hashicorp.com/role: "app"
    
server:
  ha:
    enabled: true
    replicas: 3
    raft:
      enabled: true
      config: |
        ui = true
        listener "tcp" {
          tls_disable = 0
          address = "[::]:8200"
          cluster_address = "[::]:8201"
          tls_cert_file = "/vault/userconfig/vault-tls/vault.crt"
          tls_key_file = "/vault/userconfig/vault-tls/vault.key"
        }
        storage "raft" {
          path = "/vault/data"
          retry_join {
            leader_api_addr = "http://vault-0.vault-internal:8200"
          }
          retry_join {
            leader_api_addr = "http://vault-1.vault-internal:8200"
          }
          retry_join {
            leader_api_addr = "http://vault-2.vault-internal:8200"
          }
        }
        service_registration "kubernetes" {}

# Install Vault with Helm
helm repo add hashicorp https://helm.releases.hashicorp.com
helm install vault hashicorp/vault \
  --namespace vault \
  --create-namespace \
  -f vault-values.yaml

# Enable KV secrets engine
vault secrets enable -path=secret kv-v2

# Write secrets
vault kv put secret/database username=admin password=super-secret

# Read secrets
vault kv get secret/database

# Read specific field
vault kv get -field=username secret/database

# Delete secret
vault kv delete secret/database

# List secrets
vault kv list secret/

# Enable database secrets engine
vault secrets enable database

# Configure database connection
vault write database/config/my-postgres \
    plugin_name=postgresql-database-plugin \
    connection_url="postgresql://{{username}}:{{password}}@localhost:5432/mydb" \
    allowed_roles="app-role" \
    username="vault-admin" \
    password="vault-admin-password"

# Create role
vault write database/roles/app-role \
    db_name=my-postgres \
    creation_statements="CREATE ROLE \"{{name}}\" WITH LOGIN PASSWORD '{{password}}' VALID UNTIL '{{expiration}}'; \
        GRANT SELECT ON ALL TABLES IN SCHEMA public TO \"{{name}}\";" \
    default_ttl="1h" \
    max_ttl="24h"

# Generate credentials
vault read database/creds/app-role

# Vault Agent Injector
apiVersion: apps/v1
kind: Deployment
metadata:
  name: app-deployment
spec:
  selector:
    matchLabels:
      app: myapp
  template:
    metadata:
      labels:
        app: myapp
      annotations:
        vault.hashicorp.com/agent-inject: "true"
        vault.hashicorp.com/role: "app"
        vault.hashicorp.com/agent-inject-secret-config: "secret/data/myapp"
        vault.hashicorp.com/agent-inject-template-config: |
          {{- with secret "secret/data/myapp" -}}
          export DB_USERNAME="{{ .Data.data.db_username }}"
          export DB_PASSWORD="{{ .Data.data.db_password }}"
          export API_KEY="{{ .Data.data.api_key }}"
          {{- end }}
    spec:
      serviceAccountName: app-sa
      containers:
        - name: app
          image: myapp:latest
          envFrom:
            - configMapRef:
                name: app-config

# Kubernetes Auth Method
apiVersion: v1
kind: ServiceAccount
metadata:
  name: app-sa
  namespace: default
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: vault-auth-delegator
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:auth-delegator
subjects:
  - kind: ServiceAccount
    name: app-sa
    namespace: default

# Configure Kubernetes auth in Vault
vault auth enable kubernetes

vault write auth/kubernetes/config \
    kubernetes_host="https://$KUBERNETES_PORT_443_TCP_ADDR:443" \
    token_reviewer_jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" \
    kubernetes_ca_cert=@/var/run/secrets/kubernetes.io/serviceaccount/ca.crt

# Create policy
vault policy write app-policy - <<EOF
path "secret/data/myapp" {
  capabilities = ["read"]
}
path "database/creds/app-role" {
  capabilities = ["read"]
}
EOF

# Create Kubernetes auth role
vault write auth/kubernetes/role/app \
    bound_service_account_names=app-sa \
    bound_service_account_namespaces=default \
    policies=app-policy \
    ttl=1h

#!/usr/bin/env python3
"""AWS Secrets Manager operations."""

import boto3
import json
from datetime import datetime

secretsmanager = boto3.client('secretsmanager')

def create_secret(name, secret_dict, description=""):
    """Create a new secret."""
    response = secretsmanager.create_secret(
        Name=name,
        SecretString=json.dumps(secret_dict),
        Description=description,
        Tags=[
            {'Key': 'environment', 'Value': 'production'},
            {'Key': 'managed-by', 'Value': 'automation'}
        ]
    )
    return response['ARN']

def get_secret(name):
    """Retrieve secret value."""
    response = secretsmanager.get_secret_value(SecretId=name)
    return json.loads(response['SecretString'])

def update_secret(name, secret_dict):
    """Update existing secret."""
    response = secretsmanager.put_secret_value(
        SecretId=name,
        SecretString=json.dumps(secret_dict)
    )
    return response['VersionId']

def rotate_secret(secret_name):
    """Trigger rotation."""
    response = secretsmanager.rotate_secret(
        SecretId=secret_name,
        RotationRules={
            'AutomaticallyAfterDays': 30
        }
    )
    return response['RotationEnabled']

def list_secrets():
    """List all secrets."""
    response = secretsmanager.list_secrets(
        MaxResults=100,
        Filters=[
            {'Key': 'tag-value', 'Values': ['production']}
        ]
    )
    return response['SecretList']

#!/usr/bin/env python3
"""Lambda function for secret rotation."""

import boto3
import json
import os
import pymysql

def lambda_handler(event, context):
    """Handle secret rotation."""
    
    secret_arn = event['SecretArn']
    step = event['Step']
    
    secretsmanager = boto3.client('secretsmanager')
    
    # Get current secret
    current = secretsmanager.get_secret_value(SecretId=secret_arn)
    current_creds = json.loads(current['SecretString'])
    
    if step == 'createSecret':
        # Generate new password
        import secrets
        new_password = secrets.token_urlsafe(16)
        
        # Create new secret
        new_creds = current_creds.copy()
        new_creds['password'] = new_password
        
        secretsmanager.put_secret_value(
            SecretId=event['ClientRequestToken'],
            SecretString=json.dumps(new_creds),
            VersionStages=['AWSPENDING']
        )
        
        return {'status': 'success'}
    
    elif step == 'setSecret':
        # Apply new password to database
        connection = pymysql.connect(
            host=current_creds['host'],
            user=current_creds['username'],
            password=current_creds['password']
        )
        
        with connection.cursor() as cursor:
            cursor.execute(
                f"ALTER USER '{current_creds['username']}' IDENTIFIED BY '{new_creds['password']}'"
            )
        connection.commit()
        connection.close()
        
        return {'status': 'success'}
    
    elif step == 'testSecret':
        # Test new credentials
        try:
            connection = pymysql.connect(
                host=current_creds['host'],
                user=current_creds['username'],
                password=current_creds['password'],
                connect_timeout=5
            )
            connection.close()
            return {'status': 'success'}
        except Exception as e:
            return {'status': 'failed', 'error': str(e)}
    
    elif step == 'finishSecret':
        # Mark as current
        secretsmanager.update_secret_version_stage(
            SecretId=secret_arn,
            VersionStage='AWSCURRENT',
            MoveToVersionId=event['ClientRequestToken']
        )
        
        return {'status': 'success'}
    
    return {'status': 'unknown step'}

# Terraform Azure Key Vault
resource "azurerm_key_vault" "main" {
  name                = "mykeyvault"
  location            = azurerm_resource_group.main.location
  resource_group_name = azurerm_resource_group.main.name
  
  sku_name = "standard"
  
  tenant_id = data.azurerm_client_config.current.tenant_id
  
  access_policy {
    tenant_id = data.azurerm_client_config.current.tenant_id
    object_id = data.azurerm_client_config.current.object_id
    
    secret_permissions = [
      "get", "list", "set", "delete",
      "backup", "restore", "recover"
    ]
    
    key_permissions = [
      "get", "list", "create", "delete",
      "backup", "restore", "recover"
    ]
  }
  
  network_acls {
    default_action = "Allow"
    bypass         = "AzureServices"
  }
  
  tags = {
    environment = "production"
  }
}

resource "azurerm_key_vault_secret" "database_password" {
  name         = "db-password"
  value        = random_password.db_password.result
  key_vault_id = azurerm_key_vault.main.id
  
  expiration_date = "2026-12-31T23:59:59Z"
  
  tags = {
    environment = "production"
  }
}

# External Secrets Operator for Kubernetes
apiVersion: external-secrets.io/v1beta1
kind: ClusterSecretStore
metadata:
  name: aws-secrets-manager
spec:
  provider:
    aws:
      service: SecretsManager
      region: us-east-1
      auth:
        jwt:
          serviceAccountRef:
            name: external-secrets-sa
            namespace: external-secrets
---
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
  name: database-secrets
spec:
  refreshInterval: 1h
  secretStoreRef:
    name: aws-secrets-manager
    kind: ClusterSecretStore
  target:
    name: db-credentials
    creationPolicy: Owner
    data:
      - secretKey: username
        remoteRef:
          key: prod/database
          property: username
      - secretKey: password
        remoteRef:
          key: prod/database
          property: password
      - secretKey: host
        remoteRef:
          key: prod/database
          property: host
---
apiVersion: v1
kind: Secret
metadata:
  name: db-credentials
type: Opaque
stringData:
  # Populated by External Secrets Operator

# Rotation policy by secret type
rotation_policy:
  database_credentials:
    frequency: "30_days"
    auto_rotate: true
    backup_before_rotation: true
    
  api_keys:
    frequency: "90_days"
    auto_rotate: true
    notify_before: 7_days
    
  tls_certificates:
    frequency: "auto"
    auto_rotate: true
    renew_before: 30_days
    
  ssh_keys:
    frequency: "180_days"
    auto_rotate: true
    notify_before: 14_days

# Vault audit logs
resource "vault_audit" "file" {
  type_file = "file"
  options = {
    path = "/var/log/vault/audit.log"
  }
}

resource "vault_audit" "s3" {
  type_s3 = "s3"
  options = {
    bucket = "vault-audit-logs"
    region = "us-east-1"
  }
}