Configuration Management: Building Flexible and Maintainable Applications

Proper configuration management is essential for building flexible, maintainable applications that work across different environments. This guide covers practical patterns for managing application configuration.

Environment Variables

Basic Environment Variable Usage

import os
from typing import Optional

def get_config_from_env():
    """Load configuration from environment variables."""
    config = {
        'debug': os.getenv('DEBUG', 'False').lower() == 'true',
        'database_url': os.getenv('DATABASE_URL'),
        'api_key': os.getenv('API_KEY'),
        'port': int(os.getenv('PORT', 8000)),
        'log_level': os.getenv('LOG_LEVEL', 'INFO')
    }
    
    return config

# Usage
config = get_config_from_env()
print(f"Debug mode: {config['debug']}")
print(f"Port: {config['port']}")

Safe Environment Variable Access

def get_required_env(key: str) -> str:
    """Get required environment variable."""
    value = os.getenv(key)
    if value is None:
        raise ValueError(f"Required environment variable not set: {key}")
    return value

def get_optional_env(key: str, default: str = None) -> Optional[str]:
    """Get optional environment variable with default."""
    return os.getenv(key, default)

def get_env_as_type(key: str, type_func, default=None):
    """Get environment variable as specific type."""
    value = os.getenv(key)
    if value is None:
        return default
    
    try:
        return type_func(value)
    except ValueError as e:
        raise ValueError(f"Invalid value for {key}: {e}")

# Usage
try:
    api_key = get_required_env('API_KEY')
    port = get_env_as_type('PORT', int, default=8000)
    debug = get_env_as_type('DEBUG', lambda x: x.lower() == 'true', default=False)
except ValueError as e:
    print(f"Configuration error: {e}")

Using .env Files

Loading .env with python-dotenv

from dotenv import load_dotenv, find_dotenv
import os

def load_env_file(env_file=None):
    """Load environment variables from .env file."""
    if env_file is None:
        env_file = find_dotenv()
    
    if not env_file:
        print("No .env file found")
        return False
    
    load_dotenv(env_file)
    print(f"Loaded environment from: {env_file}")
    return True

# Usage
load_env_file()
database_url = os.getenv('DATABASE_URL')

.env File Example

# .env file
DEBUG=True
DATABASE_URL=postgresql://user:password@localhost/dbname
API_KEY=your-secret-api-key
PORT=8000
LOG_LEVEL=DEBUG
REDIS_URL=redis://localhost:6379

Configuration Files

YAML Configuration

import yaml
from pathlib import Path
from typing import Dict, Any

def load_yaml_config(config_file: str) -> Dict[str, Any]:
    """Load configuration from YAML file."""
    path = Path(config_file)
    
    if not path.exists():
        raise FileNotFoundError(f"Config file not found: {config_file}")
    
    with open(path, 'r') as f:
        config = yaml.safe_load(f)
    
    return config

def save_yaml_config(config: Dict[str, Any], config_file: str):
    """Save configuration to YAML file."""
    path = Path(config_file)
    path.parent.mkdir(parents=True, exist_ok=True)
    
    with open(path, 'w') as f:
        yaml.dump(config, f, default_flow_style=False)

# Usage
config = load_yaml_config('config.yaml')
print(f"Database: {config['database']['url']}")

YAML Configuration Example

# config.yaml
app:
  name: MyApp
  debug: true
  port: 8000

database:
  url: postgresql://user:password@localhost/dbname
  pool_size: 10
  echo: false

logging:
  level: DEBUG
  format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s"

redis:
  url: redis://localhost:6379
  db: 0

JSON Configuration

import json

def load_json_config(config_file: str) -> Dict[str, Any]:
    """Load configuration from JSON file."""
    with open(config_file, 'r') as f:
        config = json.load(f)
    return config

def save_json_config(config: Dict[str, Any], config_file: str):
    """Save configuration to JSON file."""
    with open(config_file, 'w') as f:
        json.dump(config, f, indent=2)

# Usage
config = load_json_config('config.json')

Pydantic Settings

Type-Safe Configuration

from pydantic import BaseSettings, Field, validator
from typing import Optional

class DatabaseSettings(BaseSettings):
    """Database configuration."""
    url: str
    pool_size: int = 10
    echo: bool = False
    
    class Config:
        env_prefix = "DB_"

class AppSettings(BaseSettings):
    """Application settings."""
    name: str = "MyApp"
    debug: bool = False
    port: int = 8000
    api_key: str
    database: DatabaseSettings = DatabaseSettings()
    
    @validator('port')
    def port_must_be_valid(cls, v):
        if not 1 <= v <= 65535:
            raise ValueError('Port must be between 1 and 65535')
        return v
    
    class Config:
        env_file = '.env'
        case_sensitive = False

# Usage
try:
    settings = AppSettings()
    print(f"App: {settings.name}")
    print(f"Port: {settings.port}")
except Exception as e:
    print(f"Configuration error: {e}")

Nested Configuration

from pydantic import BaseSettings, HttpUrl

class LoggingSettings(BaseSettings):
    level: str = "INFO"
    format: str = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"

class CacheSettings(BaseSettings):
    enabled: bool = True
    ttl: int = 3600
    backend: str = "redis"

class Settings(BaseSettings):
    """Complete application settings."""
    app_name: str
    debug: bool = False
    database_url: str
    logging: LoggingSettings = LoggingSettings()
    cache: CacheSettings = CacheSettings()
    
    class Config:
        env_file = '.env'
        env_nested_delimiter = '__'

# Usage
settings = Settings(
    app_name="MyApp",
    database_url="postgresql://localhost/db"
)

Secrets Management

Using Environment Variables for Secrets

import os
from typing import Optional

class SecretsManager:
    """Manage sensitive configuration."""
    
    @staticmethod
    def get_secret(key: str, default: Optional[str] = None) -> str:
        """Get secret from environment."""
        value = os.getenv(key)
        if value is None:
            if default is None:
                raise ValueError(f"Secret not found: {key}")
            return default
        return value
    
    @staticmethod
    def validate_secrets(required_keys: list) -> bool:
        """Validate all required secrets are set."""
        missing = []
        for key in required_keys:
            if key not in os.environ:
                missing.append(key)
        
        if missing:
            raise ValueError(f"Missing required secrets: {missing}")
        return True

# Usage
try:
    SecretsManager.validate_secrets(['API_KEY', 'DATABASE_PASSWORD'])
    api_key = SecretsManager.get_secret('API_KEY')
except ValueError as e:
    print(f"Error: {e}")

Using python-dotenv with .env.local

from pathlib import Path

def load_secrets():
    """Load secrets from .env.local (not committed to git)."""
    env_local = Path('.env.local')
    
    if env_local.exists():
        load_dotenv(env_local)
        print("Loaded secrets from .env.local")
    else:
        print("Warning: .env.local not found")

# Usage
load_secrets()

Configuration Profiles

Environment-Specific Configuration

from enum import Enum
from typing import Dict, Any

class Environment(str, Enum):
    DEVELOPMENT = "development"
    STAGING = "staging"
    PRODUCTION = "production"

class ConfigManager:
    """Manage environment-specific configuration."""
    
    CONFIGS = {
        Environment.DEVELOPMENT: {
            'debug': True,
            'database_url': 'sqlite:///dev.db',
            'log_level': 'DEBUG'
        },
        Environment.STAGING: {
            'debug': False,
            'database_url': 'postgresql://staging-db',
            'log_level': 'INFO'
        },
        Environment.PRODUCTION: {
            'debug': False,
            'database_url': 'postgresql://prod-db',
            'log_level': 'WARNING'
        }
    }
    
    @classmethod
    def get_config(cls, env: Environment = None) -> Dict[str, Any]:
        """Get configuration for environment."""
        if env is None:
            env = Environment(os.getenv('ENVIRONMENT', 'development'))
        
        config = cls.CONFIGS[env].copy()
        
        # Override with environment variables
        config.update({
            k: os.getenv(k, v) 
            for k, v in config.items()
        })
        
        return config

# Usage
config = ConfigManager.get_config(Environment.PRODUCTION)
print(f"Debug: {config['debug']}")

Configuration Validation

Schema Validation

from pydantic import BaseModel, validator, Field
from typing import Optional

class DatabaseConfig(BaseModel):
    """Database configuration with validation."""
    url: str
    pool_size: int = Field(default=10, ge=1, le=100)
    timeout: int = Field(default=30, ge=1)
    
    @validator('url')
    def validate_url(cls, v):
        if not v.startswith(('postgresql://', 'mysql://', 'sqlite://')):
            raise ValueError('Invalid database URL')
        return v

class AppConfig(BaseModel):
    """Application configuration."""
    name: str = Field(..., min_length=1)
    port: int = Field(default=8000, ge=1, le=65535)
    debug: bool = False
    database: DatabaseConfig

# Usage
try:
    config = AppConfig(
        name="MyApp",
        port=8000,
        database=DatabaseConfig(
            url="postgresql://localhost/db",
            pool_size=20
        )
    )
    print(f"Config valid: {config}")
except Exception as e:
    print(f"Validation error: {e}")

Dynamic Configuration

Configuration Reloading

import time
from pathlib import Path
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler

class ConfigReloader(FileSystemEventHandler):
    """Reload configuration when file changes."""
    
    def __init__(self, config_file: str, callback):
        self.config_file = Path(config_file)
        self.callback = callback
    
    def on_modified(self, event):
        if Path(event.src_path) == self.config_file:
            print(f"Configuration changed: {event.src_path}")
            self.callback()

def watch_config(config_file: str, callback):
    """Watch configuration file for changes."""
    observer = Observer()
    handler = ConfigReloader(config_file, callback)
    
    observer.schedule(handler, str(Path(config_file).parent))
    observer.start()
    
    return observer

# Usage
def on_config_change():
    print("Reloading configuration...")
    # Reload config here

# observer = watch_config('config.yaml', on_config_change)

Common Pitfalls and Best Practices

❌ Bad: Hardcoded Configuration

# DON'T: Hardcode configuration
DATABASE_URL = "postgresql://user:password@localhost/db"
API_KEY = "secret-key-123"

✅ Good: Use Environment Variables

# DO: Use environment variables
DATABASE_URL = os.getenv('DATABASE_URL')
API_KEY = os.getenv('API_KEY')

❌ Bad: Committing Secrets

# DON'T: Commit .env file with secrets
# .env (committed to git)
API_KEY=secret-key-123

✅ Good: Use .gitignore

# DO: Add .env to .gitignore
.env
.env.local
secrets/

❌ Bad: No Validation

# DON'T: Use configuration without validation
port = os.getenv('PORT')

✅ Good: Validate Configuration

# DO: Validate configuration
port = int(os.getenv('PORT', 8000))
if not 1 <= port <= 65535:
    raise ValueError('Invalid port')

Summary

Effective configuration management requires:

Environment variables for sensitive data
Configuration files for complex settings
Type-safe settings with Pydantic
Secrets management for sensitive values
Environment-specific profiles for different deployments
Configuration validation to catch errors early
Dynamic reloading for runtime updates
Clear separation between code and configuration

These patterns ensure flexible, secure, and maintainable application configuration across different environments.