Infrastructure Testing: Terraform Testing, Policy as Code

Introduction

Infrastructure as Code has transformed how we manage infrastructure, but with great power comes great responsibility. According to Gartner, 95% of cloud security failures will be customer-managed through 2025. Infrastructure testing is your safety net against misconfigurations, security vulnerabilities, and costly downtime.

Key Statistics:

70% of infrastructure failures are due to configuration errors
Average cost of cloud misconfiguration: $4.5 million per incident
Organizations with IaC testing catch 85% of issues before production
Test-driven infrastructure reduces incident rates by 60%

Testing Pyramid for Infrastructure

┌─────────────────────────────────────────────┐
│           Policy Validation                  │
│         (OPA, Sentinel, Checkov)            │
├─────────────────────────────────────────────┤
│         Integration Testing                  │
│       (Terratest, CloudMock)                │
├─────────────────────────────────────────────┤
│          Unit Testing                        │
│      (terraform validate, plan)             │
├─────────────────────────────────────────────┤
│           Linting                            │
│      (tflint, tfsec, checkov)               │
└─────────────────────────────────────────────┘

Unit Testing

Terraform Validation

# Install tfsec for security scanning
brew install tfsec

# Run tfsec on Terraform files
tfsec .

# Run with custom rules
tfsec --config tfsec.yaml .

# tfsec.yaml configuration
detectors:
  exclude:
    - "EXVD003"  # Exclude specific finding
rules:
  add:
    - id: "CUSTOM001"
      description: "S3 bucket should have versioning enabled"
      match:
        type: "resource"
        resources:
          - "aws_s3_bucket"
      severity: "ERROR"
      good_example: |
        resource "aws_s3_bucket" "example" {
          versioning {
            enabled = true
          }
        }
      bad_example: |
        resource "aws_s3_bucket" "example" {}

# Install tflint for Terraform linting
brew install tflint

# Initialize TFLint
tflint --init

# Run with specific rule set
tflint --module --filter=aws_vpc

# .tflint.hcl configuration
config {
  module = true
  force = false
}

plugin "aws" {
  enabled = true
  version = "0.29.0"
  source  = "github.com/terraform-linters/tflint-ruleset-aws"
}

rule {
  id = "aws_instance_invalid_type"
  enabled = true
}

rule {
  id = "terraform_deprecated_interpolation"
  enabled = false
}

Pre-Commit Hooks

# .pre-commit-config.yaml
repos:
  - repo: https://github.com/antonbabenko/pre-commit-hooks
    rev: v1.4.0
    hooks:
      - id: terraform_fmt
      - id: terraform_validate
        args: ['--args=-json']
      - id: terraform_tflint
        args: ['--args=--format=compact']
      - id: terraform_tfsec
        args: ['--args=--format=default']

  - repo: https://github.com/aquasecurity/tfsec-pr-commenter-action
    rev: v1.0.0
    hooks:
      - id: tfsec

  - repo: https://github.com/bridgecrewio/checkov
    rev: v3.1.50
    hooks:
      - id: checkov
        args: ['--directory', '.']

Integration Testing with Terratest

Basic Terratest Example

package test

import (
    "testing"
    "github.com/stretchr/testify/assert"
    "github.com/gruntwork-io/terratest/modules/terraform"
    "github.com/gruntwork-io/terratest/modules/random"
    "github.com/gruntwork-io/terratest/modules/aws"
)

func TestTerraformS3Bucket(t *testing.T) {
    t.Parallel()
    
    // Generate unique name
    bucketName := fmt.Sprintf("terratest-bucket-%s", random.UniqueId())
    
    terraformOptions := &terraform.Options{
        TerraformDir: "../examples/s3-bucket",
        Vars: map[string]interface{}{
            "bucket_name": bucketName,
            "enable_versioning": true,
            "tags": map[string]string{
                "Environment": "test",
                "Project":    "terratest",
            },
        },
    }
    
    // Defer destroy
    defer terraform.Destroy(t, terraformOptions)
    
    // Apply
    terraform.InitAndApply(t, terraformOptions)
    
    // Verify bucket exists
    assert.True(t, aws.GetS3BucketExists(t, bucketName, "us-east-1"))
    
    // Verify versioning is enabled
    versioning := aws.GetS3BucketVersioning(t, bucketName, "us-east-1")
    assert.True(t, versioning.Enabled)
    
    // Verify tags
    tags := aws.GetS3BucketTags(t, bucketName, "us-east-1")
    assert.Equal(t, "test", tags["Environment"])
}

Testing Kubernetes Resources

package test

import (
    "testing"
    "github.com/stretchr/testify/assert"
    "github.com/gruntwork-io/terratest/modules/k8s"
    "github.com/gruntwork-io/terratest/modules/helm"
)

func TestKubernetesDeployment(t *testing.T) {
    t.Parallel()
    
    // Define options
    options := k8s.KubectlOptions{
        ContextName: "minikube",
        Namespace:    "default",
    }
    
    // Apply manifest
    k8s.KubectlApply(t, options, "deployment.yaml")
    
    // Wait for deployment
    k8s.WaitUntilDeploymentAvailable(
        t, 
        options, 
        "myapp", 
        10, 
        30*time.Second,
    )
    
    // Verify pods are running
    pods := k8s.GetPods(t, options, map[string]string{
        "app": "myapp",
    })
    assert.Greater(t, len(pods), 0)
}

Testing Database Migrations

package test

import (
    "testing"
    "github.com/stretchr/testify/assert"
    "github.com/gruntwork-io/terratest/modules/sql"
    "github.com/gruntwork-io/terratest/modules/terraform"
)

func TestDatabaseMigration(t *testing.T) {
    terraformOptions := &terraform.Options{
        TerraformDir: "../examples/rds",
    }
    
    defer terraform.Destroy(t, terraformOptions)
    terraform.InitAndApply(t, terraformOptions)
    
    // Get database endpoint
    dbEndpoint := terraform.Output(t, terraformOptions, "db_endpoint")
    
    // Connect and run migrations
    db := sql.OpenConnection(t, "postgres", "user=admin password=pass host="+dbEndpoint)
    defer db.Close()
    
    // Verify schema
    result := db.QueryRow("SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'public'")
    var count int
    result.Scan(&count)
    assert.Greater(t, count, 0)
}

Policy as Code with OPA

OPA Gatekeeper

# Constraint template for required labels
apiVersion: templates.gatekeeper.sh/v1beta1
kind: ConstraintTemplate
metadata:
  name: k8srequiredlabels
spec:
  crd:
    spec:
      names:
        kind: K8sRequiredLabels
      validation:
        openAPIV3Schema:
          type: object
          properties:
            labels:
              type: array
              items:
                type: object
                properties:
                  key:
                    type: string
                  allowedRegex:
                    type: string
  targets:
    - target: admission.k8s.gatekeeper.sh
      rego: |
        package k8srequiredlabels
        
        violation[{"msg": msg}] {
          provided := {k | v := input.request.object.metadata.labels[k]}
          required := {k | k := input.parameters.labels[_].key}
          missing := required - provided
          count(missing) > 0
          msg := sprintf("Missing required labels: %v", [missing])
        }
        
        violation[{"msg": msg}] {
          key := input.parameters.labels[_].key
          allowedRegex := input.parameters.labels[_].allowedRegex
          value := input.request.object.metadata.labels[key]
          not regex.match(allowedRegex, value)
          msg := sprintf("Label %s value %s does not match regex %s", [key, value, allowedRegex])
        }
---
# Apply constraint
apiVersion: constraints.gatekeeper.sh/v1beta1
kind: K8sRequiredLabels
metadata:
  name: require-labels
spec:
  match:
    kinds:
      - apiGroups: ["", "apps", "batch"]
        kinds: ["Pod", "Deployment", "StatefulSet"]
  parameters:
    labels:
      - key: "environment"
        allowedRegex: "^(production|staging|development)$"
      - key: "team"
      - key: "cost-center"

Terraform Policy with OPA

# terraform.rego
package terraform

import input as tfplan

# Deny resources without tags
deny[msg] {
    resource := tfplan.resource_changes[_]
    not resource.change.after.tags
    msg := sprintf("Resource %s (%s) must have tags", [resource.address, resource.type])
}

# Require specific instance types
deny[msg] {
    resource := tfplan.resource_changes[_]
    resource.type == "aws_instance"
    not contains(["t3.micro", "t3.small", "t3.medium"], resource.change.after.instance_type)
    msg := sprintf("Instance type %s not allowed. Use t3.micro, t3.small, or t3.medium", [resource.change.after.instance_type])
}

# Require encryption for S3
deny[msg] {
    resource := tfplan.resource_changes[_]
    resource.type == "aws_s3_bucket"
    not resource.change.after.server_side_encryption_configuration
    msg := "S3 bucket must have server-side encryption enabled"
}

# Require private subnets
deny[msg] {
    resource := tfplan.resource_changes[_]
    resource.type == "aws_subnet"
    resource.change.after.map_public_ip_on_launch
    msg := "Subnets must not map public IPs"
}

# Test policy against plan
opa eval --format pretty --data terraform.rego --input plan.json "deny"

Checkov Integration

# checkov.yaml
framework:
  - terraform
  - kubernetes
  - dockerfile

skip-check:
  - CKV_AWS_123  # Skip specific check

branch-max-depth: 5

directory:
  - ./
  - ../other-terraform/

output:
  file: checkov_results.json
  format: json

# GitHub Actions
name: Checkov Scan
on: [push, pull_request]

jobs:
  scan:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: bridgecrewio/checkov-action@master
        with:
          directory: .
          framework: terraform
          output_format: sarif
          output_file_path: results.sarif

Terratest Advanced Patterns

Testing Module Composition

package test

import (
    "testing"
    "github.com/stretchr/testify/assert"
    "github.com/gruntwork-io/terratest/modules/terraform"
)

func TestModuleComposition(t *testing.T) {
    // Test VPC module with nested subnets
    terraformOptions := &terraform.Options{
        TerraformDir: "../examples/vpc-module",
        Vars: map[string]interface{}{
            "cidr_block": "10.0.0.0/16",
            "availability_zones": []string{"us-east-1a", "us-east-1b"},
            "public_subnets": []string{"10.0.1.0/24", "10.0.2.0/24"},
            "private_subnets": []string{"10.0.10.0/24", "10.0.20.0/24"},
        },
    }
    
    defer terraform.Destroy(t, terraformOptions)
    terraform.InitAndApply(t, terraformOptions)
    
    // Verify outputs
    vpcId := terraform.Output(t, terraformOptions, "vpc_id")
    assert.NotEmpty(t, vpcId)
    
    publicSubnets := terraform.OutputList(t, terraformOptions, "public_subnet_ids")
    assert.Equal(t, 2, len(publicSubnets))
    
    privateSubnets := terraform.OutputList(t, terraformOptions, "private_subnet_ids")
    assert.Equal(t, 2, len(privateSubnets))
}

Testing State Management

package test

import (
    "testing"
    "os"
    "github.com/stretchr/testify/assert"
    "github.com/gruntwork-io/terratest/modules/terraform"
)

func TestStateLocking(t *testing.T) {
    // Set up remote state
    os.Setenv("TF_STATE_BACKEND", "s3")
    
    terraformOptions := &terraform.Options{
        TerraformDir: "../examples/stateful",
        BackendConfig: map[string]interface{}{
            "bucket": os.Getenv("TF_STATE_BUCKET"),
            "key":    "test-state/terraform.tfstate",
            "region": "us-east-1",
        },
    }
    
    // Acquire lock
    lockId, err := terraform.GetLockId(t, terraformOptions)
    assert.NoError(t, err)
    assert.NotEmpty(t, lockId)
    
    defer terraform.Unlock(t, terraformOptions, lockId)
    
    terraform.Apply(t, terraformOptions)
}

CI/CD Pipeline Integration

# GitLab CI Terraform Testing
stages:
  - validate
  - test
  - security
  - plan
  - apply

variables:
  TF_STATE_BUCKET: "terraform-state-bucket"
  TF_VERSION: "1.6.0"

validate:
  stage: validate
  image: hashicorp/terraform:$TF_VERSION
  script:
    - terraform init -backend=false
    - terraform validate
    - terraform fmt -check -recursive

test:
  stage: test
  image: golang:1.21
  before_script:
    - go install github.com/gruntwork-io/terratest/cmd/terratest@latest
  script:
    - go test -v -timeout 30m ./test/...

security:
  stage: security
  image: bridgecrew/checkov:latest
  script:
    - checkov -d . --framework terraform
  allow_failure: true

plan:
  stage: plan
  image: hashicorp/terraform:$TF_VERSION
  script:
    - terraform init
    - terraform plan -out=tfplan
    - terraform show -json tfplan > plan.json
  artifacts:
    paths:
      - tfplan
      - plan.json
    expire_in: 1 week
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"