Application Monitoring and Profiling: APM, Tracing, and Node.js Profiling

npm install dd-trace

// Must be the FIRST import in your entry file
import tracer from 'dd-trace';

tracer.init({
    service: 'api-server',
    env: process.env.NODE_ENV,
    version: process.env.APP_VERSION,
    logInjection: true,  // adds trace IDs to logs
    runtimeMetrics: true, // CPU, memory, event loop
});

// Everything after this is automatically instrumented:
// - HTTP requests (Express, Fastify, Koa)
// - Database queries (pg, mysql2, mongoose)
// - Redis operations
// - External HTTP calls (axios, node-fetch)
import express from 'express';

import tracer from 'dd-trace';

async function processOrder(orderId) {
    // Create a custom span for business logic
    const span = tracer.startSpan('order.process');
    span.setTag('order.id', orderId);

    try {
        const order = await db.getOrder(orderId);
        span.setTag('order.total', order.total);

        await chargePayment(order);
        await updateInventory(order);

        span.setTag('order.status', 'completed');
        return order;
    } catch (err) {
        span.setTag('error', true);
        span.setTag('error.message', err.message);
        throw err;
    } finally {
        span.finish();
    }
}

npm install newrelic

// newrelic.js (config file)
exports.config = {
    app_name: ['My Application'],
    license_key: process.env.NEW_RELIC_LICENSE_KEY,
    logging: { level: 'info' },
    allow_all_headers: true,
    distributed_tracing: { enabled: true },
};

// Entry file — require newrelic FIRST
require('newrelic');
const express = require('express');

npm install @opentelemetry/sdk-node @opentelemetry/auto-instrumentations-node
npm install @opentelemetry/exporter-otlp-http

// tracing.js — initialize before anything else
import { NodeSDK } from '@opentelemetry/sdk-node';
import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node';
import { OTLPTraceExporter } from '@opentelemetry/exporter-otlp-http';
import { Resource } from '@opentelemetry/resources';
import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions';

const sdk = new NodeSDK({
    resource: new Resource({
        [SemanticResourceAttributes.SERVICE_NAME]: 'api-server',
        [SemanticResourceAttributes.SERVICE_VERSION]: '1.0.0',
    }),
    traceExporter: new OTLPTraceExporter({
        url: 'http://otel-collector:4318/v1/traces',
    }),
    instrumentations: [
        getNodeAutoInstrumentations({
            '@opentelemetry/instrumentation-fs': { enabled: false },  // too noisy
        }),
    ],
});

sdk.start();

process.on('SIGTERM', () => sdk.shutdown());

npm install -g clinic

# Doctor: overall health check — identifies the type of problem
clinic doctor -- node server.js

# Flame: CPU profiling — shows where time is spent
clinic flame -- node server.js

# Bubbleprof: async profiling — shows I/O bottlenecks
clinic bubbleprof -- node server.js

# Generate V8 CPU profile
node --prof server.js

# Run load test while profiling
ab -n 1000 -c 10 http://localhost:3000/api/users

# Process the profile (creates readable output)
node --prof-process isolate-*.log > profile.txt

# Look for "Heavy (bottom up)" section
head -100 profile.txt

// Detect memory leaks with periodic heap snapshots
import v8 from 'v8';
import fs from 'fs';

function takeHeapSnapshot() {
    const filename = `heap-${Date.now()}.heapsnapshot`;
    const snapshot = v8.writeHeapSnapshot(filename);
    console.log(`Heap snapshot written to ${snapshot}`);
}

// Take snapshot every 5 minutes in development
if (process.env.NODE_ENV === 'development') {
    setInterval(takeHeapSnapshot, 5 * 60 * 1000);
}

# Open in Chrome DevTools:
# DevTools → Memory → Load profile → select .heapsnapshot file
# Compare two snapshots to find what's growing

// Monitor event loop lag — high lag = blocked event loop
import { monitorEventLoopDelay } from 'perf_hooks';

const histogram = monitorEventLoopDelay({ resolution: 20 });
histogram.enable();

setInterval(() => {
    const lag = histogram.percentile(99) / 1e6;  // convert ns to ms
    if (lag > 100) {
        console.warn(`Event loop lag p99: ${lag.toFixed(2)}ms — possible blocking operation`);
    }
    histogram.reset();
}, 10000);

import { Registry, Histogram, Counter, Gauge } from 'prom-client';

const registry = new Registry();

// Track external API call performance
const externalApiDuration = new Histogram({
    name: 'external_api_duration_seconds',
    help: 'Duration of external API calls',
    labelNames: ['service', 'endpoint', 'status'],
    buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5],
    registers: [registry],
});

// Wrap external calls with timing
async function callExternalAPI(service, endpoint, fn) {
    const end = externalApiDuration.startTimer({ service, endpoint });
    try {
        const result = await fn();
        end({ status: 'success' });
        return result;
    } catch (err) {
        end({ status: 'error' });
        throw err;
    }
}

// Usage
const user = await callExternalAPI('user-service', '/users', () =>
    fetch('http://user-service/users/42').then(r => r.json())
);

npm install @sentry/node @sentry/profiling-node

import * as Sentry from '@sentry/node';
import { nodeProfilingIntegration } from '@sentry/profiling-node';

Sentry.init({
    dsn: process.env.SENTRY_DSN,
    environment: process.env.NODE_ENV,
    release: process.env.APP_VERSION,
    integrations: [
        nodeProfilingIntegration(),
    ],
    tracesSampleRate: 0.1,    // sample 10% of transactions
    profilesSampleRate: 0.1,  // sample 10% for profiling
});

// Express error handler — must be last middleware
app.use(Sentry.Handlers.errorHandler());

// Capture errors manually
try {
    riskyOperation();
} catch (err) {
    Sentry.captureException(err, {
        extra: { userId, orderId },
        tags: { component: 'payment' },
    });
    throw err;
}