Distributed Tracing with Jaeger in Go

Introduction

Distributed tracing tracks requests across microservices, providing visibility into system behavior. This guide covers implementing distributed tracing with Jaeger in Go.

Jaeger helps you understand request flows, identify bottlenecks, and debug issues in distributed systems by tracking spans across services.

Jaeger Fundamentals

Trace and Span Concepts

package main

import (
	"context"
	"fmt"
	"time"

	"github.com/opentracing/opentracing-go"
	"github.com/uber/jaeger-client-go"
	"github.com/uber/jaeger-client-go/config"
)

// InitJaeger initializes Jaeger tracer
func InitJaeger(serviceName string) (opentracing.Tracer, error) {
	cfg := &config.Configuration{
		ServiceName: serviceName,
		Sampler: &config.SamplerConfig{
			Type:  "const",
			Param: 1,
		},
		Reporter: &config.ReporterConfig{
			LogSpans: true,
		},
	}

	tracer, _, err := cfg.NewTracer()
	if err != nil {
		return nil, fmt.Errorf("failed to initialize tracer: %w", err)
	}

	return tracer, nil
}

// TraceExample demonstrates basic tracing
func TraceExample(tracer opentracing.Tracer) {
	// Create root span
	span := tracer.StartSpan("operation")
	defer span.Finish()

	// Create child span
	childSpan := tracer.StartSpan(
		"child-operation",
		opentracing.ChildOf(span.Context()),
	)
	defer childSpan.Finish()

	// Add tags
	span.SetTag("component", "my-service")
	span.SetTag("http.method", "GET")
	span.SetTag("http.url", "/api/users")

	// Add logs
	span.LogKV(
		"event", "request_received",
		"timestamp", time.Now(),
	)

	childSpan.LogKV(
		"event", "processing",
		"duration_ms", 100,
	)
}

Good: Proper Distributed Tracing Implementation

package main

import (
	"context"
	"fmt"
	"io"
	"log"
	"net/http"
	"time"

	"github.com/opentracing/opentracing-go"
	"github.com/opentracing/opentracing-go/ext"
	"github.com/uber/jaeger-client-go"
	"github.com/uber/jaeger-client-go/config"
)

// TracedHTTPClient wraps HTTP client with tracing
type TracedHTTPClient struct {
	client *http.Client
	tracer opentracing.Tracer
}

// NewTracedHTTPClient creates a new traced HTTP client
func NewTracedHTTPClient(tracer opentracing.Tracer) *TracedHTTPClient {
	return &TracedHTTPClient{
		client: &http.Client{
			Timeout: 5 * time.Second,
		},
		tracer: tracer,
	}
}

// Do performs an HTTP request with tracing
func (c *TracedHTTPClient) Do(ctx context.Context, req *http.Request) (*http.Response, error) {
	// Get span from context or create new one
	span, ctx := opentracing.StartSpanFromContext(ctx, "http-request")
	defer span.Finish()

	// Set span tags
	ext.HTTPMethod.Set(span, req.Method)
	ext.HTTPUrl.Set(span, req.URL.String())
	ext.Component.Set(span, "http-client")

	// Inject trace context into request headers
	if err := c.tracer.Inject(span.Context(), opentracing.HTTPHeaders, opentracing.HTTPHeadersCarrier(req.Header)); err != nil {
		log.Printf("Failed to inject trace context: %v", err)
	}

	// Perform request
	resp, err := c.client.Do(req)

	// Set response status
	if resp != nil {
		ext.HTTPStatusCode.Set(span, uint16(resp.StatusCode))
	}

	// Set error if occurred
	if err != nil {
		ext.Error.Set(span, true)
		span.LogKV("error", err.Error())
	}

	return resp, err
}

// UserService demonstrates service with tracing
type UserService struct {
	tracer opentracing.Tracer
	client *TracedHTTPClient
}

// NewUserService creates a new user service
func NewUserService(tracer opentracing.Tracer) *UserService {
	return &UserService{
		tracer: tracer,
		client: NewTracedHTTPClient(tracer),
	}
}

// GetUser retrieves a user with tracing
func (s *UserService) GetUser(ctx context.Context, userID string) (map[string]interface{}, error) {
	span, ctx := opentracing.StartSpanFromContext(ctx, "GetUser")
	defer span.Finish()

	span.SetTag("user.id", userID)

	// Simulate database call
	span.LogKV("event", "database_query_start")
	time.Sleep(100 * time.Millisecond)
	span.LogKV("event", "database_query_end")

	user := map[string]interface{}{
		"id":    userID,
		"name":  "John Doe",
		"email": "[email protected]",
	}

	span.SetTag("user.found", true)
	return user, nil
}

// CreateUser creates a user with tracing
func (s *UserService) CreateUser(ctx context.Context, name, email string) (map[string]interface{}, error) {
	span, ctx := opentracing.StartSpanFromContext(ctx, "CreateUser")
	defer span.Finish()

	span.SetTag("user.name", name)
	span.SetTag("user.email", email)

	// Validate input
	validateSpan, _ := opentracing.StartSpanFromContext(ctx, "ValidateInput")
	if name == "" || email == "" {
		ext.Error.Set(validateSpan, true)
		validateSpan.LogKV("error", "invalid input")
		validateSpan.Finish()
		return nil, fmt.Errorf("invalid input")
	}
	validateSpan.Finish()

	// Save to database
	saveSpan, _ := opentracing.StartSpanFromContext(ctx, "SaveToDatabase")
	time.Sleep(50 * time.Millisecond)
	saveSpan.Finish()

	user := map[string]interface{}{
		"id":    "user-123",
		"name":  name,
		"email": email,
	}

	return user, nil
}

// HTTP Handler with tracing
func (s *UserService) HandleGetUser(w http.ResponseWriter, r *http.Request) {
	// Extract trace context from request
	spanCtx, _ := s.tracer.Extract(opentracing.HTTPHeaders, opentracing.HTTPHeadersCarrier(r.Header))
	span := s.tracer.StartSpan("HandleGetUser", ext.RPCServerOption(spanCtx))
	defer span.Finish()

	ctx := opentracing.ContextWithSpan(r.Context(), span)

	userID := r.URL.Query().Get("id")
	user, err := s.GetUser(ctx, userID)

	if err != nil {
		ext.Error.Set(span, true)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.Header().Set("Content-Type", "application/json")
	fmt.Fprintf(w, "%v", user)
}

Bad: Improper Tracing Implementation

package main

import (
	"context"
	"net/http"
)

// BAD: No tracing
type BadUserService struct{}

// BAD: No span creation
func (s *BadUserService) GetUser(ctx context.Context, userID string) (map[string]interface{}, error) {
	// No tracing
	// No span tags
	// No error logging
	return map[string]interface{}{}, nil
}

// BAD: No trace context propagation
func (s *BadUserService) HandleGetUser(w http.ResponseWriter, r *http.Request) {
	// No trace extraction
	// No span creation
	// No context propagation
	w.WriteHeader(http.StatusOK)
}

Problems:

No tracing implementation
No span creation
No trace context propagation
No error tracking

Advanced Tracing Patterns

Baggage and Context Propagation

package main

import (
	"context"

	"github.com/opentracing/opentracing-go"
)

// SetBaggage sets baggage in span
func SetBaggage(span opentracing.Span, key, value string) {
	span.SetBaggageItem(key, value)
}

// GetBaggage gets baggage from span
func GetBaggage(span opentracing.Span, key string) string {
	return span.BaggageItem(key)
}

// PropagateContext propagates context across services
func PropagateContext(ctx context.Context, tracer opentracing.Tracer) context.Context {
	span := opentracing.SpanFromContext(ctx)
	if span == nil {
		return ctx
	}

	// Set baggage items
	span.SetBaggageItem("user_id", "user-123")
	span.SetBaggageItem("request_id", "req-456")

	return ctx
}

Sampling Strategies

package main

import (
	"github.com/uber/jaeger-client-go/config"
)

// ConstSampler samples all traces
func ConstSampler() *config.SamplerConfig {
	return &config.SamplerConfig{
		Type:  "const",
		Param: 1, // Sample all
	}
}

// ProbabilisticSampler samples a percentage of traces
func ProbabilisticSampler(rate float64) *config.SamplerConfig {
	return &config.SamplerConfig{
		Type:  "probabilistic",
		Param: rate, // 0.1 = 10%
	}
}

// RateLimitingSampler limits traces per second
func RateLimitingSampler(tracesPerSecond float64) *config.SamplerConfig {
	return &config.SamplerConfig{
		Type:  "rate_limiting",
		Param: tracesPerSecond,
	}
}

Metrics from Traces

package main

import (
	"context"
	"time"

	"github.com/opentracing/opentracing-go"
)

// TraceMetrics extracts metrics from traces
type TraceMetrics struct {
	OperationName string
	Duration      time.Duration
	Success       bool
	ErrorMessage  string
}

// ExtractMetrics extracts metrics from span
func ExtractMetrics(span opentracing.Span) TraceMetrics {
	return TraceMetrics{
		OperationName: span.OperationName(),
		Success:       true,
	}
}

// RecordMetrics records metrics
func RecordMetrics(ctx context.Context, tracer opentracing.Tracer, operation string, duration time.Duration, err error) {
	span, _ := opentracing.StartSpanFromContext(ctx, operation)
	defer span.Finish()

	span.SetTag("duration_ms", duration.Milliseconds())
	if err != nil {
		span.SetTag("error", true)
		span.LogKV("error", err.Error())
	}
}

Best Practices

1. Consistent Naming

// Use consistent operation names
span := tracer.StartSpan("service.operation")

2. Meaningful Tags

// Add meaningful tags for filtering
span.SetTag("user.id", userID)
span.SetTag("http.status_code", 200)
span.SetTag("db.query_time_ms", 50)

3. Error Tracking

// Always track errors
if err != nil {
	ext.Error.Set(span, true)
	span.LogKV("error", err.Error())
}

4. Sampling Strategy

// Choose appropriate sampling strategy
// Use const for development, probabilistic for production

Common Pitfalls

1. No Context Propagation

Always propagate trace context across service boundaries.

2. Missing Error Tags

Always tag errors for visibility.

3. Excessive Spans

Don’t create too many spans; it impacts performance.

4. No Sampling

Use sampling in production to manage overhead.

Resources

Summary

Distributed tracing provides visibility into microservices. Key takeaways:

Initialize Jaeger tracer for your service
Create spans for operations
Propagate trace context across services
Add meaningful tags and logs
Track errors in spans
Use appropriate sampling strategies
Monitor trace metrics

By implementing distributed tracing, you can effectively debug and optimize your microservices.