Protocol Buffers and Message Serialization

Introduction

Protocol Buffers (protobuf) is Google’s language-neutral, platform-neutral, extensible mechanism for serializing structured data. It’s more efficient than JSON or XML for many use cases, making it ideal for microservices communication. This guide covers using Protocol Buffers in Go.

Protocol Buffers provide automatic code generation, backward compatibility, and efficient binary serialization, making them perfect for high-performance systems.

Protocol Buffer Basics

Defining Messages

// product.proto
syntax = "proto3";

package product;

option go_package = "github.com/example/product/pb";

// Product represents a product
message Product {
  string id = 1;
  string name = 2;
  string description = 3;
  double price = 4;
  int32 stock = 5;
  repeated string tags = 6;
  map<string, string> attributes = 7;
}

// Order represents an order
message Order {
  string id = 1;
  string customer_id = 2;
  repeated OrderItem items = 3;
  double total = 4;
  OrderStatus status = 5;
  google.protobuf.Timestamp created_at = 6;
}

message OrderItem {
  string product_id = 1;
  int32 quantity = 2;
  double price = 3;
}

enum OrderStatus {
  UNKNOWN = 0;
  PENDING = 1;
  PROCESSING = 2;
  SHIPPED = 3;
  DELIVERED = 4;
  CANCELLED = 5;
}

Good: Proper Protocol Buffer Usage

package main

import (
	"fmt"
	"log"

	"google.golang.org/protobuf/proto"
	pb "github.com/example/product/pb"
	"google.golang.org/protobuf/types/known/timestamppb"
)

// SerializeProduct serializes a product to bytes
func SerializeProduct(product *pb.Product) ([]byte, error) {
	return proto.Marshal(product)
}

// DeserializeProduct deserializes bytes to a product
func DeserializeProduct(data []byte) (*pb.Product, error) {
	product := &pb.Product{}
	if err := proto.Unmarshal(data, product); err != nil {
		return nil, fmt.Errorf("failed to unmarshal: %w", err)
	}
	return product, nil
}

// CreateProduct creates a new product
func CreateProduct(id, name string, price float64) *pb.Product {
	return &pb.Product{
		Id:          id,
		Name:        name,
		Description: "A quality product",
		Price:       price,
		Stock:       100,
		Tags:        []string{"electronics", "gadget"},
		Attributes: map[string]string{
			"color":  "black",
			"weight": "500g",
		},
	}
}

// CreateOrder creates a new order
func CreateOrder(customerID string, items []*pb.OrderItem) *pb.Order {
	var total float64
	for _, item := range items {
		total += item.Price * float64(item.Quantity)
	}

	return &pb.Order{
		Id:         fmt.Sprintf("order-%d", time.Now().Unix()),
		CustomerId: customerID,
		Items:      items,
		Total:      total,
		Status:     pb.OrderStatus_PENDING,
		CreatedAt:  timestamppb.Now(),
	}
}

// ValidateProduct validates a product
func ValidateProduct(product *pb.Product) error {
	if product.Id == "" {
		return fmt.Errorf("product ID is required")
	}
	if product.Name == "" {
		return fmt.Errorf("product name is required")
	}
	if product.Price < 0 {
		return fmt.Errorf("product price cannot be negative")
	}
	if product.Stock < 0 {
		return fmt.Errorf("product stock cannot be negative")
	}
	return nil
}

func main() {
	// Create product
	product := CreateProduct("prod-001", "Laptop", 999.99)

	// Validate
	if err := ValidateProduct(product); err != nil {
		log.Fatal(err)
	}

	// Serialize
	data, err := SerializeProduct(product)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("Serialized size: %d bytes\n", len(data))

	// Deserialize
	deserialized, err := DeserializeProduct(data)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("Deserialized: %v\n", deserialized)

	// Create order
	items := []*pb.OrderItem{
		{
			ProductId: "prod-001",
			Quantity:  2,
			Price:     999.99,
		},
	}
	order := CreateOrder("cust-001", items)
	fmt.Printf("Order: %v\n", order)
}

Bad: Improper Protocol Buffer Usage

package main

import (
	"encoding/json"
	pb "github.com/example/product/pb"
)

// BAD: Using JSON instead of protobuf
func BadSerializeProduct(product *pb.Product) ([]byte, error) {
	// Inefficient: converting protobuf to JSON
	return json.Marshal(product)
}

// BAD: No validation
func BadCreateProduct(id, name string, price float64) *pb.Product {
	// No validation of inputs
	// No error handling
	return &pb.Product{
		Id:    id,
		Name:  name,
		Price: price,
	}
}

// BAD: No error handling
func BadDeserializeProduct(data []byte) *pb.Product {
	product := &pb.Product{}
	// Ignoring errors
	proto.Unmarshal(data, product)
	return product
}

Problems:

Using JSON instead of protobuf
No input validation
No error handling
Inefficient serialization

Advanced Message Patterns

Nested Messages

message Address {
  string street = 1;
  string city = 2;
  string state = 3;
  string zip = 4;
}

message Customer {
  string id = 1;
  string name = 2;
  Address billing_address = 3;
  Address shipping_address = 4;
  repeated Address previous_addresses = 5;
}

Oneof Fields

message PaymentMethod {
  string id = 1;
  oneof method {
    CreditCard credit_card = 2;
    BankAccount bank_account = 3;
    PayPalAccount paypal = 4;
  }
}

message CreditCard {
  string number = 1;
  string expiry = 2;
  string cvv = 3;
}

message BankAccount {
  string account_number = 1;
  string routing_number = 2;
}

message PayPalAccount {
  string email = 1;
}

Using Oneof in Go

package main

import (
	"fmt"
	pb "github.com/example/payment/pb"
)

// ProcessPayment processes a payment based on method
func ProcessPayment(payment *pb.PaymentMethod) error {
	switch method := payment.Method.(type) {
	case *pb.PaymentMethod_CreditCard:
		return processCreditCard(method.CreditCard)
	case *pb.PaymentMethod_BankAccount:
		return processBankAccount(method.BankAccount)
	case *pb.PaymentMethod_Paypal:
		return processPayPal(method.Paypal)
	default:
		return fmt.Errorf("unknown payment method")
	}
}

func processCreditCard(card *pb.CreditCard) error {
	fmt.Printf("Processing credit card: %s\n", card.Number)
	return nil
}

func processBankAccount(account *pb.BankAccount) error {
	fmt.Printf("Processing bank account: %s\n", account.AccountNumber)
	return nil
}

func processPayPal(paypal *pb.PayPalAccount) error {
	fmt.Printf("Processing PayPal: %s\n", paypal.Email)
	return nil
}

Serialization Comparison

package main

import (
	"encoding/json"
	"fmt"
	"time"

	"google.golang.org/protobuf/proto"
	pb "github.com/example/product/pb"
)

// CompareSerializationMethods compares different serialization methods
func CompareSerializationMethods() {
	product := &pb.Product{
		Id:    "prod-001",
		Name:  "Laptop",
		Price: 999.99,
		Stock: 100,
		Tags:  []string{"electronics", "gadget", "computers"},
	}

	// Protocol Buffers
	start := time.Now()
	pbData, _ := proto.Marshal(product)
	pbTime := time.Since(start)
	fmt.Printf("Protobuf - Size: %d bytes, Time: %v\n", len(pbData), pbTime)

	// JSON
	start = time.Now()
	jsonData, _ := json.Marshal(product)
	jsonTime := time.Since(start)
	fmt.Printf("JSON - Size: %d bytes, Time: %v\n", len(jsonData), jsonTime)

	// Size comparison
	fmt.Printf("Size reduction: %.1f%%\n", float64(len(jsonData)-len(pbData))/float64(len(jsonData))*100)
}

Backward Compatibility

// Version 1
message UserV1 {
  string id = 1;
  string name = 2;
  string email = 3;
}

// Version 2 - Adding new fields
message UserV2 {
  string id = 1;
  string name = 2;
  string email = 3;
  string phone = 4;  // New field
  string address = 5; // New field
}

// Version 3 - Deprecating fields
message UserV3 {
  string id = 1;
  string name = 2;
  string email = 3;
  string phone = 4;
  string address = 5;
  reserved 6; // Reserved for future use
  reserved "deprecated_field"; // Reserved field name
}

Custom Marshaling

package main

import (
	"fmt"
	"time"

	"google.golang.org/protobuf/proto"
	pb "github.com/example/product/pb"
)

// CustomMarshaler handles custom marshaling logic
type CustomMarshaler struct {
	product *pb.Product
}

// MarshalJSON implements custom JSON marshaling
func (cm *CustomMarshaler) MarshalJSON() ([]byte, error) {
	// Custom logic before marshaling
	fmt.Println("Custom marshaling logic")
	return proto.Marshal(cm.product)
}

// UnmarshalJSON implements custom JSON unmarshaling
func (cm *CustomMarshaler) UnmarshalJSON(data []byte) error {
	// Custom logic after unmarshaling
	fmt.Println("Custom unmarshaling logic")
	return proto.Unmarshal(data, cm.product)
}

Best Practices

1. Schema Versioning

Always version your schemas and maintain backward compatibility.

2. Field Numbering

Never reuse field numbers. Use reserved keywords for deprecated fields.

3. Validation

Always validate deserialized data:

func ValidateOrder(order *pb.Order) error {
	if order.Id == "" {
		return fmt.Errorf("order ID is required")
	}
	if len(order.Items) == 0 {
		return fmt.Errorf("order must have at least one item")
	}
	return nil
}

4. Error Handling

Always handle serialization/deserialization errors:

data, err := proto.Marshal(product)
if err != nil {
	return fmt.Errorf("failed to marshal: %w", err)
}

Common Pitfalls

1. Reusing Field Numbers

Never reuse field numbers as it breaks compatibility.

2. Removing Fields

Don’t remove fields; mark them as reserved instead.

3. Changing Field Types

Avoid changing field types as it breaks compatibility.

4. No Validation

Always validate data after deserialization.

Resources

Summary

Protocol Buffers provide efficient, backward-compatible message serialization. Key takeaways:

Use protobuf for efficient binary serialization
Define clear schemas with proper versioning
Never reuse field numbers
Maintain backward compatibility
Always validate deserialized data
Use appropriate field types and structures
Document your schemas thoroughly

By following these practices, you can build robust, efficient message serialization systems in Go.