Readers, Writers, and Buffers in Go

The io package is fundamental to Go’s design philosophy. Readers and Writers are interfaces that enable composable I/O operations. This guide covers practical techniques for working with readers, writers, and buffers.

Understanding Readers and Writers

Reader Interface

package main

import (
	"fmt"
	"io"
	"strings"
)

func main() {
	// Reader interface: Read(p []byte) (n int, err error)
	reader := strings.NewReader("Hello, World!")

	// Read into buffer
	buffer := make([]byte, 5)
	n, err := reader.Read(buffer)
	if err != nil {
		fmt.Println("Error:", err)
	}

	fmt.Printf("Read %d bytes: %s\n", n, string(buffer[:n]))
}

Writer Interface

package main

import (
	"fmt"
	"os"
)

func main() {
	// Writer interface: Write(p []byte) (n int, err error)
	writer := os.Stdout

	// Write data
	n, err := writer.Write([]byte("Hello, World!\n"))
	if err != nil {
		fmt.Println("Error:", err)
	}

	fmt.Printf("Wrote %d bytes\n", n)
}

Working with Buffers

Buffered Reader

package main

import (
	"bufio"
	"fmt"
	"strings"
)

func main() {
	reader := strings.NewReader("Line 1\nLine 2\nLine 3")
	buffered := bufio.NewReader(reader)

	// Read line by line
	for {
		line, err := buffered.ReadString('\n')
		if err != nil {
			break
		}
		fmt.Print(line)
	}
}

Buffered Writer

package main

import (
	"bufio"
	"fmt"
	"os"
)

func main() {
	writer := bufio.NewWriter(os.Stdout)

	// Write data
	writer.WriteString("Hello, ")
	writer.WriteString("World!\n")

	// Flush to ensure data is written
	writer.Flush()
}

Buffer Pool

package main

import (
	"bytes"
	"fmt"
	"sync"
)

func main() {
	// Create a buffer pool
	pool := sync.Pool{
		New: func() interface{} {
			return new(bytes.Buffer)
		},
	}

	// Get buffer from pool
	buf := pool.Get().(*bytes.Buffer)
	defer pool.Put(buf)

	// Use buffer
	buf.WriteString("Hello, World!")
	fmt.Println(buf.String())

	// Reset for reuse
	buf.Reset()
}

Composing Readers and Writers

Chaining Operations

package main

import (
	"compress/gzip"
	"fmt"
	"io"
	"strings"
)

func main() {
	// Create a reader
	reader := strings.NewReader("Hello, World!")

	// Chain with gzip compression
	pr, pw := io.Pipe()

	go func() {
		gzipWriter := gzip.NewWriter(pw)
		io.Copy(gzipWriter, reader)
		gzipWriter.Close()
		pw.Close()
	}()

	// Read compressed data
	gzipReader, _ := gzip.NewReader(pr)
	io.Copy(os.Stdout, gzipReader)
}

MultiReader

package main

import (
	"fmt"
	"io"
	"strings"
)

func main() {
	reader1 := strings.NewReader("Hello, ")
	reader2 := strings.NewReader("World!")

	// Combine multiple readers
	combined := io.MultiReader(reader1, reader2)

	// Read from combined reader
	buffer := make([]byte, 100)
	n, _ := combined.Read(buffer)
	fmt.Println(string(buffer[:n]))
}

MultiWriter

package main

import (
	"bytes"
	"fmt"
	"io"
	"os"
)

func main() {
	var buf bytes.Buffer

	// Write to multiple destinations
	multi := io.MultiWriter(os.Stdout, &buf)

	fmt.Fprintf(multi, "Hello, World!\n")

	fmt.Println("Buffer contents:", buf.String())
}

Practical Examples

Copy with Progress

package main

import (
	"fmt"
	"io"
	"strings"
)

type ProgressWriter struct {
	Total   int64
	Written int64
}

func (pw *ProgressWriter) Write(p []byte) (n int, err error) {
	n = len(p)
	pw.Written += int64(n)
	percent := (pw.Written * 100) / pw.Total
	fmt.Printf("Progress: %d%%\n", percent)
	return n, nil
}

func main() {
	source := strings.NewReader("Hello, World!")
	dest := &ProgressWriter{Total: 13}

	io.Copy(dest, source)
}

Tee Reader

package main

import (
	"bytes"
	"fmt"
	"io"
	"strings"
)

func main() {
	source := strings.NewReader("Hello, World!")
	var buf bytes.Buffer

	// Tee: read from source and write to buffer
	tee := io.TeeReader(source, &buf)

	// Read from tee
	data := make([]byte, 100)
	n, _ := tee.Read(data)

	fmt.Println("Read:", string(data[:n]))
	fmt.Println("Tee buffer:", buf.String())
}

Limited Reader

package main

import (
	"fmt"
	"io"
	"strings"
)

func main() {
	source := strings.NewReader("Hello, World!")

	// Limit to 5 bytes
	limited := io.LimitedReader{R: source, N: 5}

	data := make([]byte, 100)
	n, _ := limited.Read(data)

	fmt.Println("Read:", string(data[:n]))
}

Pipe Reader/Writer

package main

import (
	"fmt"
	"io"
	"time"
)

func main() {
	reader, writer := io.Pipe()

	// Write in goroutine
	go func() {
		for i := 1; i <= 3; i++ {
			fmt.Fprintf(writer, "Message %d\n", i)
			time.Sleep(100 * time.Millisecond)
		}
		writer.Close()
	}()

	// Read in main goroutine
	data := make([]byte, 100)
	for {
		n, err := reader.Read(data)
		if err == io.EOF {
			break
		}
		fmt.Print(string(data[:n]))
	}
}

Best Practices

✅ Good Practices

// Use io.Copy for efficient copying
io.Copy(destination, source)

// Use buffered I/O for performance
buffered := bufio.NewReader(file)

// Compose readers and writers
combined := io.MultiReader(r1, r2)

// Close resources properly
defer file.Close()

// Use io.ReadAll for small data
data, err := io.ReadAll(reader)

// Use io.ReadFull for exact amount
io.ReadFull(reader, buffer)

❌ Anti-Patterns

// Don't read entire files into memory
data, _ := ioutil.ReadAll(largeFile)

// Don't ignore errors
io.Copy(dst, src) // Should check error

// Don't forget to close
file, _ := os.Open("file.txt")
// Missing defer file.Close()

// Don't use unbuffered I/O for large data
// Use bufio for better performance

Resources

Summary

Readers, Writers, and Buffers are core to Go’s I/O:

Use Reader/Writer interfaces for composability
Use buffered I/O for performance
Compose operations with MultiReader/MultiWriter
Use io.Copy for efficient data transfer
Close resources with defer
Handle errors properly

With these tools, you can build efficient, composable I/O operations in Go.