Readers, Writers, and Buffers in Go

The io package is fundamental to Go’s design philosophy. Readers and Writers are interfaces that enable composable I/O operations. This guide covers practical techniques for working with readers, writers, and buffers. For more context, see Go Installation Guide, Go Ecosystem Overview, Go Best Practices.

Understanding Readers and Writers

Reader Interface

package main

import (
	"fmt"
	"io"
	"strings"
)

func main() {
	// Reader interface: Read(p []byte) (n int, err error)
	reader := strings.NewReader("Hello, World!")

	// Read into buffer
	buffer := make([]byte, 5)
	n, err := reader.Read(buffer)
	if err != nil {
		fmt.Println("Error:", err)
	}

	fmt.Printf("Read %d bytes: %s\n", n, string(buffer[:n]))
}

Writer Interface

package main

import (
	"fmt"
	"os"
)

func main() {
	// Writer interface: Write(p []byte) (n int, err error)
	writer := os.Stdout

	// Write data
	n, err := writer.Write([]byte("Hello, World!\n"))
	if err != nil {
		fmt.Println("Error:", err)
	}

	fmt.Printf("Wrote %d bytes\n", n)
}

Working with Buffers

Buffered Reader

package main

import (
	"bufio"
	"fmt"
	"strings"
)

func main() {
	reader := strings.NewReader("Line 1\nLine 2\nLine 3")
	buffered := bufio.NewReader(reader)

	// Read line by line
	for {
		line, err := buffered.ReadString('\n')
		if err != nil {
			break
		}
		fmt.Print(line)
	}
}

Buffered Writer

package main

import (
	"bufio"
	"fmt"
	"os"
)

func main() {
	writer := bufio.NewWriter(os.Stdout)

	// Write data
	writer.WriteString("Hello, ")
	writer.WriteString("World!\n")

	// Flush to ensure data is written
	writer.Flush()
}

Buffer Pool

package main

import (
	"bytes"
	"fmt"
	"sync"
)

func main() {
	// Create a buffer pool
	pool := sync.Pool{
		New: func() interface{} {
			return new(bytes.Buffer)
		},
	}

	// Get buffer from pool
	buf := pool.Get().(*bytes.Buffer)
	defer pool.Put(buf)

	// Use buffer
	buf.WriteString("Hello, World!")
	fmt.Println(buf.String())

	// Reset for reuse
	buf.Reset()
}

Composing Readers and Writers

Chaining Operations

package main

import (
	"compress/gzip"
	"fmt"
	"io"
	"strings"
)

func main() {
	// Create a reader
	reader := strings.NewReader("Hello, World!")

	// Chain with gzip compression
	pr, pw := io.Pipe()

	go func() {
		gzipWriter := gzip.NewWriter(pw)
		io.Copy(gzipWriter, reader)
		gzipWriter.Close()
		pw.Close()
	}()

	// Read compressed data
	gzipReader, _ := gzip.NewReader(pr)
	io.Copy(os.Stdout, gzipReader)
}

MultiReader

package main

import (
	"fmt"
	"io"
	"strings"
)

func main() {
	reader1 := strings.NewReader("Hello, ")
	reader2 := strings.NewReader("World!")

	// Combine multiple readers
	combined := io.MultiReader(reader1, reader2)

	// Read from combined reader
	buffer := make([]byte, 100)
	n, _ := combined.Read(buffer)
	fmt.Println(string(buffer[:n]))
}

MultiWriter

package main

import (
	"bytes"
	"fmt"
	"io"
	"os"
)

func main() {
	var buf bytes.Buffer

	// Write to multiple destinations
	multi := io.MultiWriter(os.Stdout, &buf)

	fmt.Fprintf(multi, "Hello, World!\n")

	fmt.Println("Buffer contents:", buf.String())
}

Practical Examples

Copy with Progress

package main

import (
	"fmt"
	"io"
	"strings"
)

type ProgressWriter struct {
	Total   int64
	Written int64
}

func (pw *ProgressWriter) Write(p []byte) (n int, err error) {
	n = len(p)
	pw.Written += int64(n)
	percent := (pw.Written * 100) / pw.Total
	fmt.Printf("Progress: %d%%\n", percent)
	return n, nil
}

func main() {
	source := strings.NewReader("Hello, World!")
	dest := &ProgressWriter{Total: 13}

	io.Copy(dest, source)
}

Tee Reader

package main

import (
	"bytes"
	"fmt"
	"io"
	"strings"
)

func main() {
	source := strings.NewReader("Hello, World!")
	var buf bytes.Buffer

	// Tee: read from source and write to buffer
	tee := io.TeeReader(source, &buf)

	// Read from tee
	data := make([]byte, 100)
	n, _ := tee.Read(data)

	fmt.Println("Read:", string(data[:n]))
	fmt.Println("Tee buffer:", buf.String())
}

Limited Reader

package main

import (
	"fmt"
	"io"
	"strings"
)

func main() {
	source := strings.NewReader("Hello, World!")

	// Limit to 5 bytes
	limited := io.LimitedReader{R: source, N: 5}

	data := make([]byte, 100)
	n, _ := limited.Read(data)

	fmt.Println("Read:", string(data[:n]))
}

Pipe Reader/Writer

package main

import (
	"fmt"
	"io"
	"time"
)

func main() {
	reader, writer := io.Pipe()

	// Write in goroutine
	go func() {
		for i := 1; i <= 3; i++ {
			fmt.Fprintf(writer, "Message %d\n", i)
			time.Sleep(100 * time.Millisecond)
		}
		writer.Close()
	}()

	// Read in main goroutine
	data := make([]byte, 100)
	for {
		n, err := reader.Read(data)
		if err == io.EOF {
			break
		}
		fmt.Print(string(data[:n]))
	}
}

Best Practices

✅ Good Practices

// Use io.Copy for efficient copying
io.Copy(destination, source)

// Use buffered I/O for performance
buffered := bufio.NewReader(file)

// Compose readers and writers
combined := io.MultiReader(r1, r2)

// Close resources properly
defer file.Close()

// Use io.ReadAll for small data
data, err := io.ReadAll(reader)

// Use io.ReadFull for exact amount
io.ReadFull(reader, buffer)

❌ Anti-Patterns

// Don't read entire files into memory
data, _ := ioutil.ReadAll(largeFile)

// Don't ignore errors
io.Copy(dst, src) // Should check error

// Don't forget to close
file, _ := os.Open("file.txt")
// Missing defer file.Close()

// Don't use unbuffered I/O for large data
// Use bufio for better performance

Resources

Summary

Readers, Writers, and Buffers are core to Go’s I/O:

Use Reader/Writer interfaces for composability
Use buffered I/O for performance
Compose operations with MultiReader/MultiWriter
Use io.Copy for efficient data transfer
Close resources with defer
Handle errors properly

With these tools, you can build efficient, composable I/O operations in Go.