File System Operations and Manipulation in Go

Introduction

File system operations are fundamental to CLI applications. Go provides powerful packages for reading, writing, and manipulating files and directories. This guide covers comprehensive file system operations.

Proper file handling ensures your applications are robust, efficient, and handle edge cases gracefully.

File Reading and Writing

Basic File Operations

package main

import (
	"fmt"
	"io"
	"log"
	"os"
)

// ReadFile reads entire file into memory
func ReadFile(filename string) ([]byte, error) {
	data, err := os.ReadFile(filename)
	if err != nil {
		return nil, fmt.Errorf("failed to read file: %w", err)
	}
	return data, nil
}

// WriteFile writes data to file
func WriteFile(filename string, data []byte) error {
	if err := os.WriteFile(filename, data, 0644); err != nil {
		return fmt.Errorf("failed to write file: %w", err)
	}
	return nil
}

// AppendToFile appends data to file
func AppendToFile(filename string, data []byte) error {
	file, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
	if err != nil {
		return fmt.Errorf("failed to open file: %w", err)
	}
	defer file.Close()

	if _, err := file.Write(data); err != nil {
		return fmt.Errorf("failed to write to file: %w", err)
	}

	return nil
}

// CopyFile copies a file
func CopyFile(src, dst string) error {
	source, err := os.Open(src)
	if err != nil {
		return fmt.Errorf("failed to open source: %w", err)
	}
	defer source.Close()

	destination, err := os.Create(dst)
	if err != nil {
		return fmt.Errorf("failed to create destination: %w", err)
	}
	defer destination.Close()

	if _, err := io.Copy(destination, source); err != nil {
		return fmt.Errorf("failed to copy: %w", err)
	}

	return nil
}

Good: Proper File System Implementation

package main

import (
	"bufio"
	"fmt"
	"io"
	"io/fs"
	"log"
	"os"
	"path/filepath"
	"strings"
)

// FileManager handles file operations
type FileManager struct {
	basePath string
}

// NewFileManager creates a new file manager
func NewFileManager(basePath string) *FileManager {
	return &FileManager{
		basePath: basePath,
	}
}

// ReadLines reads file line by line
func (fm *FileManager) ReadLines(filename string) ([]string, error) {
	path := filepath.Join(fm.basePath, filename)

	file, err := os.Open(path)
	if err != nil {
		return nil, fmt.Errorf("failed to open file: %w", err)
	}
	defer file.Close()

	var lines []string
	scanner := bufio.NewScanner(file)

	for scanner.Scan() {
		lines = append(lines, scanner.Text())
	}

	if err := scanner.Err(); err != nil {
		return nil, fmt.Errorf("scanner error: %w", err)
	}

	return lines, nil
}

// WriteLines writes lines to file
func (fm *FileManager) WriteLines(filename string, lines []string) error {
	path := filepath.Join(fm.basePath, filename)

	file, err := os.Create(path)
	if err != nil {
		return fmt.Errorf("failed to create file: %w", err)
	}
	defer file.Close()

	writer := bufio.NewWriter(file)
	for _, line := range lines {
		if _, err := writer.WriteString(line + "\n"); err != nil {
			return fmt.Errorf("failed to write line: %w", err)
		}
	}

	if err := writer.Flush(); err != nil {
		return fmt.Errorf("failed to flush: %w", err)
	}

	return nil
}

// ListFiles lists files in directory
func (fm *FileManager) ListFiles(dir string) ([]string, error) {
	path := filepath.Join(fm.basePath, dir)

	entries, err := os.ReadDir(path)
	if err != nil {
		return nil, fmt.Errorf("failed to read directory: %w", err)
	}

	var files []string
	for _, entry := range entries {
		if !entry.IsDir() {
			files = append(files, entry.Name())
		}
	}

	return files, nil
}

// ListDirectories lists directories
func (fm *FileManager) ListDirectories(dir string) ([]string, error) {
	path := filepath.Join(fm.basePath, dir)

	entries, err := os.ReadDir(path)
	if err != nil {
		return nil, fmt.Errorf("failed to read directory: %w", err)
	}

	var dirs []string
	for _, entry := range entries {
		if entry.IsDir() {
			dirs = append(dirs, entry.Name())
		}
	}

	return dirs, nil
}

// WalkDirectory walks directory tree
func (fm *FileManager) WalkDirectory(dir string, callback func(path string, info fs.FileInfo) error) error {
	path := filepath.Join(fm.basePath, dir)

	return filepath.Walk(path, func(path string, info fs.FileInfo, err error) error {
		if err != nil {
			return err
		}

		return callback(path, info)
	})
}

// CreateDirectory creates a directory
func (fm *FileManager) CreateDirectory(dir string) error {
	path := filepath.Join(fm.basePath, dir)

	if err := os.MkdirAll(path, 0755); err != nil {
		return fmt.Errorf("failed to create directory: %w", err)
	}

	return nil
}

// DeleteFile deletes a file
func (fm *FileManager) DeleteFile(filename string) error {
	path := filepath.Join(fm.basePath, filename)

	if err := os.Remove(path); err != nil {
		return fmt.Errorf("failed to delete file: %w", err)
	}

	return nil
}

// DeleteDirectory deletes a directory
func (fm *FileManager) DeleteDirectory(dir string) error {
	path := filepath.Join(fm.basePath, dir)

	if err := os.RemoveAll(path); err != nil {
		return fmt.Errorf("failed to delete directory: %w", err)
	}

	return nil
}

// FileExists checks if file exists
func (fm *FileManager) FileExists(filename string) bool {
	path := filepath.Join(fm.basePath, filename)
	_, err := os.Stat(path)
	return err == nil
}

// GetFileInfo gets file information
func (fm *FileManager) GetFileInfo(filename string) (os.FileInfo, error) {
	path := filepath.Join(fm.basePath, filename)
	return os.Stat(path)
}

// GetFileSize gets file size
func (fm *FileManager) GetFileSize(filename string) (int64, error) {
	info, err := fm.GetFileInfo(filename)
	if err != nil {
		return 0, err
	}
	return info.Size(), nil
}

// ChangePermissions changes file permissions
func (fm *FileManager) ChangePermissions(filename string, perm os.FileMode) error {
	path := filepath.Join(fm.basePath, filename)

	if err := os.Chmod(path, perm); err != nil {
		return fmt.Errorf("failed to change permissions: %w", err)
	}

	return nil
}

Bad: Improper File Operations

package main

import (
	"os"
)

// BAD: No error handling
func BadReadFile(filename string) []byte {
	data, _ := os.ReadFile(filename)
	return data
}

// BAD: No resource cleanup
func BadWriteFile(filename string, data []byte) {
	file, _ := os.Create(filename)
	// Missing defer file.Close()
	file.Write(data)
}

// BAD: No path validation
func BadDeleteFile(filename string) {
	os.Remove(filename)
}

Problems:

No error handling
No resource cleanup
No path validation
No permission checks

Advanced File Operations

Temporary Files

package main

import (
	"fmt"
	"os"
	"path/filepath"
)

// CreateTempFile creates a temporary file
func CreateTempFile(pattern string) (*os.File, error) {
	file, err := os.CreateTemp("", pattern)
	if err != nil {
		return nil, fmt.Errorf("failed to create temp file: %w", err)
	}
	return file, nil
}

// CreateTempDirectory creates a temporary directory
func CreateTempDirectory(pattern string) (string, error) {
	dir, err := os.MkdirTemp("", pattern)
	if err != nil {
		return "", fmt.Errorf("failed to create temp directory: %w", err)
	}
	return dir, nil
}

// WithTempFile executes function with temporary file
func WithTempFile(pattern string, fn func(*os.File) error) error {
	file, err := CreateTempFile(pattern)
	if err != nil {
		return err
	}
	defer os.Remove(file.Name())

	return fn(file)
}

File Watching

package main

import (
	"fmt"
	"log"
	"os"
	"time"
)

// FileWatcher watches for file changes
type FileWatcher struct {
	filename string
	lastMod  time.Time
	interval time.Duration
}

// NewFileWatcher creates a new file watcher
func NewFileWatcher(filename string, interval time.Duration) *FileWatcher {
	return &FileWatcher{
		filename: filename,
		interval: interval,
	}
}

// Watch watches for file changes
func (fw *FileWatcher) Watch(callback func()) error {
	info, err := os.Stat(fw.filename)
	if err != nil {
		return err
	}

	fw.lastMod = info.ModTime()

	ticker := time.NewTicker(fw.interval)
	defer ticker.Stop()

	for range ticker.C {
		info, err := os.Stat(fw.filename)
		if err != nil {
			return err
		}

		if info.ModTime().After(fw.lastMod) {
			fw.lastMod = info.ModTime()
			callback()
		}
	}

	return nil
}

Atomic File Operations

package main

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
)

// AtomicWrite performs atomic file write
func AtomicWrite(filename string, data []byte) error {
	// Write to temporary file
	tmpFile, err := os.CreateTemp(filepath.Dir(filename), ".tmp")
	if err != nil {
		return fmt.Errorf("failed to create temp file: %w", err)
	}
	defer os.Remove(tmpFile.Name())

	if _, err := tmpFile.Write(data); err != nil {
		tmpFile.Close()
		return fmt.Errorf("failed to write: %w", err)
	}

	if err := tmpFile.Close(); err != nil {
		return fmt.Errorf("failed to close temp file: %w", err)
	}

	// Atomic rename
	if err := os.Rename(tmpFile.Name(), filename); err != nil {
		return fmt.Errorf("failed to rename: %w", err)
	}

	return nil
}

Best Practices

1. Always Close Files

file, err := os.Open(filename)
if err != nil {
	return err
}
defer file.Close()

2. Use Buffered I/O

// Use bufio for better performance
scanner := bufio.NewScanner(file)

3. Validate Paths

// Validate paths to prevent directory traversal
path := filepath.Join(basePath, userPath)

4. Handle Permissions

// Set appropriate permissions
os.WriteFile(filename, data, 0644)

Common Pitfalls

1. Not Closing Files

Always defer file.Close() to prevent resource leaks.

2. Ignoring Errors

Always check and handle errors.

3. No Path Validation

Validate paths to prevent security issues.

4. Inefficient I/O

Use buffered I/O for better performance.

Resources

Summary

Proper file system operations are essential for CLI tools. Key takeaways:

Always close files with defer
Use buffered I/O for efficiency
Validate paths for security
Handle errors properly
Use atomic writes for safety
Set appropriate permissions
Clean up temporary files

By mastering file operations, you can build robust CLI applications.