Unicode and String Encoding in Go

import "unicode/utf8"

s := "Hello, 世界!"
fmt.Println(len(s))                    // 14 bytes
fmt.Println(utf8.RuneCountInString(s)) // 10 characters

// Encode a rune to bytes
buf := make([]byte, utf8.UTFMax)  // UTFMax = 4
n := utf8.EncodeRune(buf, '世')
fmt.Printf("世 = %X (%d bytes)\n", buf[:n], n)  // E4 B8 96 (3 bytes)

// Decode a rune from bytes
r, size := utf8.DecodeRuneInString("世界")
fmt.Printf("first rune: %c, byte width: %d\n", r, size)  // 世, 3

composed   := "café"   // U+00E9 for é (NFC form)
decomposed := "cafe\u0301"  // e + combining acute (NFD form)

fmt.Println(composed == decomposed)  // false — different bytes!
fmt.Println(len(composed))           // 5 bytes
fmt.Println(len(decomposed))         // 6 bytes

go get golang.org/x/text

import "golang.org/x/text/unicode/norm"

// NFC: Canonical Decomposition followed by Canonical Composition
// (most common form; what databases and most systems produce)
nfc := norm.NFC.String(decomposed)
fmt.Println(nfc == composed)  // true — both are now NFC

// NFD: Canonical Decomposition
// (base characters with combining marks as separate code points)
nfd := norm.NFD.String(composed)
fmt.Println(len(nfd))  // 6 — é is decomposed to e + accent

// NFKC: Compatibility Decomposition + Composition
// (normalizes visual variants: fi → fi, ① → 1, etc.)
nfkc := norm.NFKC.String("first")  // fi ligature
fmt.Println(nfkc)  // "first" — ligature expanded

import "golang.org/x/text/unicode/grapheme"

// Family emoji: 👨‍👩‍👧‍👦 = 7 code points (4 persons + 3 zero-width joiners)
family := "👨‍👩‍👧‍👦"
fmt.Println(utf8.RuneCountInString(family))  // 7
fmt.Println(grapheme.GraphemeClusterCount(family))  // 1 — one visible character

// Correct string truncation by visible characters
func truncateByGrapheme(s string, maxClusters int) string {
    count := 0
    for i := 0; i < len(s); {
        _, size := grapheme.FirstGraphemeCluster(s[i:], -1)
        if count >= maxClusters {
            return s[:i] + "…"
        }
        i += size
        count++
    }
    return s
}

import (
    "golang.org/x/text/cases"
    "golang.org/x/text/language"
)

// Case-fold for case-insensitive comparison (handles Unicode correctly)
c := cases.Fold()
fmt.Println(c.String("CAFÉ") == c.String("café"))   // true
fmt.Println(c.String("Straße") == c.String("STRASSE"))  // true (German ß folds to ss)

// ToUpper/ToLower with locale — Turkish 'i' vs English 'I'
turkishUpper := cases.Upper(language.Turkish)
englishUpper := cases.Upper(language.English)
fmt.Println(turkishUpper.String("istanbul"))  // İSTANBUL (dotted İ)
fmt.Println(englishUpper.String("istanbul"))  // ISTANBUL (dotless I)

import (
    "golang.org/x/text/collate"
    "golang.org/x/text/language"
)

words := []string{"Österreich", "Adalbert", "Zürich", "Bayern", "Äpfel"}

// Byte sort — wrong for German
sort.Strings(words)
// ["Adalbert", "Bayern", "Zürich", "Österreich", "Äpfel"]
// ↑ Wrong: Ä and Ö sort after Z instead of near A and O

// Locale-aware sort — correct
cl := collate.New(language.German)
cl.SortStrings(words)
// ["Adalbert", "Äpfel", "Bayern", "Österreich", "Zürich"]
// ↑ Correct: Ä sorts near A, Ö sorts near O

import (
    "golang.org/x/text/encoding/charmap"
    "golang.org/x/text/transform"
    "io"
    "os"
)

// Read a Windows-1252 encoded file and convert to UTF-8
func readWindows1252(path string) (string, error) {
    f, err := os.Open(path)
    if err != nil {
        return "", err
    }
    defer f.Close()

    // Wrap the reader with a decoder
    decoder := charmap.Windows1252.NewDecoder()
    reader := transform.NewReader(f, decoder)

    data, err := io.ReadAll(reader)
    return string(data), err
}

// Convert UTF-8 string to Latin-1 bytes for legacy system
func toLatin1(s string) ([]byte, error) {
    encoder := charmap.ISO8859_1.NewEncoder()
    result, _, err := transform.String(encoder, s)
    return []byte(result), err
}

import "unicode/utf8"

func validateUTF8(input string) error {
    if !utf8.ValidString(input) {
        return fmt.Errorf("input contains invalid UTF-8 sequences")
    }
    return nil
}

// Check for control characters and private-use code points
func sanitizeUserInput(s string) string {
    return strings.Map(func(r rune) rune {
        // Drop control characters (except tab, newline, carriage return)
        if unicode.IsControl(r) && r != '\t' && r != '\n' && r != '\r' {
            return -1  // drop the rune
        }
        // Replace replacement character (invalid UTF-8 decoded character)
        if r == utf8.RuneError {
            return -1
        }
        return r
    }, s)
}