Text Processing & String Algorithms in Python: Comprehensive Guide

# Strings are immutable
text = "Hello"
text[0] = "J"  # TypeError: 'str' object does not support item assignment

# Operations create new strings
text = "Hello"
new_text = text.replace("H", "J")  # Creates new string
print(text)      # Output: Hello (original unchanged)
print(new_text)  # Output: Jello

text = "Hello World"

# Case conversion
print(text.upper())       # Output: HELLO WORLD
print(text.lower())       # Output: hello world
print(text.capitalize())  # Output: Hello world
print(text.title())       # Output: Hello World
print(text.swapcase())    # Output: hELLO wORLD

# Check case
print("HELLO".isupper())  # Output: True
print("hello".islower())  # Output: True

text = "The quick brown fox jumps over the lazy dog"

# Searching
print(text.find("fox"))           # Output: 16 (index)
print(text.find("cat"))           # Output: -1 (not found)
print(text.count("the"))          # Output: 1
print(text.count("o"))            # Output: 4

# Checking content
print(text.startswith("The"))     # Output: True
print(text.endswith("dog"))       # Output: True
print("fox" in text)              # Output: True

# Replacing
new_text = text.replace("fox", "cat")
print(new_text)  # Output: The quick brown cat jumps over the lazy dog

# Replace with limit
text = "aaa"
print(text.replace("a", "b", 2))  # Output: bba (replace first 2)

# Splitting
text = "apple,banana,cherry"
fruits = text.split(",")
print(fruits)  # Output: ['apple', 'banana', 'cherry']

# Split with limit
text = "a:b:c:d"
parts = text.split(":", 2)
print(parts)  # Output: ['a', 'b', 'c:d']

# Splitting on whitespace
text = "  hello   world  "
words = text.split()  # Splits on any whitespace
print(words)  # Output: ['hello', 'world']

# Joining
fruits = ['apple', 'banana', 'cherry']
result = ", ".join(fruits)
print(result)  # Output: apple, banana, cherry

# Joining with different separators
numbers = ['1', '2', '3', '4']
print("-".join(numbers))  # Output: 1-2-3-4
print("".join(numbers))   # Output: 1234

# Stripping whitespace
text = "  hello world  "
print(f"'{text.strip()}'")   # Output: 'hello world'
print(f"'{text.lstrip()}'")  # Output: 'hello world  '
print(f"'{text.rstrip()}'")  # Output: '  hello world'

# Stripping specific characters
text = "###hello###"
print(text.strip("#"))  # Output: hello

# Padding
text = "hello"
print(text.ljust(10, "-"))  # Output: hello-----
print(text.rjust(10, "-"))  # Output: -----hello
print(text.center(11, "-")) # Output: ---hello---

# Type checking
print("123".isdigit())        # Output: True
print("abc".isalpha())        # Output: True
print("abc123".isalnum())     # Output: True
print("   ".isspace())        # Output: True
print("Hello".istitle())      # Output: True

# Practical validation
def is_valid_identifier(name):
    """Check if string is valid Python identifier"""
    return name.isidentifier()

print(is_valid_identifier("my_var"))    # Output: True
print(is_valid_identifier("123var"))    # Output: False
print(is_valid_identifier("my-var"))    # Output: False

def is_palindrome(text):
    """Check if text is a palindrome (ignoring spaces and case)"""
    # Remove spaces and convert to lowercase
    cleaned = text.replace(" ", "").lower()
    # Compare with reverse
    return cleaned == cleaned[::-1]

# Test cases
test_cases = [
    "racecar",
    "A man a plan a canal Panama",
    "hello",
    "Madam",
]

for text in test_cases:
    result = "is" if is_palindrome(text) else "is not"
    print(f"'{text}' {result} a palindrome")

# Output:
# 'racecar' is a palindrome
# 'A man a plan a canal Panama' is a palindrome
# 'hello' is not a palindrome
# 'Madam' is a palindrome

def are_anagrams(word1, word2):
    """Check if two words are anagrams"""
    # Sort characters in both words
    return sorted(word1.lower()) == sorted(word2.lower())

# Test cases
pairs = [
    ("listen", "silent"),
    ("hello", "world"),
    ("Dormitory", "Dirty room"),
]

for word1, word2 in pairs:
    result = "are" if are_anagrams(word1, word2) else "are not"
    print(f"'{word1}' and '{word2}' {result} anagrams")

# Output:
# 'listen' and 'silent' are anagrams
# 'hello' and 'world' are not anagrams
# 'Dormitory' and 'Dirty room' are anagrams

def longest_common_substring(text1, text2):
    """Find the longest common substring"""
    if not text1 or not text2:
        return ""
    
    longest = ""
    
    # Check all substrings of text1
    for i in range(len(text1)):
        for j in range(i + 1, len(text1) + 1):
            substring = text1[i:j]
            if substring in text2 and len(substring) > len(longest):
                longest = substring
    
    return longest

# Test cases
pairs = [
    ("abcdef", "fbdamn"),
    ("programming", "gaming"),
    ("hello", "world"),
]

for text1, text2 in pairs:
    result = longest_common_substring(text1, text2)
    print(f"LCS of '{text1}' and '{text2}': '{result}'")

# Output:
# LCS of 'abcdef' and 'fbdamn': 'bd'
# LCS of 'programming' and 'gaming': 'gramin'
# LCS of 'hello' and 'world': 'o'

def levenshtein_distance(text1, text2):
    """Calculate edit distance between two strings"""
    m, n = len(text1), len(text2)
    
    # Create DP table
    dp = [[0] * (n + 1) for _ in range(m + 1)]
    
    # Initialize base cases
    for i in range(m + 1):
        dp[i][0] = i
    for j in range(n + 1):
        dp[0][j] = j
    
    # Fill DP table
    for i in range(1, m + 1):
        for j in range(1, n + 1):
            if text1[i - 1] == text2[j - 1]:
                dp[i][j] = dp[i - 1][j - 1]
            else:
                dp[i][j] = 1 + min(
                    dp[i - 1][j],      # deletion
                    dp[i][j - 1],      # insertion
                    dp[i - 1][j - 1]   # substitution
                )
    
    return dp[m][n]

# Test cases
pairs = [
    ("kitten", "sitting"),
    ("saturday", "sunday"),
    ("hello", "hello"),
]

for text1, text2 in pairs:
    distance = levenshtein_distance(text1, text2)
    print(f"Distance between '{text1}' and '{text2}': {distance}")

# Output:
# Distance between 'kitten' and 'sitting': 3
# Distance between 'saturday' and 'sunday': 3
# Distance between 'hello' and 'hello': 0

from collections import Counter

def analyze_text(text):
    """Analyze word frequency in text"""
    # Convert to lowercase and split
    words = text.lower().split()
    
    # Remove punctuation
    import string
    words = [word.strip(string.punctuation) for word in words]
    
    # Count frequencies
    word_freq = Counter(words)
    
    return word_freq

# Test text
text = """
Python is great. Python is powerful. Python is easy to learn.
Learning Python is fun. I love Python programming.
"""

frequencies = analyze_text(text)

# Display top 5 words
print("Top 5 most frequent words:")
for word, count in frequencies.most_common(5):
    print(f"  {word}: {count}")

# Output:
# Top 5 most frequent words:
#   python: 5
#   is: 3
#   learning: 1
#   great: 1
#   powerful: 1

# f-strings (Python 3.6+) - Most modern and readable
name = "Alice"
age = 30
print(f"{name} is {age} years old")  # Output: Alice is 30 years old

# Format with expressions
print(f"Next year: {age + 1}")  # Output: Next year: 31

# Format with alignment and padding
print(f"{name:>10}")  # Right align
print(f"{name:<10}")  # Left align
print(f"{name:^10}")  # Center align

# Format numbers
price = 19.99
print(f"Price: ${price:.2f}")  # Output: Price: $19.99

# .format() method (older but still useful)
print("{} is {} years old".format(name, age))
print("{name} is {age} years old".format(name=name, age=age))

# % formatting (legacy, not recommended)
print("%s is %d years old" % (name, age))

import unicodedata
import re

def normalize_text(text):
    """Normalize text for comparison"""
    # Remove accents
    text = ''.join(
        c for c in unicodedata.normalize('NFD', text)
        if unicodedata.category(c) != 'Mn'
    )
    
    # Convert to lowercase
    text = text.lower()
    
    # Remove extra whitespace
    text = ' '.join(text.split())
    
    # Remove special characters
    text = re.sub(r'[^a-z0-9\s]', '', text)
    
    return text

# Test cases
texts = [
    "Café",
    "HELLO   WORLD",
    "Hello, World!",
    "Naïve résumé",
]

for text in texts:
    normalized = normalize_text(text)
    print(f"'{text}' -> '{normalized}'")

# Output:
# 'Café' -> 'cafe'
# 'HELLO   WORLD' -> 'hello world'
# 'Hello, World!' -> 'hello world'
# 'Naïve résumé' -> 'naive resume'

import re

def tokenize_simple(text):
    """Simple word tokenization"""
    # Split on whitespace and punctuation
    tokens = re.findall(r'\b\w+\b', text.lower())
    return tokens

def tokenize_sentences(text):
    """Split text into sentences"""
    # Split on sentence boundaries
    sentences = re.split(r'[.!?]+', text)
    return [s.strip() for s in sentences if s.strip()]

# Test text
text = "Hello world! How are you? I'm fine, thanks."

print("Word tokens:")
print(tokenize_simple(text))
# Output: ['hello', 'world', 'how', 'are', 'you', 'i', 'm', 'fine', 'thanks']

print("\nSentence tokens:")
print(tokenize_sentences(text))
# Output: ['Hello world', 'How are you', "I'm fine, thanks"]

import time

text_list = ["word"] * 10000

# ❌ Inefficient: String concatenation in loop
start = time.time()
result = ""
for word in text_list:
    result += word + " "
time_concat = time.time() - start

# ✓ Efficient: Using join()
start = time.time()
result = " ".join(text_list)
time_join = time.time() - start

print(f"Concatenation: {time_concat:.6f}s")
print(f"Join: {time_join:.6f}s")
print(f"Join is {time_concat/time_join:.1f}x faster")

# Output (approximate):
# Concatenation: 0.015234s
# Join: 0.000234s
# Join is 65.0x faster

import re
import time

text = "The quick brown fox jumps over the lazy dog. " * 1000
pattern = "fox"

# Method 1: Using 'in' operator
start = time.time()
for _ in range(10000):
    result = pattern in text
time_in = time.time() - start

# Method 2: Using find()
start = time.time()
for _ in range(10000):
    result = text.find(pattern)
time_find = time.time() - start

# Method 3: Using regex
start = time.time()
regex = re.compile(pattern)
for _ in range(10000):
    result = regex.search(text)
time_regex = time.time() - start

print(f"'in' operator: {time_in:.6f}s")
print(f"find(): {time_find:.6f}s")
print(f"regex: {time_regex:.6f}s")

# Output (approximate):
# 'in' operator: 0.001234s
# find(): 0.001456s
# regex: 0.045678s

# ✓ Good: Use appropriate methods for the task
def process_text_efficient(text):
    # Use split() for simple tokenization
    words = text.split()
    
    # Use list comprehension for filtering
    long_words = [w for w in words if len(w) > 5]
    
    # Use join() for concatenation
    result = " ".join(long_words)
    
    return result

# ❌ Avoid: Unnecessary operations
def process_text_inefficient(text):
    # Avoid repeated string operations
    words = []
    for char in text:
        if char != " ":
            words.append(char)
    
    # Avoid repeated concatenation
    result = ""
    for word in words:
        result = result + word + " "
    
    return result

import re

def clean_phone_number(phone):
    """Extract digits from phone number"""
    digits = re.sub(r'\D', '', phone)
    if len(digits) == 10:
        return f"({digits[:3]}) {digits[3:6]}-{digits[6:]}"
    return None

def clean_email(email):
    """Normalize email address"""
    return email.lower().strip()

def clean_csv_field(field):
    """Clean CSV field"""
    # Remove quotes and extra whitespace
    field = field.strip('"').strip()
    return field

# Test cases
print(clean_phone_number("555-123-4567"))  # Output: (555) 123-4567
print(clean_email("  [email protected]  "))  # Output: [email protected]
print(clean_csv_field('"  hello  "'))  # Output: hello

import re
from datetime import datetime

def parse_log_line(line):
    """Parse Apache log line"""
    pattern = r'(\S+) - - \[([^\]]+)\] "(\S+) (\S+) (\S+)" (\d+) (\d+)'
    match = re.match(pattern, line)
    
    if match:
        ip, timestamp, method, path, protocol, status, size = match.groups()
        return {
            'ip': ip,
            'timestamp': timestamp,
            'method': method,
            'path': path,
            'protocol': protocol,
            'status': int(status),
            'size': int(size)
        }
    return None

# Example log line
log_line = '192.168.1.1 - - [16/Dec/2025:10:30:45 +0000] "GET /index.html HTTP/1.1" 200 1234'
parsed = parse_log_line(log_line)

if parsed:
    print(f"IP: {parsed['ip']}")
    print(f"Method: {parsed['method']}")
    print(f"Status: {parsed['status']}")

from urllib.parse import urlparse, parse_qs, urlencode

def analyze_url(url):
    """Parse and analyze URL"""
    parsed = urlparse(url)
    
    return {
        'scheme': parsed.scheme,
        'netloc': parsed.netloc,
        'path': parsed.path,
        'params': parse_qs(parsed.query)
    }

# Example URL
url = "https://example.com/search?q=python&sort=date"
result = analyze_url(url)

print(f"Scheme: {result['scheme']}")
print(f"Domain: {result['netloc']}")
print(f"Path: {result['path']}")
print(f"Query params: {result['params']}")

# Output:
# Scheme: https
# Domain: example.com
# Path: /search
# Query params: {'q': ['python'], 'sort': ['date']}

def similarity_score(text1, text2):
    """Calculate simple similarity score"""
    # Convert to sets of words
    words1 = set(text1.lower().split())
    words2 = set(text2.lower().split())
    
    # Calculate Jaccard similarity
    intersection = len(words1 & words2)
    union = len(words1 | words2)
    
    return intersection / union if union > 0 else 0

# Test cases
texts = [
    ("The cat sat on the mat", "The cat sat on the mat"),
    ("The cat sat on the mat", "A dog sat on the rug"),
    ("Python is great", "Python is awesome"),
]

for text1, text2 in texts:
    score = similarity_score(text1, text2)
    print(f"Similarity: {score:.2%}")

# Output:
# Similarity: 100.00%
# Similarity: 25.00%
# Similarity: 50.00%

# ✓ Good: Use built-in methods
text = "  hello world  "
cleaned = text.strip().lower()

# ❌ Avoid: Manual character manipulation
cleaned = ""
for char in text:
    if char != " ":
        cleaned += char.lower()

# ✓ Good: Use join()
words = ["hello", "world", "python"]
result = " ".join(words)

# ❌ Avoid: String concatenation
result = ""
for word in words:
    result += word + " "

import re

# ✓ Good: Raw string for regex
pattern = r'\d{3}-\d{4}'

# ❌ Avoid: Regular string (confusing escaping)
pattern = '\\d{3}-\\d{4}'

import re

# ✓ Good: Compile once
email_pattern = re.compile(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b')

emails = ["[email protected]", "[email protected]"]
for email in emails:
    if email_pattern.match(email):
        print(f"Valid: {email}")

# ❌ Avoid: Recompile each time
for email in emails:
    if re.match(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b', email):
        print(f"Valid: {email}")

# ✓ Good: Explicit encoding/decoding
text = "Café"
encoded = text.encode('utf-8')
decoded = encoded.decode('utf-8')

# ✓ Good: Use unicodedata for normalization
import unicodedata
normalized = unicodedata.normalize('NFC', text)