Public Data Sources: Complete Guide for Data Science

from datasets import load_dataset

# Load a dataset directly in Python
dataset = load_dataset("squad", split="train")
print(f"SQuAD: {len(dataset)} examples")
print(dataset[0])
# {'id': '5733be284776f41900661182', 'title': '...', 'context': '...', 'question': '...', 'answers': {...}}

# Stream large datasets without downloading fully
streamed = load_dataset("c4", "en", split="train", streaming=True)
for i, example in enumerate(streamed):
    if i >= 5:
        break
    print(f"Example {i}: {example['text'][:100]}...")

# Filter and transform with built-in methods
filtered = dataset.filter(lambda x: len(x["context"]) > 500)
print(f"Long context examples: {len(filtered)}")

# Map transformations
def add_length(example):
    example["question_length"] = len(example["question"])
    return example

dataset = dataset.map(add_length)

import requests

def get_world_bank_indicator(indicator: str, country: str = "all") -> list[dict]:
    """Fetch data from World Bank API."""
    url = f"https://api.worldbank.org/v2/country/{country}/indicator/{indicator}"
    params = {"format": "json", "per_page": 100, "date": "2000:2025"}
    response = requests.get(url, params=params)
    return response.json()[1]  # First element is metadata

# Example: GDP per capita for all countries
gdp_data = get_world_bank_indicator("NY.GDP.PCAP.PP.KD")
for entry in gdp_data[:5]:
    print(f"{entry['country']['value']}: {entry['value']}")

import pandas as pd
from fredapi import Fred

fred = Fred(api_key="your_fred_api_key")

# Fetch unemployment rate
unemployment = fred.get_series("UNRATE", observation_start="2020-01-01")
print(unemployment.head())

# Search for series
search = fred.search("GDP", limit=5)
print(search[["title", "id", "frequency"]])

from datasets import get_dataset_config_names, load_dataset
import pandas as pd

# Search for available datasets
configs = get_dataset_config_names("squad")
print(f"SQuAD configs: {configs}")

# Load specific configuration
dataset = load_dataset("squad_v2", split="train")
df = pd.DataFrame(dataset)
print(f"Loaded {len(df)} examples")

# Filter for questions containing "why"
why_questions = df[df['question'].str.contains('why', case=False)]
print(f"Why questions: {len(why_questions)}")

# Install: pip install kagglehub
import kagglehub

# Download latest version of a dataset
path = kagglehub.dataset_download("datasnaek/youtube-new")
print(f"Dataset downloaded to: {path}")

# List files
import os
for file in os.listdir(path):
    print(file)

# Initialize DVC
pip install dvc
dvc init

# Track a dataset
dvc add data/raw/dataset.csv
git add data/raw/dataset.csv.dvc .gitignore
git commit -m "Add raw dataset"

# Switch between versions
git checkout <commit-hash>
dvc checkout

# Verify data version integrity
import hashlib

def verify_dataset_integrity(filepath: str, expected_hash: str) -> bool:
    """Verify a dataset file matches its DVC-tracked hash."""
    sha256 = hashlib.sha256()
    with open(filepath, "rb") as f:
        for chunk in iter(lambda: f.read(4096), b""):
            sha256.update(chunk)
    actual_hash = sha256.hexdigest()
    return actual_hash == expected_hash

print(f"Dataset integrity: {verify_dataset_integrity('data/dataset.csv', 'abc123...')}")

import great_expectations as ge

# Load a dataset as a Great Expectations DataFrame
df = ge.read_csv("data/dataset.csv")

# Define expectations
expectations = [
    df.expect_column_values_to_not_be_null("id"),
    df.expect_column_values_to_be_between("age", 0, 120),
    df.expect_column_values_to_be_in_set("category", ["A", "B", "C"]),
    df.expect_column_median_to_be_between("salary", 30000, 200000),
    df.expect_column_values_to_be_unique("email"),
]

# Run validation
results = [exp for exp in expectations]
passed = sum(1 for r in results if r["success"])
print(f"Passed: {passed}/{len(results)} expectations")

# Generate HTML report
# df.save_expectation_suite("my_suite.json")
# ge.render.render_to_html(results, "data_quality_report.html")

from faker import Faker
import pandas as pd
import random

fake = Faker()
Faker.seed(42)

def generate_customer_data(n: int = 1000) -> pd.DataFrame:
    """Generate synthetic customer records."""
    data = []
    for _ in range(n):
        data.append({
            "customer_id": fake.uuid4(),
            "name": fake.name(),
            "email": fake.email(),
            "age": random.randint(18, 80),
            "income": round(random.gammavariate(alpha=5, beta=10000), 2),
            "signup_date": fake.date_between(start_date="-3y", end_date="today"),
            "country": fake.country(),
            "is_active": random.random() > 0.2,
        })
    return pd.DataFrame(data)

df = generate_customer_data(5000)
print(f"Generated {len(df)} synthetic records")
print(df.describe())

# pip install sdv
from sdv.single_table import GaussianCopulaSynthesizer
from sdv.datasets.demo import download_demo

# Load real data and train synthesizer
real_data, metadata = download_demo(
    modality="single_table",
    dataset_name="fake_hotel_booking"
)

synthesizer = GaussianCopulaSynthesizer(metadata)
synthesizer.fit(real_data)

# Generate synthetic data
synthetic_data = synthesizer.sample(num_rows=500)

# Evaluate quality
from sdv.evaluation.single_table import evaluate_quality
quality_report = evaluate_quality(
    real_data, synthetic_data, metadata
)
print(f"Quality score: {quality_report.get_score():.2f}")

import requests
from bs4 import BeautifulSoup
import pandas as pd
import time

def scrape_table(url: str, table_class: str) -> pd.DataFrame:
    """Scrape an HTML table into a DataFrame."""
    response = requests.get(url, headers={"User-Agent": "Mozilla/5.0"})
    soup = BeautifulSoup(response.content, "html.parser")
    table = soup.find("table", class_=table_class)

    rows = []
    for tr in table.find_all("tr"):
        cells = tr.find_all(["td", "th"])
        rows.append([cell.get_text(strip=True) for cell in cells])

    return pd.DataFrame(rows[1:], columns=rows[0]) if len(rows) > 1 else pd.DataFrame()

# Example: scrape a Wikipedia table
# df = scrape_table("https://en.wikipedia.org/wiki/List_of_countries_by_GDP_(nominal)", "wikitable")
# print(df.head())