Edge AI and On-Device AI: Running AI Without the Cloud

Cloud AI:
User → Internet → Cloud Server → AI Processing → Internet → User

Edge AI:
User → Local Device → AI Processing → User

# Install Ollama
curl -fsSL https://ollama.com/install.sh | sh

# Pull a model
ollama pull llama2
ollama pull mistral
ollama pull codellama

# Run a model
ollama run llama2 "Explain quantum computing"

# API access
curl http://localhost:11434/api/generate -d '{
  "model": "llama2",
  "prompt": "Hello!",
  "stream": false
}'

# Clone and build
git clone https://github.com/ggerganov/llama.cpp
cd llama.cpp
make

# Download model (convert to GGUF first)
# Run inference
./main -m models/llama2-7b.gguf -n 256 -t 8

from mlc_llm import MLCEngine

# Create engine
engine = MLCEngine(
    model="Llama-2-7b-chat-hf",
    device="cuda"
)

# Generate
result = engine.generate(
    prompt="Write a Python function to add two numbers",
    temperature=0.7
)

print(result)

# Google LiteRT — deploy on Android/iOS with NPU delegation
import ai_edge

# Load and compile model for target device
model = ai_edge.Interpreter(
    model_path="gemma-3-4b-int4.tflite",
    delegates=[
        ai_edge.delegate.GpuDelegate(),
        ai_edge.delegate.NpuDelegate()
    ]
)

model.allocate_tensors()
input_details = model.get_input_details()
output_details = model.get_output_details()

model.set_tensor(input_details[0]["index"], tokenized_input)
model.invoke()
output = model.get_tensor(output_details[0]["index"])

import onnxruntime as ort

options = ort.SessionOptions()
options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL

# Try NPU provider, fall back to CPU
providers = [
    ("QnnExecutionProvider", {"backend": "htp"}),  # Qualcomm
    ("CoreMLExecutionProvider", {}),                # Apple
    "CPUExecutionProvider"
]

session = ort.InferenceSession(
    "model.onnx",
    sess_options=options,
    providers=providers
)

results = session.run(
    output_names=["logits"],
    input_feed={"input": tokenized_input}
)

Recommended GPUs for Local AI:
- NVIDIA RTX 3090/4090 (24GB) - Best consumer
- NVIDIA RTX 4080 (16GB) - Good balance
- NVIDIA A100 (40-80GB) - Professional
- Apple Silicon M3 Max - Mac users

# llama.cpp CPU mode
./main -m model.gguf --n-gpu-layers 0

# Smaller models for CPU
# - Phi-2 (2.7B)
# - TinyLlama (1.1B)
# - Qwen-1.8B

# Complete local AI assistant example
import ollama
import gradio as gr

def chat(message, history):
    response = ollama.chat(
        model='llama2',
        messages=[
            {"role": "system", "content": "You are a helpful assistant."},
            *[{"role": "r" if h[0] == "user" else "assistant", 
               "content": h[1]} for h in history],
            {"role": "user", "content": message}
        ]
    )
    return response['message']['content']

# Create interface
gr.ChatInterface(
    fn=chat,
    title="Local AI Assistant",
    description="Running Llama2 locally"
).launch()

class PrivacyFirstAI:
    """AI that never sends data externally"""
    
    def __init__(self):
        self.model = OllamaModel("llama2:7b")
    
    def process(self, user_input: str) -> str:
        # All processing stays local
        return self.model.generate(user_input)
    
    def summarize_document(self, document_path: str) -> str:
        # Read local file
        with open(document_path) as f:
            content = f.read()
        
        # Process locally
        prompt = f"Summarize: {content}"
        return self.model.generate(prompt)
    
    def analyze_code(self, code: str) -> dict:
        # Analyze code without sending anywhere
        prompt = f"Analyze this code for issues:\n{code}"
        analysis = self.model.generate(prompt)
        
        return {
            "issues": analysis,
            "processed_locally": True
        }

# Edge device example (Raspberry Pi)
from edge_tpu import EdgeTPU
import tflite_runtime

# Load optimized model
model = EdgeTPU.isntall()
interpreter = tflite_runtime.interpreter.Interpreter(
    model_path='model_quantized.tflite',
    experimental_delegates=[load_delegate('libedgetpu.so.1')]
)

# Run inference
def predict(input_data):
    interpreter.invoke()
    return interpreter.output(0)

# Ollama server on edge hardware
curl -fsSL https://ollama.com/install.sh | sh
ollama pull phi-4:q4_K_M
ollama run phi-4:q4_K_M

# llama.cpp server for GGUF models
./server -m phi-4-q4_K_M.gguf --host 0.0.0.0 --port 8080

# vLLM on Jetson Orin (with TensorRT)
docker run --runtime nvidia vllm:v0.8.0 \
    --model phi-4 \
    --quantization awq \
    --max-model-len 4096

Use case: Private AI chatbot
Tech: Ollama + Llama2
Features:
- Fully offline capable
- Your data never leaves your machine
- Custom knowledge base (local files)
- No subscription costs

Use case: Meeting transcription
Tech: Whisper.cpp
Features:
- Real-time transcription
- Works offline
- Multiple languages
- Custom vocabulary

Use case: Local voice assistant
Tech: Raspberry Pi + Whisper + Llama
Features:
- Responds to voice commands
- Controls smart home devices
- Privacy-first (no cloud)
- Works without internet

Use case: Local content filtering
Tech: Fine-tuned model
Features:
- Screens content locally
- No data sent externally
- Customizable filters
- Real-time processing

Use case: Local coding assistant
Tech: CodeLlama via Ollama
Features:
- Code completion
- Bug detection
- Refactoring suggestions
- Works offline

from llama_cpp import Llama

llm = Llama(
    model_path="qwen2.5-1.5b-instruct-q4_K_M.gguf",
    n_ctx=2048,
    n_gpu_layers=-1,
    n_threads=4,
    verbose=False
)

response = llm(
    "Explain what an NPU is in one paragraph.",
    max_tokens=256,
    temperature=0.7,
    stop=["</s>"]
)
print(response["choices"][0]["text"])

# Convert to 4-bit quantized
./quantize \
  models/llama2-7b.gguf \
  models/llama2-7b-q4.gguf \
  q4_0

# Original: 13.5 GB → Q4 Quantized: ~4 GB

import coremltools as ct
import torch

model = torch.load("phi-4-7b-fp16.pt")
example_input = torch.randn(1, 128)
traced_model = torch.jit.trace(model, example_input)

mlmodel = ct.convert(
    traced_model,
    convert_to="mlprogram",
    compute_units=ct.ComputeUnit.ALL,  # CPU+GPU+Neural Engine
    minimum_deployment_target=ct.target.iOS18,
    weights=ct.quantization_utils.quantize_weights(
        traced_model, nbits=4, granularity="per_block"
    )
)

mlmodel.save("Phi-4-7B-NE.mlpackage")

import qnn_wrapper as qnn

context = qnn.QnnContext(
    model_path="phi-4-7b-int4.serialized",
    backend="htp",           # Hexagon Tensor Processor
    device_id="0",
)

result = context.inference(
    input_tensor={"input_ids": [[1, 45, 233, ...]]},
    output_names=["logits"],
    config={"htp_soc": "snapdragon_8_elite_gen5"}
)

# QNN SDK achieves ~30 tok/s for Phi-4 7B on Snapdragon 8 Elite Gen 5
print(result["logits"])

# PyTorch pruning example
import torch.nn.utils.prune as prune

# Prune 30% of connections
prune.l1_unstructured(
    model.linear_layer, 
    name="weight", 
    amount=0.3
)

# Distillation example
teacher_model = load_teacher("Llama-70b")
student_model = load_student("TinyLlama-1b")

# Train student to mimic teacher
for batch in data:
    teacher_output = teacher_model(batch)
    student_output = student_model(batch)
    
    loss = distillation_loss(
        teacher_output, 
        student_output
    )
    loss.backward()

# PyTorch pruning example
import torch.nn.utils.prune as prune

# Prune 30% of connections
prune.l1_unstructured(
    model.linear_layer, 
    name="weight", 
    amount=0.3
)

# Distillation example
teacher_model = load_teacher("Llama-70b")
student_model = load_student("TinyLlama-1b")

# Train student to mimic teacher
for batch in data:
    teacher_output = teacher_model(batch)
    student_output = student_model(batch)
    
    loss = distillation_loss(
        teacher_output, 
        student_output
    )
    loss.backward()

// Using WebLLM in browser
import { CreateMLCEngine } from "@mlc-ai/web-llm";

const engine = await CreateMLCEngine(
  "Llama-3-8B-Instruct-q4f32_1",
  { initProgressCallback: (progress) => console.log(progress) }
);

const response = await engine.chat.completions.create({
  messages: [{ role: "user", content: "Hello!" }],
  temperature: 0.7
});

console.log(response.choices[0].message.content);

// Run BERT in browser
import { pipeline } from '@xenova/transformers';

const classifier = await pipeline(
  'sentiment-analysis', 
  'Xenova/distilbert-base-uncased-finetuned-sst-2-english'
);

const result = await classifier(
  'I love using on-device AI!'
);
// [{ label: 'POSITIVE', score: 0.999 }]

// Transformers.js v3 — run LLM in browser with WebGPU
import { pipeline, env } from "@huggingface/transformers";

env.allowLocalModels = false;

const generator = await pipeline(
    "text-generation",
    "HuggingFaceTB/SmolLM2-360M-Instruct",
    { device: "webgpu", dtype: "q4" }
);

const response = await generator(
    "Explain edge AI in one sentence.",
    { max_new_tokens: 100 }
);

console.log(response[0].generated_text);

// Production pattern: run inference in a Web Worker
const worker = new Worker("ai-worker.js");

worker.postMessage({ type: "generate", prompt: "Hello!" });
worker.onmessage = (event) => {
    console.log(event.data.response);
};

# Guidelines
- 8GB RAM: 7B models (quantized)
- 16GB RAM: 13B models (quantized)
- 32GB RAM: 34B models (quantized)
- Apple Silicon: M-series optimized models

# Verify model integrity
import hashlib

def verify_model(model_path: str, expected_hash: str) -> bool:
    with open(model_path, 'rb') as f:
        actual_hash = hashlib.sha256(f.read()).hexdigest()
    return actual_hash == expected_hash

# Privacy-first pattern: validate no data leaves the device
import psutil
import socket

def assert_no_network_egress():
    """Assert that this code path makes no external connections."""
    connections = psutil.net_connections()
    outgoing = [
        c for c in connections
        if c.status == "ESTABLISHED"
        and c.raddr and not c.raddr.ip.startswith(("127.", "::1"))
    ]
    if outgoing:
        raise RuntimeError(f"Unexpected network egress: {outgoing}")

def process_document_locally(text: str) -> str:
    """Summarize a document entirely on-device."""
    assert_no_network_egress()
    prompt = f"Summarize this: {text[:2000]}"
    response = llm(prompt, max_tokens=256)
    return response["choices"][0]["text"]

# Verify Core ML model integrity before inference
import coremltools as ct

def load_verified_model(path: str, expected_hash: str):
    """Load a Core ML model only if its hash matches."""
    import hashlib
    with open(path, "rb") as f:
        actual_hash = hashlib.sha256(f.read()).hexdigest()
    if actual_hash != expected_hash:
        raise ValueError("Model integrity check failed")
    return ct.models.MLModel(path)