Web Automation with Selenium: Browser Automation and Testing

Selenium enables automating web browsers for testing, web scraping, and repetitive tasks. It’s the industry standard for browser automation across multiple programming languages.

Installation and Setup

Install Selenium

# Install Selenium
pip install selenium

# Install WebDriver Manager (handles driver downloads)
pip install webdriver-manager

Download WebDriver

# Using webdriver-manager (automatic)
from selenium import webdriver
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.chrome.service import Service

# Automatic driver management
driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()))

# Manual driver download
# Download from: https://chromedriver.chromium.org/
# Place in PATH or specify path explicitly
driver = webdriver.Chrome('/path/to/chromedriver')

Basic Browser Automation

Opening and Navigating

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.chrome.service import Service
import time

# Create driver
driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()))

try:
    # Navigate to URL
    driver.get('https://www.example.com')
    
    # Get page title
    print(f"Page title: {driver.title}")
    
    # Get current URL
    print(f"Current URL: {driver.current_url}")
    
    # Wait for page to load
    time.sleep(2)
    
finally:
    # Close browser
    driver.quit()

Finding Elements

from selenium.webdriver.common.by import By

# Find single element
element = driver.find_element(By.ID, 'element_id')
element = driver.find_element(By.NAME, 'element_name')
element = driver.find_element(By.CLASS_NAME, 'class_name')
element = driver.find_element(By.TAG_NAME, 'div')
element = driver.find_element(By.CSS_SELECTOR, '.class > div')
element = driver.find_element(By.XPATH, '//div[@id="main"]')
element = driver.find_element(By.LINK_TEXT, 'Click Here')
element = driver.find_element(By.PARTIAL_LINK_TEXT, 'Click')

# Find multiple elements
elements = driver.find_elements(By.CLASS_NAME, 'item')
for element in elements:
    print(element.text)

# Find nested elements
parent = driver.find_element(By.ID, 'parent')
child = parent.find_element(By.CLASS_NAME, 'child')

Interacting with Elements

from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains

# Text input
input_field = driver.find_element(By.ID, 'search')
input_field.send_keys('search query')
input_field.send_keys(Keys.RETURN)  # Press Enter

# Clear input
input_field.clear()

# Click element
button = driver.find_element(By.ID, 'submit')
button.click()

# Submit form
form = driver.find_element(By.TAG_NAME, 'form')
form.submit()

# Get element text
text = element.text
print(f"Element text: {text}")

# Get element attribute
href = element.get_attribute('href')
print(f"Link: {href}")

# Check if element is displayed
if element.is_displayed():
    print("Element is visible")

# Check if element is enabled
if element.is_enabled():
    print("Element is enabled")

# Get element size and location
size = element.size
location = element.location
print(f"Size: {size}, Location: {location}")

Waiting for Elements

Implicit Wait

from selenium import webdriver
import time

driver = webdriver.Chrome()

# Implicit wait: Wait up to 10 seconds for any element
driver.implicitly_wait(10)

# This will wait up to 10 seconds if element not found
element = driver.find_element(By.ID, 'dynamic_element')

Explicit Wait

from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By

# Wait for element to be present
wait = WebDriverWait(driver, 10)
element = wait.until(EC.presence_of_element_located((By.ID, 'element_id')))

# Wait for element to be clickable
element = wait.until(EC.element_to_be_clickable((By.ID, 'button_id')))

# Wait for element to be visible
element = wait.until(EC.visibility_of_element_located((By.ID, 'element_id')))

# Wait for element to be invisible
wait.until(EC.invisibility_of_element_located((By.ID, 'loading_spinner')))

# Wait for text to be present
wait.until(EC.text_to_be_present_in_element((By.ID, 'status'), 'Success'))

# Wait for URL to change
wait.until(EC.url_changes(driver.current_url))

# Custom wait condition
def element_has_text(locator, text):
    def _predicate(driver):
        element = driver.find_element(*locator)
        return text in element.text
    return _predicate

wait.until(element_has_text((By.ID, 'message'), 'Expected text'))

Advanced Interactions

Mouse Actions

from selenium.webdriver.common.action_chains import ActionChains

actions = ActionChains(driver)

# Hover over element
element = driver.find_element(By.ID, 'menu')
actions.move_to_element(element).perform()

# Double click
actions.double_click(element).perform()

# Right click
actions.context_click(element).perform()

# Drag and drop
source = driver.find_element(By.ID, 'draggable')
target = driver.find_element(By.ID, 'droppable')
actions.drag_and_drop(source, target).perform()

# Click and hold
actions.click_and_hold(element).perform()
actions.release().perform()

# Move by offset
actions.move_by_offset(100, 50).perform()

Keyboard Actions

from selenium.webdriver.common.keys import Keys

# Type text
input_field = driver.find_element(By.ID, 'search')
input_field.send_keys('search term')

# Special keys
input_field.send_keys(Keys.RETURN)
input_field.send_keys(Keys.TAB)
input_field.send_keys(Keys.ESCAPE)
input_field.send_keys(Keys.BACKSPACE)
input_field.send_keys(Keys.DELETE)

# Keyboard shortcuts
input_field.send_keys(Keys.CONTROL + 'a')  # Select all
input_field.send_keys(Keys.CONTROL + 'c')  # Copy
input_field.send_keys(Keys.CONTROL + 'v')  # Paste

# Arrow keys
input_field.send_keys(Keys.ARROW_UP)
input_field.send_keys(Keys.ARROW_DOWN)
input_field.send_keys(Keys.ARROW_LEFT)
input_field.send_keys(Keys.ARROW_RIGHT)

from selenium.webdriver.support.select import Select

# Select by visible text
dropdown = driver.find_element(By.ID, 'country')
select = Select(dropdown)
select.select_by_visible_text('United States')

# Select by value
select.select_by_value('us')

# Select by index
select.select_by_index(0)

# Get all options
options = select.options
for option in options:
    print(option.text)

# Get selected option
selected = select.first_selected_option
print(f"Selected: {selected.text}")

# Deselect (for multi-select)
select.deselect_by_visible_text('Option 1')
select.deselect_all()

Handling Windows and Alerts

Window Management

# Get window handles
window_handles = driver.window_handles
print(f"Number of windows: {len(window_handles)}")

# Switch to window
driver.switch_to.window(window_handles[1])

# Get current window handle
current_handle = driver.current_window_handle

# Open new window
driver.execute_script("window.open('https://example.com', '_blank');")

# Close current window
driver.close()

# Switch back to main window
driver.switch_to.window(window_handles[0])

Alert Handling

from selenium.webdriver.common.alert import Alert

# Accept alert
alert = Alert(driver)
alert.accept()

# Dismiss alert
alert.dismiss()

# Get alert text
alert_text = alert.text
print(f"Alert: {alert_text}")

# Type in alert
alert.send_keys('input text')
alert.accept()

# Wait for alert
wait = WebDriverWait(driver, 10)
wait.until(EC.alert_is_present())
alert = driver.switch_to.alert
alert.accept()

Web Scraping with Selenium

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time

driver = webdriver.Chrome()

try:
    # Navigate to page
    driver.get('https://example.com/products')
    
    # Wait for products to load
    wait = WebDriverWait(driver, 10)
    wait.until(EC.presence_of_all_elements_located((By.CLASS_NAME, 'product')))
    
    # Scroll to load more products
    for _ in range(5):
        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        time.sleep(2)
    
    # Extract data
    products = driver.find_elements(By.CLASS_NAME, 'product')
    
    data = []
    for product in products:
        name = product.find_element(By.CLASS_NAME, 'name').text
        price = product.find_element(By.CLASS_NAME, 'price').text
        link = product.find_element(By.TAG_NAME, 'a').get_attribute('href')
        
        data.append({
            'name': name,
            'price': price,
            'link': link
        })
    
    # Print results
    for item in data:
        print(f"{item['name']}: {item['price']} - {item['link']}")

finally:
    driver.quit()

Automated Testing

Basic Test Example

import unittest
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.chrome.service import Service

class TestLoginPage(unittest.TestCase):
    
    def setUp(self):
        """Set up test fixtures"""
        self.driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()))
        self.driver.get('https://example.com/login')
    
    def tearDown(self):
        """Clean up after tests"""
        self.driver.quit()
    
    def test_login_success(self):
        """Test successful login"""
        # Find elements
        username_field = self.driver.find_element(By.ID, 'username')
        password_field = self.driver.find_element(By.ID, 'password')
        login_button = self.driver.find_element(By.ID, 'login_btn')
        
        # Enter credentials
        username_field.send_keys('testuser')
        password_field.send_keys('password123')
        
        # Click login
        login_button.click()
        
        # Wait for redirect
        wait = WebDriverWait(self.driver, 10)
        wait.until(EC.url_changes(self.driver.current_url))
        
        # Assert successful login
        self.assertIn('dashboard', self.driver.current_url)
    
    def test_login_invalid_credentials(self):
        """Test login with invalid credentials"""
        username_field = self.driver.find_element(By.ID, 'username')
        password_field = self.driver.find_element(By.ID, 'password')
        login_button = self.driver.find_element(By.ID, 'login_btn')
        
        username_field.send_keys('invalid')
        password_field.send_keys('wrong')
        login_button.click()
        
        # Wait for error message
        wait = WebDriverWait(self.driver, 10)
        error = wait.until(EC.presence_of_element_located((By.CLASS_NAME, 'error')))
        
        # Assert error is displayed
        self.assertIn('Invalid credentials', error.text)
    
    def test_form_validation(self):
        """Test form validation"""
        login_button = self.driver.find_element(By.ID, 'login_btn')
        
        # Click without entering data
        login_button.click()
        
        # Check for validation messages
        username_error = self.driver.find_element(By.ID, 'username_error')
        self.assertTrue(username_error.is_displayed())

if __name__ == '__main__':
    unittest.main()

Browser Options and Configuration

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.chrome.service import Service

# Configure Chrome options
options = Options()

# Headless mode (no GUI)
options.add_argument('--headless')

# Disable images for faster loading
options.add_argument('--blink-settings=imagesEnabled=false')

# Set window size
options.add_argument('--window-size=1920,1080')

# Disable notifications
options.add_argument('--disable-notifications')

# Disable popups
options.add_argument('--disable-popup-blocking')

# User agent
options.add_argument('user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36')

# Proxy
options.add_argument('--proxy-server=http://proxy.example.com:8080')

# Create driver with options
driver = webdriver.Chrome(
    service=Service(ChromeDriverManager().install()),
    options=options
)

# Set page load timeout
driver.set_page_load_timeout(30)

# Set script timeout
driver.set_script_timeout(30)

JavaScript Execution

# Execute JavaScript
result = driver.execute_script("return 1 + 1;")
print(f"Result: {result}")

# Get page source
page_source = driver.page_source

# Scroll to element
element = driver.find_element(By.ID, 'target')
driver.execute_script("arguments[0].scrollIntoView(true);", element)

# Highlight element
driver.execute_script("arguments[0].style.border='3px solid red';", element)

# Get element value
value = driver.execute_script("return arguments[0].value;", element)

# Set element value
driver.execute_script("arguments[0].value = 'new value';", element)

# Remove element
driver.execute_script("arguments[0].remove();", element)

# Get cookies
cookies = driver.get_cookies()
for cookie in cookies:
    print(f"{cookie['name']}: {cookie['value']}")

# Add cookie
driver.add_cookie({'name': 'test', 'value': 'value'})

# Delete cookie
driver.delete_cookie('test')

Best Practices

Use explicit waits: Prefer WebDriverWait over implicit waits
Handle exceptions: Catch NoSuchElementException and TimeoutException
Close browser: Always use try/finally or context managers
Headless mode: Use for faster execution in CI/CD
Unique locators: Use IDs when available, avoid fragile XPaths
Page Object Model: Organize code for maintainability
Minimize waits: Use appropriate wait times to balance speed and reliability

Common Pitfalls

Bad Practice:

# Don't: Use sleep for waiting
time.sleep(5)
element = driver.find_element(By.ID, 'element')

# Don't: No exception handling
driver.find_element(By.ID, 'element').click()

# Don't: Hardcoded waits
driver.implicitly_wait(30)

# Don't: Forget to close browser
driver = webdriver.Chrome()
driver.get('https://example.com')

Good Practice:

# Do: Use explicit waits
wait = WebDriverWait(driver, 10)
element = wait.until(EC.presence_of_element_located((By.ID, 'element')))

# Do: Handle exceptions
try:
    element = driver.find_element(By.ID, 'element')
    element.click()
except NoSuchElementException:
    print("Element not found")

# Do: Use appropriate waits
driver.set_page_load_timeout(10)

# Do: Close browser properly
try:
    driver.get('https://example.com')
finally:
    driver.quit()

Conclusion

Selenium is the industry standard for web automation and testing. Master element location, waiting strategies, and interaction patterns to build robust automation scripts. Use explicit waits, handle exceptions properly, and organize code with Page Object Model for maintainability. Combine with testing frameworks like unittest or pytest for comprehensive test automation.