Rust & Selenium: High-Speed Web Scraping with Proxies

cargo new rust_selenium_scraper
cd

[dependencies]
thirtyfour = "0.31.0" # Check for the latest compatible version
tokio = { version = "1", features = ["full"] } # Use the latest 1.x version with full features

use thirtyfour::prelude::*;
use tokio::time::{sleep, Duration};

#[tokio::main]
async fn main() -> WebDriverResult<()> {
    // Specify Chrome browser capabilities
    let caps = DesiredCapabilities::chrome();

    // Connect to the running ChromeDriver server
    let driver = WebDriver::new("http://localhost:9515", caps).await?;

    // Navigate to the target page
    driver.goto("http://quotes.toscrape.com/scroll").await?;
    println!("Successfully navigated to the page!");

    // Pause for a few seconds to observe
    sleep(Duration::from_secs(3)).await;

    // Close the browser session
    driver.quit().await?;
    println!("Browser session closed.");

    Ok(())
}

// Find all elements with the CSS class "quote"
let quote_elements = driver.find_all(By::Css(".quote")).await?;

// Create a vector to store the results (Quote Text, Author)
let mut collected_quotes: Vec<(String, String)> = Vec::new();

// Loop through each quote element found
for quote_element in quote_elements {
    // Find the text span inside the current quote element
    let text = quote_element.find(By::Css(".text")).await?.text().await?;
    // Find the author span inside the current quote element
    let author = quote_element.find(By::Css(".author")).await?.text().await?;
    // Add the extracted data as a tuple to our vector
    collected_quotes.push((text, author));
}

// Print the collected quotes
println!("\n--- Collected Quotes ---");
for (quote_text, author_name) in &collected_quotes {
    println!("\"{}\" - {}", quote_text, author_name);
}
println!("----------------------\n");

use thirtyfour::prelude::*; // Note: Tokio sleep is not needed for this part
// use tokio::time::{sleep, Duration};

#[tokio::main]
async fn main() -> WebDriverResult<()> {
    let caps = DesiredCapabilities::chrome();
    let driver = WebDriver::new("http://localhost:9515", caps).await?;

    driver.goto("http://quotes.toscrape.com/scroll").await?;
    println!("Navigated to quotes page.");

    // Find all elements with the CSS class "quote"
    let quote_elements = driver.find_all(By::Css(".quote")).await?;
    println!("Found {} quote elements initially.", quote_elements.len());

    // Create a vector to store the results (Quote Text, Author)
    let mut collected_quotes: Vec<(String, String)> = Vec::new();

    // Loop through each quote element found
    for quote_element in quote_elements {
        let text = quote_element.find(By::Css(".text")).await?.text().await?;
        let author = quote_element.find(By::Css(".author")).await?.text().await?;
        collected_quotes.push((text, author));
    }

    // Print the collected quotes
    println!("\n--- Collected Quotes (Initial Load) ---");
    for (quote_text, author_name) in &collected_quotes {
        println!("\"{}\" - {}", quote_text, author_name);
    }
    println!("-------------------------------------\n");

    driver.quit().await?;
    println!("Browser session closed.");

    Ok(())
}

use thirtyfour::prelude::*;
use tokio::time::{sleep, Duration}; // Make sure sleep is imported

#[tokio::main]
async fn main() -> WebDriverResult<()> {
    let caps = DesiredCapabilities::chrome();
    let driver = WebDriver::new("http://localhost:9515", caps).await?;

    driver.goto("http://quotes.toscrape.com/scroll").await?;
    println!("Navigated to quotes page.");

    let scroll_attempts = 3; // How many times to scroll down
    println!("Attempting to scroll down {} times...", scroll_attempts);

    for i in 0..scroll_attempts {
        // Find all current quote elements
        let current_quotes = driver.find_all(By::Css(".quote")).await?;

        // Get the last element, if any exist
        if let Some(last_quote) = current_quotes.last() {
            println!("Scrolling attempt {}: Bringing last quote into view...", i + 1);
            last_quote.scroll_into_view().await?;
            // Wait a moment for new content to potentially load
            sleep(Duration::from_secs(2)).await;
        } else {
            println!("No quote elements found to scroll to.");
            break; // Exit loop if no quotes are found
        }
    }

    println!("Scrolling finished.");

    // Now, collect ALL quotes present after scrolling
    let all_quote_elements = driver.find_all(By::Css(".quote")).await?;
    println!("Found {} quote elements after scrolling.", all_quote_elements.len());

    let mut final_quotes: Vec<(String, String)> = Vec::new();
    for quote_element in all_quote_elements {
        // Use a helper function or inline error handling for potentially missing elements
        let text_result = quote_element.find(By::Css(".text")).await;
        let author_result = quote_element.find(By::Css(".author")).await;

        if let (Ok(text_elem), Ok(author_elem)) = (text_result, author_result) {
            let text = text_elem.text().await?;
            let author = author_elem.text().await?;
            final_quotes.push((text, author));
        } else {
            println!("Warning: Could not find text or author for a quote element.");
        }
    }

    // Print the final collected quotes
    println!("\n--- Collected Quotes (After Scrolling) ---");
    for (quote_text, author_name) in &final_quotes {
        println!("\"{}\" - {}", quote_text, author_name);
    }
    println!("Total quotes collected: {}", final_quotes.len());
    println!("----------------------------------------\n");

    driver.quit().await?;
    println!("Browser session closed.");

    Ok(())
}

use thirtyfour::prelude::*;
use tokio::time::{sleep, Duration};

#[tokio::main]
async fn main() -> WebDriverResult<()> {
    let caps = DesiredCapabilities::chrome();
    let driver = WebDriver::new("http://localhost:9515", caps).await?;

    // Start at the main scroll page (or any page with a login link)
    driver.goto("http://quotes.toscrape.com/scroll").await?;
    println!("Navigated to initial page.");

    // Find the login link - using XPath to find link by its text content
    println!("Looking for login link...");
    let login_link = driver.find(By::XPath("//a[contains(text(), 'Login')]")).await?;
    println!("Found login link, clicking...");
    login_link.click().await?;

    // Wait briefly for the login page to load
    sleep(Duration::from_secs(2)).await;
    println!("On login page.");

    // Find username field by its ID
    println!("Finding username field...");
    let username_input = driver.find(By::Css("#username")).await?;
    println!("Entering username...");
    username_input.send_keys("user").await?; // Example username

    // Find password field by its ID
    println!("Finding password field...");
    let password_input = driver.find(By::Css("#password")).await?;
    println!("Entering password...");
    password_input.send_keys("password").await?; // Example password

    // Find the submit button (input type=submit)
    println!("Finding submit button...");
    let submit_button = driver.find(By::Css("input[type='submit']")).await?;
    println!("Clicking submit...");
    submit_button.click().await?;

    // Wait to see the result (logged-in page or error)
    println!("Submitted login form. Waiting...");
    sleep(Duration::from_secs(5)).await;

    // Check the current URL or look for an element indicating success/failure (optional)
    let current_url = driver.current_url().await?;
    println!("Current URL after login attempt: {}", current_url);

    driver.quit().await?;
    println!("Browser session closed.");

    Ok(())
}

use thirtyfour::{
    common::capabilities::proxy::Proxy, prelude::*, CapabilitiesHelper,
};

let proxy_url = "http://your-proxy-endpoint:port"; // Replace with your actual proxy address
let mut caps = DesiredCapabilities::chrome();
let mut proxy_config = Proxy::manual();
proxy_config.with_http_proxy(proxy_url)?;
proxy_config.with_ssl_proxy(proxy_url)?; // Often the same as HTTP proxy
caps.set_proxy(proxy_config)?;
// Now create the driver with these capabilities
let driver = WebDriver::new("http://localhost:9515", caps).await?;

use thirtyfour::{
    common::capabilities::proxy::Proxy, prelude::*, CapabilitiesHelper,
};
use tokio::time::{sleep, Duration};

#[tokio::main]
async fn main() -> WebDriverResult<()> {
    // --- Proxy Configuration ---
    // Replace with your actual proxy endpoint (authentification not yet setup!)
    let proxy_address = "http://rp.evomi.com:1000"; // Example Evomi HTTP endpoint for residential
    let mut caps = DesiredCapabilities::chrome();
    let mut proxy_settings = Proxy::manual();

    // Configure for HTTP and HTTPS traffic
    proxy_settings.with_http_proxy(proxy_address)?;
    proxy_settings.with_ssl_proxy(proxy_address)?; // Use same proxy for HTTPS

    // Apply proxy settings to capabilities
    caps.set_proxy(proxy_settings)?;
    println!("Proxy configured: {}", proxy_address);

    // --- End Proxy Configuration ---
    println!("Connecting to WebDriver...");
    let driver = WebDriver::new("http://localhost:9515", caps).await?;
    println!("WebDriver connection established.");

    // Navigate to an IP checking service
    let check_url = "https://geo.evomi.com/";
    println!("Navigating to IP checker: {}", check_url);
    driver.goto(check_url).await?;

    // Wait for page elements to load (adjust timing as needed)
    sleep(Duration::from_secs(5)).await;

    // Try to find the displayed IP address (Selector might vary depending on the site)
    // For geo.evomi.com, the IP is usually in a specific element. Let's try a common pattern.
    // NOTE: Inspect geo.evomi.com manually to find the correct selector if this fails.
    let ip_element_result = driver.find(By::Css(".ip-address-val")).await; // Adjust selector as needed!

    match ip_element_result {
        Ok(ip_element) => {
            let displayed_ip = ip_element.text().await?;
            println!("IP Address displayed on page: {}", displayed_ip);
        }
        Err(_) => {
            println!(
                "Could not find the IP address element automatically. Please check the page manually."
            );
            // Optional: Grab the whole body text as a fallback
            let body_text = driver.find(By::Css("body")).await?.text().await?;
            println!("Page body content:\n{}", body_text);
        }
    }

    driver.quit().await?;
    println!("Browser session closed.");

    Ok(())
}