Scrape Reddit with Python: Powering Data with Proxies

import time  # We'll need this later for pauses
import requests
from bs4 import BeautifulSoup

def fetch_page_content(target_url, proxy_config=None):
    """
    Fetches HTML content for a specific URL, with optional proxy support.

    Args:
        target_url (str): The URL to retrieve content from.
        proxy_config (dict, optional): Dictionary defining proxies for HTTP/HTTPS.
                                       Defaults to None.

    Returns:
        str: HTML content of the page, or None if an error occurs.
    """
    # Using a realistic User-Agent header is important to avoid blocks
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
    }
    try:
        response = requests.get(
            target_url,
            headers=headers,
            proxies=proxy_config,
            timeout=15  # Increased timeout
        )
        # Check for successful response (HTTP status code 200)
        if response.status_code == 200:
            return response.text
        else:
            print(f"Failed to fetch {target_url}. Status code: {response.status_code}")
            return None
    except requests.exceptions.RequestException as e:
        # Catch potential network errors, timeouts, etc.
        print(f"Error fetching {target_url}: {e}")
        return None

def parse_subreddit_metadata(page_html):
    """
    Extracts subreddit title, description, and subscriber count from HTML.

    Args:
        page_html (str): The HTML content of the subreddit page.

    Returns:
        dict: A dictionary containing the metadata, or 'Not found' values.
    """
    if not page_html:
        return {
            'Title': 'Not found',
            'Description': 'Not found',
            'Subscribers': 'Not found'
        }

    parsed_html = BeautifulSoup(page_html, 'html.parser')

    # Reddit uses custom elements; we need to find the right one
    # Note: Selectors might change if Reddit updates its structure.
    header_element = parsed_html.find('shreddit-subreddit-header')

    if header_element:
        # Attributes within the element hold the data
        title = header_element.get('display-name', 'Not found') # Default if attr missing
        description = header_element.get('description', 'Not found')
        sub_count = header_element.get('subscribers', 'Not found')
        return {
            'Title': title,
            'Description': description,
            'Subscribers': sub_count
        }
    else:
        print("Could not find the subreddit header element.")
        return {
            'Title': 'Not found',
            'Description': 'Not found',
            'Subscribers': 'Not found'
        }

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options # For headless mode etc.
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

def scrape_posts_selenium(subreddit_url, scroll_attempts=5, delay=2):
    """
    Uses Selenium to load a subreddit page, scroll down, and extract post titles and URLs.

    Args:
        subreddit_url (str): The URL of the subreddit.
        scroll_attempts (int): How many times to simulate scrolling down.
        delay (int): Seconds to wait between scrolls for content to load.

    Returns:
        tuple: A tuple containing two lists: (post_titles, post_urls).
    """
    print(f"Setting up Selenium WebDriver...")
    options = Options()
    options.add_argument("--headless=new") # Run Chrome in headless mode (no UI window)
    options.add_argument("--no-sandbox") # Often needed in Linux environments
    options.add_argument("--disable-dev-shm-usage") # Overcome limited resource problems
    options.add_argument('user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36')
    # If using proxies with Selenium, setup is different (more below)
    driver = webdriver.Chrome(options=options)

    print(f"Navigating to {subreddit_url}...")
    driver.get(subreddit_url)

    # Wait a bit for initial page load before scrolling
    time.sleep(delay + 1)

    print(f"Scrolling down {scroll_attempts} times...")
    body_element = driver.find_element(By.TAG_NAME, 'body')
    for i in range(scroll_attempts):
        body_element.send_keys(Keys.PAGE_DOWN)
        print(f"Scroll {i+1}/{scroll_attempts}, waiting {delay}s...")
        time.sleep(delay) # Allow time for new posts to load via JavaScript

    print("Scrolling finished. Extracting posts...")
    post_titles = []
    post_urls = []

    # Selector targets the link element containing the post title
    # This XPath might need adjustment if Reddit's structure changes.
    post_elements_xpath = '//a[@slot="full-post-link"]'

    try:
        # Wait until at least some post elements are present
        WebDriverWait(driver, 10).until(
            EC.presence_of_all_elements_located((By.XPATH, post_elements_xpath))
        )
        post_elements = driver.find_elements(By.XPATH, post_elements_xpath)
        print(f"Found {len(post_elements)} post elements.")

        for element in post_elements:
            try:
                # Extract URL from the href attribute
                href = element.get_attribute('href')
                if href:
                    # Ensure URL is absolute
                    if href.startswith("/"):
                        href = "https://www.reddit.com" + href
                    post_urls.append(href)

                    # Find the title text within the link element
                    # The structure might involve nested elements.
                    # Adjust the selector based on actual page structure if needed
                    title_element = element.find_element(By.TAG_NAME, 'faceplate-screen-reader-content') # Specific to potential Reddit structure
                    title = title_element.text if title_element else 'Title not found'
                    post_titles.append(title.strip()) # Add stripped title
                else:
                    print("Found element without href.")
            except Exception as e_inner:
                print(f"Error extracting data from one post element: {e_inner}")
                # Append placeholders if extraction fails for one element
                # post_urls.append('URL extraction error')
                # post_titles.append('Title extraction error')

    except Exception as e_outer:
        print(f"Error finding post elements or timeout: {e_outer}")

    driver.quit() # Important: Close the browser window/process
    print("WebDriver closed.")
    return post_titles, post_urls

# Example Test Run (without proxies for now)
test_url = 'https://www.reddit.com/r/learnpython/' # Using a different subreddit
titles, urls = scrape_posts_selenium(test_url, scroll_attempts=3, delay=3) # Fewer scrolls for test

print(f"\n--- Extracted {len(titles)} Titles ---")
#for t in titles: print(t) # Uncomment to print titles
print(f"\n--- Extracted {len(urls)} URLs ---")
#for u in urls: print(u) # Uncomment to print URLs

import time
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

# Assuming scrape_posts_selenium is defined elsewhere and returns (list, list)
# from your_module import scrape_posts_selenium
# Example placeholder function
def scrape_posts_selenium(url, scroll_attempts, delay):
    print(f"Placeholder: Scraping posts from {url}...")
    # Simulate finding some posts
    return ["Post Title 1", "Post Title 2", "Post Title 3", "Post Title 4", "Post Title 5", "Post Title 6"], \
           [f"{url}/post1", f"{url}/post2", f"{url}/post3", f"{url}/post4", f"{url}/post5", f"{url}/post6"]

test_url = "https://www.reddit.com/r/example" # Placeholder test URL


def scrape_comments_selenium(post_urls, max_posts_to_scrape=10, comment_delay=2.5):
    """
    Visits a list of post URLs, scrolls to load comments, and extracts them using Selenium.

    Args:
        post_urls (list): A list of URLs for the Reddit posts.
        max_posts_to_scrape (int): Limit the number of posts to process (for testing/efficiency).
        comment_delay (float): Seconds to wait during comment loading scrolls.

    Returns:
        dict: A dictionary mapping each post URL to a list of its extracted comments.
    """
    print(f"Setting up Selenium WebDriver for comments...")
    options = Options()
    options.add_argument("--headless=new")
    options.add_argument("--no-sandbox")
    options.add_argument("--disable-dev-shm-usage")
    options.add_argument(
        'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
    )
    # Proxy setup for Selenium would go here if needed
    driver = webdriver.Chrome(options=options)

    all_comments_data = {}
    processed_count = 0
    print(f"Processing {min(len(post_urls), max_posts_to_scrape)} post URLs for comments...")

    for url in post_urls[:max_posts_to_scrape]:  # Limit processing
        print(f"\nProcessing comments for: {url}")
        try:
            driver.get(url)
            time.sleep(comment_delay + 1)  # Initial load wait

            # Scroll down to try and load most comments
            # Using JavaScript execution for scrolling might be more robust here
            last_height = driver.execute_script("return document.body.scrollHeight")
            scroll_pause_time = comment_delay
            scroll_attempts = 0
            max_scroll_attempts = 15  # Limit scroll attempts per page

            while scroll_attempts < max_scroll_attempts:
                print(f"Scrolling down post page (Attempt {scroll_attempts + 1})...")
                driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
                time.sleep(scroll_pause_time)
                new_height = driver.execute_script("return document.body.scrollHeight")
                if new_height == last_height:
                    print("Reached bottom or no new content loaded.")
                    break  # Exit scroll loop if height doesn't change
                last_height = new_height
                scroll_attempts += 1

            # Extract comments after scrolling
            current_post_comments = []
            try:
                # This selector targets the comment elements. It's highly subject to change.
                comment_elements_selector = 'shreddit-comment'
                # Wait briefly for comment elements to potentially appear
                WebDriverWait(driver, 5).until(
                    EC.presence_of_all_elements_located((By.TAG_NAME, comment_elements_selector))
                )
                comment_elements = driver.find_elements(By.TAG_NAME, comment_elements_selector)
                print(f"Found {len(comment_elements)} comment elements.")

                for comment_el in comment_elements:
                    try:
                        # Try finding the comment text container within the element
                        # The specific structure (div ID/class) might vary significantly
                        comment_text_div = comment_el.find_element(By.CSS_SELECTOR, 'div[id$="-comment-rtjson-content"]')
                        if comment_text_div:
                             comment_text = comment_text_div.text
                             current_post_comments.append(comment_text.strip())
                        # else: # Alternative selector if the above fails
                        #     comment_text = comment_el.text # Less precise fallback
                        #     current_post_comments.append(comment_text.strip())
                    except Exception as e_comment:
                        # print(f"Could not extract text from one comment element: {e_comment}")
                        pass  # Silently skip comments that can't be parsed cleanly

            except Exception as e_find_comments:
                print(f"Error finding or processing comment elements on {url}: {e_find_comments}")

            all_comments_data[url] = current_post_comments
            print(f"Extracted {len(current_post_comments)} comments for this post.")
            processed_count += 1

        except Exception as e_page_load:
            print(f"Error loading or processing page {url}: {e_page_load}")
            all_comments_data[url] = [] # Ensure entry exists even on failure

    driver.quit()
    print(f"\nFinished processing {processed_count} posts. WebDriver closed.")
    return all_comments_data


# Example: Get URLs first, then comments for a few posts
post_titles_list, post_urls_list = scrape_posts_selenium(test_url, scroll_attempts=3, delay=3)

if post_urls_list:
    comments_result = scrape_comments_selenium(post_urls_list, max_posts_to_scrape=5) # Scrape comments for first 5 posts
    # print("\n--- Comments Data ---")
    # for post_link, comments in comments_result.items():
    #     print(f"\nComments for {post_link}:")
    #     if comments:
    #         for i, comment in enumerate(comments[:3]): # Print first 3 comments
    #             print(f"  {i+1}. {comment[:100]}...") # Truncate long comments
    #     else:
    #         print("  No comments extracted.")
else:
    print("No post URLs found to scrape comments.")

import pandas as pd  # openpyxl was already installed, pandas uses it for .xlsx files

def parse_subreddit_metadata(page_html):
    # ... (parsing logic remains the same until the return statement) ...
    if header_element:
        # ... (extract title, description, sub_count) ...
        metadata_dict = {
            'Title': [title],  # Put values in lists for DataFrame creation
            'Description': [description],
            'Subscribers': [sub_count]
        }
        return pd.DataFrame(metadata_dict)
    else:
        print("Could not find the subreddit header element.")
        return pd.DataFrame({  # Return empty or 'Not found' DataFrame
            'Title': ['Not found'],
            'Description': ['Not found'],
            'Subscribers': ['Not found']
        })

def scrape_posts_selenium(subreddit_url, scroll_attempts=5, delay=2):
    # ... (WebDriver setup, navigation, scrolling, extraction loop) ...
    driver.quit()
    print("WebDriver closed.")
    # Create DataFrames from the lists
    df_posts = pd.DataFrame({
        'Title': post_titles,
        'URL': post_urls
    })
    # Return the DataFrame AND the list of URLs for the next step
    return df_posts, post_urls

def scrape_comments_selenium(post_urls, max_posts_to_scrape=10, comment_delay=2.5):
    # ... (WebDriver setup, loop through URLs, scrolling, comment extraction) ...
    # Instead of adding to all_comments_data dict directly, build lists
    all_post_links = []
    all_comments = []

    # Inside the loop after extracting current_post_comments:
    for comment_text in current_post_comments:
        all_post_links.append(url)  # Add the post URL for each comment
        all_comments.append(comment_text)

    # ... (end of loop) ...
    driver.quit()
    print(f"\nFinished processing {processed_count} posts. WebDriver closed.")

    # Create DataFrame from the aggregated lists
    df_comments = pd.DataFrame({
        'Post URL': all_post_links,
        'Comment': all_comments
    })
    return df_comments

# --- Main Execution & Export ---
target_subreddit_url = 'https://www.reddit.com/r/programming/'
output_filename = 'reddit_programming_data.xlsx'

print("Step 1: Fetching initial page content...")
# Use requests for the static metadata if possible (less resource intensive)
initial_html = fetch_page_content(target_subreddit_url)
df_meta = parse_subreddit_metadata(initial_html)
subreddit_title = df_meta['Title'].iloc[0] if not df_meta.empty else 'Subreddit'
print(f"Parsed metadata for: {subreddit_title}")

print("\nStep 2: Scraping post titles and URLs with Selenium...")
# Limit scrolls/posts for faster execution during testing
df_posts_data, list_of_post_urls = scrape_posts_selenium(
    target_subreddit_url, scroll_attempts=4, delay=2.5
)
print(f"Found {len(list_of_post_urls)} posts.")

print("\nStep 3: Scraping comments for selected posts with Selenium...")
if list_of_post_urls:
    # Limit comment scraping to avoid excessive run time
    df_comments_data = scrape_comments_selenium(
        list_of_post_urls, max_posts_to_scrape=8, comment_delay=3
    )
    print(f"Extracted {len(df_comments_data)} comments in total.")
else:
    df_comments_data = pd.DataFrame({'Post URL': [], 'Comment': []}) # Empty DataFrame
    print("Skipping comment scraping as no post URLs were found.")

print(f"\nStep 4: Exporting data to {output_filename}...")
try:
    with pd.ExcelWriter(output_filename, engine='openpyxl') as writer:
        df_meta.to_excel(writer, sheet_name='Metadata', index=False)
        df_posts_data.to_excel(writer, sheet_name=f'{subreddit_title}_Posts', index=False)
        df_comments_data.to_excel(writer, sheet_name=f'{subreddit_title}_Comments', index=False)
    print("Data successfully exported.")
except Exception as e:
    print(f"Error exporting data to Excel: {e}")

# Example proxy setup for 'requests' (replace with your actual details)
# Format: protocol: 'http://username:password@proxy_host:port'
evomi_residential_endpoint = 'rp.evomi.com:1000'  # Example Evomi endpoint structure
proxy_user = 'your_username'
proxy_pass = 'your_password'
requests_proxy_config = {
    'http': f'http://{proxy_user}:{proxy_pass}@{evomi_residential_endpoint}',
    'https': f'http://{proxy_user}:{proxy_pass}@{evomi_residential_endpoint}'  # Use http for https traffic too with some providers
    # Consult Evomi documentation for exact HTTPS proxy format if different
}
# Then pass it to the function:
# initial_html = fetch_page_content(target_subreddit_url, proxy_config=requests_proxy_config)

# Conceptual Selenium Wire Setup (requires library import and function changes)
# from seleniumwire import webdriver  # Use this instead of selenium.webdriver

# Define proxy options for Selenium Wire
wire_options = {
    'proxy': {
        'http': f'http://{proxy_user}:{proxy_pass}@{evomi_residential_endpoint}',
        'https': f'https://{proxy_user}:{proxy_pass}@{evomi_residential_endpoint}',  # Check provider docs for correct format
        'no_proxy': 'localhost,127.0.0.1'  # Exclude local addresses
    }
}

# When initializing the driver in your Selenium functions:
# options = Options()  # Your regular Chrome options (headless, user-agent etc.)
# driver = webdriver.Chrome(seleniumwire_options=wire_options, options=options)
# Note: This replaces the standard driver initialization.