Effortless YouTube Data Scraping: Tools, Proxies, and Tips

from yt_dlp import YoutubeDL
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service as ChromeService # Renamed to avoid conflict
from selenium.webdriver.chrome.options import Options
import time
import pandas as pd # Added pandas for potential later use

# --- Configuration ---
# Consider using Evomi proxies here for larger scrapes
# proxy_server = "rp.evomi.com:1000" # Example HTTP endpoint
# options.add_argument(f'--proxy-server={proxy_server}')

def find_video_links(search_term):
    """Uses Selenium to search YouTube and return video links."""
    options = Options()
    options.add_argument("--headless=new") # Runs browser in background
    # Add proxy options here if needed
    driver = webdriver.Chrome(options=options)
    search_url = f'https://www.youtube.com/results?search_query={search_term.replace(" ", "+")}'
    driver.get(search_url)
    # Allow time for dynamic content loading
    time.sleep(3) # Adjust based on network speed
    video_links = []
    # Find video anchor tags by their ID
    video_elements = driver.find_elements(By.XPATH, '//a[@id="video-title"]')
    for element in video_elements:
        link = element.get_attribute('href')
        if link: # Ensure we got a valid link
            video_links.append(link)
    driver.quit()
    return video_links[:10] # Limit results for this example

def fetch_video_metadata(video_url):
    """Uses yt-dlp to extract metadata from a single video URL."""
    ydl_opts = {
        'quiet': True,        # Suppress console output from yt-dlp
        'skip_download': True, # We only want metadata
        'forcejson': True,    # Force metadata extraction as JSON
        'noplaylist': True,   # Process only single video if URL is part of playlist
    }
    try:
        with YoutubeDL(ydl_opts) as ydl:
            info = ydl.extract_info(video_url, download=False)
            # Extract specific fields, providing defaults if missing
            metadata = {
                'URL': video_url,
                'Title': info.get("title", "N/A"),
                'Views': info.get("view_count", 0),
                'Description': info.get("description", "N/A")[:200] + '...', # Truncate long descriptions
                'Uploader': info.get("uploader", "N/A")
            }
            return metadata
    except Exception as e:
        print(f"Error fetching metadata for {video_url}: {e}")
        return None # Return None on error

# --- Main Execution ---
search_topic = 'python scraping tutorial' # Changed example query
print(f"Searching for videos related to: '{search_topic}'")
links = find_video_links(search_topic)

if links:
    print(f"Found {len(links)} video links. Fetching metadata...")
    all_metadata = []
    for link in links:
        meta = fetch_video_metadata(link)
        if meta: # Check if metadata fetch was successful
             all_metadata.append(meta)
             print(f"  - Fetched: {meta['Title']}")
        time.sleep(0.5) # Small delay between requests

    # Display collected data (or save to CSV - see later example)
    print("\n--- Collected Metadata ---")
    for meta_item in all_metadata:
        print(f"Title: {meta_item['Title']}, Views: {meta_item['Views']}, Uploader: {meta_item['Uploader']}")
else:
    print("No video links found for the search query.")

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service as ChromeService
from selenium.webdriver.chrome.options import Options
import time
import pandas as pd

# --- Configuration ---
# Consider adding Evomi proxy configurations here
# proxy_details = "rp.evomi.com:1000"
# chrome_options.add_argument(f'--proxy-server={proxy_details}')


def scrape_video_comments(video_url, max_comments=50):
    """Scrolls down a YouTube video page and extracts comments."""
    chrome_options = Options()
    chrome_options.add_argument("--headless=new")
    chrome_options.add_argument("--disable-gpu")  # Often needed for headless
    chrome_options.add_argument("--no-sandbox")   # Sometimes needed in certain environments
    # Add proxy options here if needed

    driver = webdriver.Chrome(options=chrome_options)
    driver.get(video_url)
    print(f"Loading comments for: {video_url}")

    # Wait for initial page elements
    time.sleep(5)

    # Scroll down multiple times to load comments
    last_height = driver.execute_script("return document.documentElement.scrollHeight")
    scroll_attempts = 0
    max_scroll_attempts = 15  # Limit scrolls to prevent infinite loops

    while scroll_attempts < max_scroll_attempts:
        driver.execute_script("window.scrollTo(0, document.documentElement.scrollHeight);")
        time.sleep(3)  # Wait for comments to load
        new_height = driver.execute_script("return document.documentElement.scrollHeight")

        if new_height == last_height:
            # Check if a "Show more" button exists for replies (optional)
            # If so, click it and continue scrolling
            break  # Exit if page height stops changing

        last_height = new_height
        scroll_attempts += 1

        # Optional: Check comment count and break if max_comments reached
        current_comments = driver.find_elements(By.XPATH, '//*[@id="content-text"]')
        if len(current_comments) >= max_comments:
            print(f"Reached target comment count ({max_comments}).")
            break

    # Extract comment text
    comments_list = []
    try:
        comment_elements = driver.find_elements(By.XPATH, '//*[@id="content-text"]')
        for element in comment_elements[:max_comments]:  # Limit to max_comments
             comment_text = element.text.strip()
             if comment_text:  # Avoid empty comments
                 comments_list.append(comment_text)
    except Exception as e:
        print(f"Error extracting comments: {e}")

    driver.quit()
    print(f"Extracted {len(comments_list)} comments.")
    return comments_list


# --- Main Execution ---
target_video_url = 'https://www.youtube.com/watch?v=dQw4w9WgXcQ'  # Example URL (replace with a real video)
extracted_comments = scrape_video_comments(target_video_url, max_comments=100)

if extracted_comments:
    # Print a few comments as a sample
    print("\n--- Sample Comments ---")
    for i, comment in enumerate(extracted_comments[:5]):
        print(f"{i+1}: {comment}")

    # Optionally save to CSV (see next section)
    # comments_df = pd.DataFrame({'Comment': extracted_comments})
    # comments_df.to_csv('youtube_video_comments.csv', index=False)
    # print("\nComments saved to youtube_video_comments.csv")
else:
    print("No comments were extracted.")

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service as ChromeService
from selenium.webdriver.chrome.options import Options
import time
import pandas as pd  # Ensure pandas is imported

# --- Configuration ---
# Add proxy configurations if needed
# proxy_details = "rp.evomi.com:1000"
# chrome_options.add_argument(f'--proxy-server={proxy_details}')

def scrape_and_save_comments(video_url, output_filename='youtube_scrape_data.csv', max_comments=50):
    """Scrolls, extracts comments, and saves them to a CSV file."""
    chrome_options = Options()
    chrome_options.add_argument("--headless=new")
    chrome_options.add_argument("--disable-gpu")
    chrome_options.add_argument("--no-sandbox")
    # Add proxy options here if needed

    driver = webdriver.Chrome(options=chrome_options)
    driver.get(video_url)
    print(f"Loading comments for: {video_url}")
    time.sleep(5)  # Initial wait

    # Scrolling logic
    last_height = driver.execute_script("return document.documentElement.scrollHeight")
    scroll_attempts = 0
    max_scroll_attempts = 15

    while scroll_attempts < max_scroll_attempts:
        driver.execute_script("window.scrollTo(0, document.documentElement.scrollHeight);")
        time.sleep(3)
        new_height = driver.execute_script("return document.documentElement.scrollHeight")
        if new_height == last_height:
            break
        last_height = new_height
        scroll_attempts += 1

        current_comments = driver.find_elements(By.XPATH, '//*[@id="content-text"]')
        if len(current_comments) >= max_comments:
            print(f"Reached target comment count ({max_comments}).")
            break

    # Extract comments
    comments_data = []
    try:
        comment_elements = driver.find_elements(By.XPATH, '//*[@id="content-text"]')
        for element in comment_elements[:max_comments]:
            comment_text = element.text.strip()
            if comment_text:
                # Store comment along with the video URL for context
                comments_data.append({'VideoURL': video_url, 'Comment': comment_text})
    except Exception as e:
        print(f"Error extracting comments: {e}")

    driver.quit()

    # Save data to CSV using pandas
    if comments_data:
        df = pd.DataFrame(comments_data)
        try:
            df.to_csv(output_filename, index=False, encoding='utf-8')
            print(f"Successfully saved {len(comments_data)} comments to {output_filename}")
        except Exception as e:
            print(f"Error saving comments to CSV: {e}")
    else:
        print("No comments extracted to save.")

# --- Main Execution ---
target_video = 'https://www.youtube.com/watch?v=VIDEO_ID_HERE'  # Replace VIDEO_ID_HERE with an actual ID
csv_file = 'youtube_comments_output.csv'  # Define output file name
scrape_and_save_comments(target_video, output_filename=csv_file, max_comments=100)