Collect structured data at scale - without headless overhead.

from google.cloud import bigquery
import requests
import pandas as pd

# 1. Scrape Data
api_key = "YOUR_EVOMI_API_KEY"
target_url = "https://example.com"
response = requests.get(f"https://scrape.evomi.com/api/v1/scraper/realtime?api_key={api_key}&url={target_url}")
data = response.json()

# 2. Prepare Data for BigQuery (Example: converting to DataFrame)
# Adjust structure to match your schema
df = pd.DataFrame([{"url": target_url, "content": data.get("body"), "status": 200}])

# 3. Load to BigQuery
client = bigquery.Client()
table_id = "your_project.your_dataset.your_table"

job = client.load_table_from_dataframe(df, table_id)
job.result()  # Wait for the job to complete
print("Loaded scraped data into BigQuery")