Selenium+代理IP实战：搞定JS渲染页面的数据抓取方案

# 核心爬虫库
pip install selenium
# 代理IP请求、数据解析辅助库
pip install requests lxml beautifulsoup4
# 线程休眠、随机等待辅助
pip install time random

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import requests
import time
import random

# 1. 动态获取可用代理IP
def get_proxy_ip():
    """调用代理IP接口，获取随机可用代理"""
    # 替换为你的代理IP提取接口
    proxy_api = "https://www.zdaye.com/cart/?t=TunnelProxy"
    try:
        res = requests.get(proxy_api, timeout=10)
        if res.status_code == 200:
            proxy = res.text.strip()
            # 校验代理IP格式
            if ":" in proxy:
                return proxy
        return None
    except Exception as e:
        print(f"代理IP获取失败：{str(e)}")
        return None

# 2. 配置Selenium浏览器参数（代理+无头+反检测）
def init_browser():
    chrome_options = Options()
    
    # 基础反爬配置
    chrome_options.add_argument("--no-sandbox")
    chrome_options.add_argument("--disable-dev-shm-usage")
    chrome_options.add_argument("--disable-blink-features=AutomationControlled")
    chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])
    chrome_options.add_experimental_option("useAutomationExtension", False)
    
    # 开启无头模式（后台运行，关闭可显示浏览器界面）
    chrome_options.add_argument("--headless=new")
    
    # 配置代理IP
    proxy_ip = get_proxy_ip()
    if proxy_ip:
        chrome_options.add_argument(f"--proxy-server=http://{proxy_ip}")
        print(f"当前使用代理IP：{proxy_ip}")
    else:
        print("未获取到有效代理IP，使用本地IP访问")
    
    # 初始化浏览器
    driver = webdriver.Chrome(options=chrome_options)
    # 隐藏自动化特征
    driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
        "source": "Object.defineProperty(navigator, 'webdriver', {get: () => undefined})"
    })
    # 设置页面加载超时、元素等待超时
    driver.set_page_load_timeout(30)
    driver.implicitly_wait(10)
    return driver

# 3. JS渲染页面数据抓取核心函数
def crawl_dynamic_page(url):
    driver = init_browser()
    try:
        # 访问目标页面
        driver.get(url)
        # 随机休眠2-5秒，模拟人工访问节奏
        time.sleep(random.uniform(2, 5))
        
        # 显式等待核心元素加载完成（适配JS动态渲染）
        # 此处根据目标页面核心数据元素修改定位方式
        WebDriverWait(driver, 20).until(
            EC.presence_of_element_located((By.CLASS_NAME, "target-data-item"))
        )
        
        # 可选：滚动页面触发懒加载数据
        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        time.sleep(random.uniform(1, 3))
        
        # 获取渲染完成后的完整页面源码
        page_html = driver.page_source
        
        # 数据解析（可结合BeautifulSoup/Xpath自定义解析规则）
        from bs4 import BeautifulSoup
        soup = BeautifulSoup(page_html, "lxml")
        data_list = []
        items = soup.find_all("div", class_="target-data-item")
        for item in items:
            data = {
                "标题": item.find("h3").get_text(strip=True) if item.find("h3") else "",
                "内容": item.find("p").get_text(strip=True) if item.find("p") else "",
                "链接": item.find("a")["href"] if item.find("a") else ""
            }
            data_list.append(data)
        
        print(f"成功抓取数据条数：{len(data_list)}")
        return data_list
    
    except Exception as e:
        print(f"页面抓取异常：{str(e)}")
        return None
    finally:
        # 关闭浏览器，释放资源
        driver.quit()

# 4. 批量抓取主函数（自动轮换IP重试）
def batch_crawl(url_list, retry_times=3):
    all_data = []
    for url in url_list:
        for i in range(retry_times):
            res_data = crawl_dynamic_page(url)
            if res_data:
                all_data.extend(res_data)
                break
            else:
                print(f"第{i+1}次重试抓取：{url}")
                time.sleep(random.uniform(3, 6))
    return all_data

# 程序入口
if __name__ == "__main__":
    # 待抓取的JS渲染页面链接列表
    target_urls = [
        "https://xxx.xxx.com/dynamic-page1",
        "https://xxx.xxx.com/dynamic-page2"
    ]
    result_data = batch_crawl(target_urls)
    # 可扩展：数据保存到Excel/数据库/JSON文件
    print("全部抓取任务完成")