Python爬虫代理IP配置模板｜自用半年、直接套用不踩坑

import requests
import random
import time
from requests.exceptions import ProxyError, Timeout, ConnectionError

# ====================== 【核心配置区 - 仅改这里即可】 ======================
# 替换成你自己的代理信息（静态IP/动态隧道代理都支持）
PROXY_HOST = "你的代理IP地址"
PROXY_PORT = "代理端口"
PROXY_USER = "代理账号"
PROXY_PWD = "代理密码"

# 爬虫自定义参数，按需微调
REQUEST_TIMEOUT = 10  # 单次请求超时时间
RETRY_TIMES = 3       # 失败最大重试次数
MIN_SLEEP = 0.5       # 最小请求间隔
MAX_SLEEP = 1.5       # 最大请求间隔
# ==========================================================================

# 统一获取代理配置，全局复用
def get_proxy():
    """组装代理链接，全局统一调用"""
    proxy_url = f"http://{PROXY_USER}:{PROXY_PWD}@{PROXY_HOST}:{PROXY_PORT}"
    proxies = {
        "http": proxy_url,
        "https": proxy_url
    }
    return proxies

# 模拟真实浏览器请求头，规避基础风控
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Accept": "*/*",
    "Accept-Language": "zh-CN,zh;q=0.9",
    "Connection": "keep-alive"
}

def spider_request(url, method="GET", data=None, params=None):
    """
    带代理、随机延时、失败重试的通用请求方法
    完美适配常规爬虫，不用重复写容错逻辑
    """
    # 随机休眠，模拟人工操作，避免高频请求被检测
    time.sleep(random.uniform(MIN_SLEEP, MAX_SLEEP))
    
    for i in range(RETRY_TIMES):
        try:
            # 发起带代理的网络请求
            response = requests.request(
                method=method,
                url=url,
                headers=HEADERS,
                proxies=get_proxy(),
                data=data,
                params=params,
                timeout=REQUEST_TIMEOUT,
                verify=False  # 关闭SSL验证，解决代理HTTPS报错问题
            )
            
            # 校验响应状态，只保留有效请求
            if response.status_code in [200, 201, 204]:
                return response
            else:
                print(f"请求状态码异常：{response.status_code}，正在第{i+1}次重试")
                continue
                
        except (ProxyError, ConnectionError):
            print(f"代理连接失败/IP失效，正在第{i+1}次重试")
            # 动态代理可在此添加刷新IP逻辑
            time.sleep(1)
        except Timeout:
            print(f"请求超时，正在第{i+1}次重试")
            time.sleep(1)
        except Exception as e:
            print(f"请求异常：{str(e)}，正在第{i+1}次重试")
            time.sleep(1)
    
    print(f"请求彻底失败，已达最大重试次数：{RETRY_TIMES}")
    return None

# 测试代理是否生效
if __name__ == "__main__":
    test_url = "http://httpbin.org/ip"
    res = spider_request(test_url)
    if res:
        print("✅ 代理配置生效！当前IP：", res.text)
    else:
        print("❌ 代理配置失败，请检查账号、密码、IP端口是否正确")

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import time

# ====================== 【核心配置区 - 按需修改】 ======================
PROXY_HOST = "你的代理IP地址"
PROXY_PORT = "代理端口"
PROXY_USER = "代理账号"
PROXY_PWD = "代理密码"
# ==========================================================================

def init_proxy_chrome_driver():
    """初始化带代理的无痕Chrome浏览器，自带防检测配置"""
    chrome_options = Options()
    
    # 基础反爬检测配置，规避selenium原生特征
    chrome_options.add_argument("--no-sandbox")
    chrome_options.add_argument("--disable-dev-shm-usage")
    chrome_options.add_argument("--disable-blink-features=AutomationControlled")
    chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])
    chrome_options.add_experimental_option("useAutomationExtension", False)
    
    # 绑定代理配置
    proxy_auth = f"{PROXY_USER}:{PROXY_PWD}@{PROXY_HOST}:{PROXY_PORT}"
    chrome_options.add_argument(f"--proxy-server=http://{proxy_auth}")
    
    # 无头模式（生产环境开启，调试时可注释掉）
    chrome_options.add_argument("--headless=new")
    
    driver = webdriver.Chrome(options=chrome_options)
    driver.implicitly_wait(5)
    return driver

# 浏览器代理测试
if __name__ == "__main__":
    driver = init_proxy_chrome_driver()
    driver.get("http://httpbin.org/ip")
    time.sleep(2)
    print("✅ 浏览器当前代理IP：", driver.page_source)
    driver.quit()