数据分析实战：从爬虫到数据挖掘的端到端项目全流程

import requests
import pandas as pd
import time
import random
from fake_useragent import UserAgent

def fetch_reviews_static(product_id, page_num=5):
    """
    模拟静态评论获取（实际中替换为真实API）
    返回DataFrame
    """
    ua = UserAgent()
    reviews = []
    for page in range(1, page_num+1):
        # 模拟请求参数
        url = f"https://dummyapi.com/reviews?product={product_id}&page={page}"
        headers = {'User-Agent': ua.random}
        try:
            # 这里为了演示，直接生成模拟数据
            # 实际中应为 response = requests.get(url, headers=headers)
            # 假设返回JSON
            mock_data = {
                'reviews': [
                    {
                        'content': f"商品质量非常好，性价比高，快递很快，{random.choice(['满意','推荐','下次还买'])}",
                        'score': random.randint(4,5),
                        'time': f"2026-{random.randint(1,8):02d}-{random.randint(1,28):02d}"
                    } for _ in range(10)
                ] + [
                    {
                        'content': f"收到货有瑕疵，屏幕有划痕，{random.choice(['退货','差评','不推荐'])}",
                        'score': random.randint(1,2),
                        'time': f"2026-{random.randint(1,8):02d}-{random.randint(1,28):02d}"
                    } for _ in range(3)
                ]
            }
            # 实际中: data = response.json()
            for item in mock_data['reviews']:
                reviews.append({
                    'content': item['content'],
                    'score': item['score'],
                    'time': pd.to_datetime(item['time'])
                })
            time.sleep(random.uniform(0.5, 1.5))
        except Exception as e:
            print(f"页 {page} 抓取失败: {e}")
    return pd.DataFrame(reviews)

# 使用
df_static = fetch_reviews_static(product_id='1000123', page_num=3)
print(df_static.head())

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options

def fetch_reviews_dynamic(url, scroll_times=3):
    """
    爬取动态加载评论（以京东为例）
    需安装chromedriver
    """
    chrome_options = Options()
    chrome_options.add_argument("--headless")  # 无头模式
    chrome_options.add_argument("--no-sandbox")
    chrome_options.add_argument("--disable-dev-shm-usage")
    
    driver = webdriver.Chrome(options=chrome_options)
    driver.get(url)
    reviews_data = []
    
    for _ in range(scroll_times):
        # 滚动到页面底部加载更多
        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        time.sleep(2)
        # 等待评论加载
        try:
            WebDriverWait(driver, 10).until(
                EC.presence_of_element_located((By.CSS_SELECTOR, ".comment-item"))
            )
        except:
            pass
    
    # 提取评论元素
    items = driver.find_elements(By.CSS_SELECTOR, ".comment-item")
    for item in items:
        try:
            content = item.find_element(By.CSS_SELECTOR, ".comment-content").text
            score = int(item.find_element(By.CSS_SELECTOR, ".comment-star").get_attribute("data-score"))
            time_str = item.find_element(By.CSS_SELECTOR, ".comment-time").text
            reviews_data.append({
                'content': content,
                'score': score,
                'time': pd.to_datetime(time_str, errors='coerce')
            })
        except:
            continue
    
    driver.quit()
    return pd.DataFrame(reviews_data)

# 使用示例（注释掉真实调用，避免依赖）
# df_dynamic = fetch_reviews_dynamic("https://item.jd.com/1000123.html#comment", scroll_times=5)

import pandas as pd
import re
import jieba
import jieba.posseg as pseg
from sklearn.model_selection import train_test_split

# 假设已有 df 合并数据
# df = pd.concat([df_static, df_dynamic], ignore_index=True)

# 构造示例数据（真实环境中用上面爬取结果）
df = pd.DataFrame({
    'content': [
        '手机很好用，速度飞快，拍照清晰，电池耐用',
        '垃圾产品，收到就坏了，售后态度差',
        '一般般，屏幕有点暗，但价格便宜',
        '非常满意，流畅不卡顿，推荐购买',
        '用了一个月，出现死机，很失望'
    ] * 40,
    'score': [5,1,3,5,2]*40,
    'time': pd.date_range('2026-01-01', periods=200, freq='D')
})

# 1. 删除完全重复
df.drop_duplicates(subset=['content'], inplace=True)

# 2. 评分有效性（1-5）
df = df[(df['score']>=1) & (df['score']<=5)]

# 3. 文本清洗
def clean_text(text):
    # 去除HTML标签、特殊字符、数字（保留中文和基本标点）
    text = re.sub(r'<.*?>', '', text)
    text = re.sub(r'[^\u4e00-\u9fa5a-zA-Z，。！？、；：]', ' ', text)
    text = re.sub(r'\s+', ' ', text).strip()
    return text

df['clean_content'] = df['content'].apply(clean_text)

# 4. 中文分词 + 去停用词
stopwords = set(['的', '了', '在', '是', '我', '有', '和', '就', '不', '人', '都', '一', '一个', '上', '也', '很', '到', '说', '要', '去', '你', '会', '着', '没有', '看', '好', '自己', '这', '那', '么', '吧', '啊'])

def cut_words(text):
    words = jieba.lcut(text)
    # 过滤长度<2的词和停用词
    return [w for w in words if len(w)>=2 and w not in stopwords]

df['seg_list'] = df['clean_content'].apply(cut_words)

# 5. 构造情感标签（二分类）：评分>=4为正，<=2为负，3为中性（本文忽略中性）
df['sentiment'] = df['score'].apply(lambda x: 1 if x>=4 else (0 if x<=2 else 2))
# 仅保留正负样本
df_binary = df[df['sentiment'] != 2].copy()
df_binary['sentiment'] = df_binary['sentiment'].astype(int)

print(f"总样本数: {len(df_binary)}, 正: {sum(df_binary['sentiment']==1)}, 负: {sum(df_binary['sentiment']==0)}")

import matplotlib.pyplot as plt
import seaborn as sns
from wordcloud import WordCloud

plt.rcParams['font.sans-serif'] = ['SimHei']  # 中文显示
plt.rcParams['axes.unicode_minus'] = False

fig, axes = plt.subplots(2, 2, figsize=(14, 10))

# 评分分布
sns.histplot(df['score'], bins=5, kde=False, ax=axes[0,0], color='skyblue')
axes[0,0].set_title('评分分布')

# 时间序列（按日平均评分）
df_time = df.resample('D', on='time').agg({'score':'mean', 'content':'count'}).dropna()
axes[0,1].plot(df_time.index, df_time['score'], marker='o', linestyle='-')
axes[0,1].set_title('每日平均评分趋势')
axes[0,1].set_xlabel('日期')
axes[0,1].set_ylabel('平均评分')

# 评论长度分布
df['length'] = df['clean_content'].apply(len)
sns.boxplot(data=df, x='score', y='length', ax=axes[1,0])
axes[1,0].set_title('不同评分下的评论长度')

# 词云（正向 vs 负向）
pos_words = ' '.join([' '.join(seg) for seg in df[df['sentiment']==1]['seg_list']])
neg_words = ' '.join([' '.join(seg) for seg in df[df['sentiment']==0]['seg_list']])
wordcloud_pos = WordCloud(font_path='SimHei.ttf', background_color='white', max_words=100).generate(pos_words)
wordcloud_neg = WordCloud(font_path='SimHei.ttf', background_color='black', max_words=100).generate(neg_words)

axes[1,1].imshow(wordcloud_pos, interpolation='bilinear')
axes[1,1].set_title('正向词云')
axes[1,1].axis('off')

plt.tight_layout()
plt.savefig('eda_plot.png', dpi=300)
plt.show()

from sklearn.feature_extraction.text import TfidfVectorizer

# 将分词列表转为空格分隔的字符串
df['seg_str'] = df['seg_list'].apply(lambda x: ' '.join(x))

tfidf = TfidfVectorizer(max_features=50, ngram_range=(1,2))
X_tfidf = tfidf.fit_transform(df['seg_str'])
feature_names = tfidf.get_feature_names_out()
# 按平均TF-IDF排序
avg_tfidf = X_tfidf.mean(axis=0).A1
top_idx = avg_tfidf.argsort()[-10:][::-1]
print("Top 10 关键词:")
for idx in top_idx:
    print(f"{feature_names[idx]}: {avg_tfidf[idx]:.4f}")

from gensim.models import Word2Vec
import numpy as np

# 训练Word2Vec
sentences = df['seg_list'].tolist()
w2v_model = Word2Vec(sentences, vector_size=100, window=5, min_count=2, workers=4)

def get_doc_vector(seg_list, model, vector_size=100):
    vecs = [model.wv[word] for word in seg_list if word in model.wv]
    if len(vecs) == 0:
        return np.zeros(vector_size)
    return np.mean(vecs, axis=0)

df['w2v_vec'] = df['seg_list'].apply(lambda x: get_doc_vector(x, w2v_model))
X_w2v = np.vstack(df['w2v_vec'].values)
y = df['sentiment'].values  # 注意这里包含中性（2），可过滤

from sklearn.cluster import KMeans
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler

# 只取正负样本进行聚类（也可全部）
X_cluster = X_w2v[df['sentiment'] != 2]
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X_cluster)

# 肘部法则确定k
inertias = []
for k in range(2, 8):
    km = KMeans(n_clusters=k, random_state=42, n_init=10)
    km.fit(X_scaled)
    inertias.append(km.inertia_)

plt.figure()
plt.plot(range(2,8), inertias, marker='o')
plt.xlabel('k')
plt.ylabel('Inertia')
plt.title('肘部法则')
plt.show()

# 选 k=3
kmeans = KMeans(n_clusters=3, random_state=42, n_init=10)
cluster_labels = kmeans.fit_predict(X_scaled)
df_cluster = df[df['sentiment'] != 2].copy()
df_cluster['cluster'] = cluster_labels

# 分析各簇特征
cluster_stats = df_cluster.groupby('cluster').agg({
    'score': 'mean',
    'length': 'mean',
    'sentiment': 'mean'
})
print(cluster_stats)

# 降维可视化
pca = PCA(n_components=2)
X_pca = pca.fit_transform(X_scaled)
plt.figure(figsize=(8,6))
scatter = plt.scatter(X_pca[:,0], X_pca[:,1], c=cluster_labels, cmap='viridis')
plt.colorbar(scatter)
plt.title('用户聚类可视化 (PCA)')
plt.savefig('cluster_pca.png')
plt.show()

from sklearn.linear_model import LogisticRegression
from sklearn.metrics import classification_report, accuracy_score, confusion_matrix

# 只取正负样本
df_binary = df[df['sentiment'] != 2].copy()
y_bin = df_binary['sentiment'].values

# TF-IDF 向量化
tfidf_bin = TfidfVectorizer(max_features=5000, ngram_range=(1,2))
X_tfidf_bin = tfidf_bin.fit_transform(df_binary['seg_str'])

X_train, X_test, y_train, y_test = train_test_split(X_tfidf_bin, y_bin, test_size=0.2, random_state=42)

lr = LogisticRegression(max_iter=1000, C=1.0)
lr.fit(X_train, y_train)
y_pred = lr.predict(X_test)

print("=== 逻辑回归 + TF-IDF ===")
print(classification_report(y_test, y_pred, target_names=['负向','正向']))
print(f"准确率: {accuracy_score(y_test, y_pred):.4f}")

from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Embedding, LSTM, Dense, Dropout
from tensorflow.keras.optimizers import Adam

# 准备序列数据
tokenizer = Tokenizer(num_words=10000, oov_token='<OOV>')
tokenizer.fit_on_texts(df_binary['seg_str'])
sequences = tokenizer.texts_to_sequences(df_binary['seg_str'])
maxlen = 30
X_seq = pad_sequences(sequences, maxlen=maxlen, padding='post', truncating='post')

X_train_seq, X_test_seq, y_train_seq, y_test_seq = train_test_split(
    X_seq, y_bin, test_size=0.2, random_state=42, stratify=y_bin
)

model = Sequential([
    Embedding(10000, 64, input_length=maxlen),
    LSTM(64, return_sequences=True),
    Dropout(0.5),
    LSTM(32),
    Dense(1, activation='sigmoid')
])

model.compile(optimizer=Adam(learning_rate=1e-3), loss='binary_crossentropy', metrics=['accuracy'])
history = model.fit(X_train_seq, y_train_seq, epochs=10, batch_size=32, validation_split=0.1, verbose=1)

loss, acc = model.evaluate(X_test_seq, y_test_seq, verbose=0)
print(f"LSTM 测试准确率: {acc:.4f}")

# 绘制训练曲线
plt.plot(history.history['accuracy'], label='train_acc')
plt.plot(history.history['val_accuracy'], label='val_acc')
plt.legend()
plt.title('LSTM 训练曲线')
plt.show()

coef = lr.coef_[0]
feature_names = tfidf_bin.get_feature_names_out()
# 正向影响最大的词
top_pos_idx = coef.argsort()[-10:][::-1]
print("正向最显著特征:")
for idx in top_pos_idx:
    print(f"{feature_names[idx]}: {coef[idx]:.4f}")
# 负向影响最大的词
top_neg_idx = coef.argsort()[:10]
print("\n负向最显著特征:")
for idx in top_neg_idx:
    print(f"{feature_names[idx]}: {coef[idx]:.4f}")

from sklearn.model_selection import GridSearchCV

param_grid = {'C': [0.1, 1.0, 10.0], 'penalty': ['l2']}
grid = GridSearchCV(LogisticRegression(max_iter=1000), param_grid, cv=3, scoring='accuracy')
grid.fit(X_train, y_train)
print(f"最佳参数: {grid.best_params_}, 最佳CV准确率: {grid.best_score_:.4f}")
best_lr = grid.best_estimator_

import joblib
joblib.dump(lr, 'sentiment_lr.pkl')
joblib.dump(tfidf_bin, 'tfidf_vectorizer.pkl')
# 加载预测
def predict_sentiment(text):
    text_clean = clean_text(text)
    seg = ' '.join(jieba.lcut(text_clean))
    vec = tfidf_bin.transform([seg])
    prob = lr.predict_proba(vec)[0][1]
    return prob