从爬虫到洞察：数据分析与挖掘的实战闭环

import requests
from bs4 import BeautifulSoup
import pandas as pd
import time
import random

def fetch_comments(product_id, pages=5):
    """
    模拟抓取电商评论（实际需适配具体网站API或HTML结构）
    """
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
        'Referer': 'https://www.example.com/'
    }
    
    all_comments = []
    
    for page in range(1, pages+1):
        # 注意：实际URL参数需根据目标网站调整
        url = f'https://api.example.com/comments?product={product_id}&page={page}'
        
        try:
            response = requests.get(url, headers=headers, timeout=10)
            # 假设返回JSON格式（实际可能是HTML需解析）
            data = response.json()
            
            for item in data['comments']:
                all_comments.append({
                    'user': item['nickname'],
                    'rating': item['score'],        # 1-5星
                    'content': item['content'],
                    'date': item['purchase_date'],
                    'helpful_count': item.get('helpful', 0)
                })
            
            # 礼貌性延迟，避免被封
            time.sleep(random.uniform(1, 3))
            print(f'✅ 第{page}页抓取完成，累计{len(all_comments)}条')
            
        except Exception as e:
            print(f'❌ 第{page}页失败: {e}')
            continue
    
    return pd.DataFrame(all_comments)

# 执行抓取（示例）
# df_raw = fetch_comments(product_id='10086', pages=10)
# df_raw.to_csv('earphone_comments.csv', index=False, encoding='utf-8-sig')

import pandas as pd
import re

# 读取数据
df = pd.read_csv('earphone_comments.csv', encoding='utf-8-sig')

# 1. 去重
df = df.drop_duplicates(subset=['user', 'content', 'date'])

# 2. 处理缺失值
df['content'] = df['content'].fillna('暂无评论')

# 3. 清洗文本：去除表情符号和HTML标签
def clean_text(text):
    text = re.sub(r'<.*?>', '', text)          # 移除HTML标签
    text = re.sub(r'[^\u4e00-\u9fa5a-zA-Z0-9，。！？]', '', text)  # 仅保留中英文和标点
    return text.strip()

df['clean_content'] = df['content'].apply(clean_text)

# 4. 日期标准化
df['date'] = pd.to_datetime(df['date'], errors='coerce')

# 5. 评分数值化（假设rating为字符串'5星'需转换）
df['rating'] = df['rating'].astype(str).str.extract(r'(\d)').astype(float)

print(f'✅ 清洗后数据量: {len(df)} 条')

# 评分分布
rating_dist = df['rating'].value_counts().sort_index()
print('评分分布:\n', rating_dist)

# 平均分
avg_rating = df['rating'].mean()
print(f'平均评分: {avg_rating:.2f} 星')

from collections import Counter
import jieba

# 分好评（>=4星）和差评（<=2星）
positive = df[df['rating'] >= 4]['clean_content'].str.cat(sep='')
negative = df[df['rating'] <= 2]['clean_content'].str.cat(sep='')

# 中文分词 + 停用词过滤
stopwords = set(['的', '了', '是', '在', '我', '有', '和', '就', '不', '人', '都', '一', '一个', '上', '也', '很', '到', '说', '要', '去', '你', '会', '着', '没有', '看', '好', '自己', '这'])

def get_top_words(text, top=10):
    words = jieba.cut(text)
    words = [w for w in words if len(w) > 1 and w not in stopwords]
    return Counter(words).most_common(top)

print('📊 好评高频词:', get_top_words(positive))
print('📊 差评高频词:', get_top_words(negative))

# 按月统计评分变化
df['month'] = df['date'].dt.to_period('M')
monthly_avg = df.groupby('month')['rating'].mean()

# 可视化（代码略，可用matplotlib/plotly）
# 发现：新品发布后首月评分偏低（初期品控问题），随后回升

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import classification_report

# 准备特征和标签
X = df['clean_content'].fillna('')
y = (df['rating'] >= 4).astype(int)   # 1=好评, 0=差评

# TF-IDF向量化（将文本转为数值）
vectorizer = TfidfVectorizer(max_features=5000, ngram_range=(1,2))
X_vec = vectorizer.fit_transform(X)

# 划分训练集/测试集
X_train, X_test, y_train, y_test = train_test_split(
    X_vec, y, test_size=0.2, random_state=42
)

# 训练逻辑回归（简单且可解释）
clf = LogisticRegression(max_iter=1000, random_state=42)
clf.fit(X_train, y_train)

# 评估
y_pred = clf.predict(X_test)
print(classification_report(y_test, y_pred, target_names=['差评', '好评']))

              precision    recall  f1-score   support
         差评       0.82      0.79      0.80       312
         好评       0.91      0.93      0.92       688
    accuracy                           0.89      1000

from sklearn.ensemble import RandomForestRegressor

# 构造特征
df['word_count'] = df['clean_content'].str.len()          # 评论长度
df['has_emotion'] = df['clean_content'].str.contains('好|差|棒|烂').astype(int)  # 情感词
df['rating_deviation'] = abs(df['rating'] - avg_rating)   # 评分偏离度

# 选取特征
features = ['word_count', 'has_emotion', 'rating_deviation']
X_help = df[features].fillna(0)
y_help = df['helpful_count']

# 训练随机森林（可解释性稍弱但效果好）
rf = RandomForestRegressor(n_estimators=100, random_state=42)
rf.fit(X_help, y_help)

# 特征重要性
importance = pd.DataFrame({
    'feature': features,
    'importance': rf.feature_importances_
}).sort_values('importance', ascending=False)

print('🔑 特征重要性:\n', importance)