从爬虫到挖掘：实战招聘数据获取与技能趋势分析

https://search.51job.com/list/010000,000000,0000,00,9,99,数据分析,2,{page}.html

import requests
from bs4 import BeautifulSoup
import time
import random
import pandas as pd
import re

HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
    'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',
    'Connection': 'keep-alive',
}

def parse_list_page(url):
    resp = requests.get(url, headers=HEADERS, timeout=10)
    if resp.status_code != 200:
        print(f"请求失败: {url}")
        return []
    soup = BeautifulSoup(resp.text, 'lxml')
    items = soup.select('div.el')
    results = []
    for item in items:
        try:
            title = item.select_one('div.job_title a')
            title = title.get_text(strip=True) if title else ''
            company = item.select_one('span.comp_name a')
            company = company.get_text(strip=True) if company else ''
            salary = item.select_one('span.salary')
            salary = salary.get_text(strip=True) if salary else ''
            location = item.select_one('span.location')
            location = location.get_text(strip=True) if location else ''
            time_span = item.select_one('span.time')
            publish_time = time_span.get_text(strip=True) if time_span else ''
            results.append({
                'title': title,
                'company': company,
                'salary': salary,
                'location': location,
                'publish_time': publish_time,
            })
        except Exception as e:
            print(f"解析单条出错: {e}")
            continue
    return results

def crawl_all_pages(keyword='数据分析', city='010000', pages=10):
    base_url = f"https://search.51job.com/list/{city},000000,0000,00,9,99,{keyword},2,{{}}.html"
    all_data = []
    for page in range(1, pages+1):
        url = base_url.format(page)
        print(f"正在爬取第 {page} 页...")
        data = parse_list_page(url)
        all_data.extend(data)
        time.sleep(random.uniform(1, 3))
    return pd.DataFrame(all_data)

# 运行爬虫（注意：实际运行时请先测试单页，避免被封）
df_raw = crawl_all_pages(pages=10)
print(f"共爬取 {len(df_raw)} 条记录")
df_raw.head()

def parse_salary(salary_str):
    if not salary_str or '面议' in salary_str:
        return None, None
    # 去除多余空格
    salary_str = salary_str.replace(' ', '')
    # 提取数字和单位
    pattern = r'([\d.]+)-([\d.]+)(万|千)?/?(月|年)?'
    match = re.search(pattern, salary_str)
    if not match:
        return None, None
    low = float(match.group(1))
    high = float(match.group(2))
    unit = match.group(3) if match.group(3) else '千'
    period = match.group(4) if match.group(4) else '月'
    # 转换为月薪千元
    if unit == '万':
        low *= 10
        high *= 10
    # 如果为年薪，除以12
    if period == '年':
        low /= 12
        high /= 12
    return round(low, 1), round(high, 1)

# 应用
df_raw['salary_low'], df_raw['salary_high'] = zip(*df_raw['salary'].apply(parse_salary))
# 取均值作为薪资
df_raw['salary_avg'] = (df_raw['salary_low'] + df_raw['salary_high']) / 2
# 删除无法解析的
df_clean = df_raw.dropna(subset=['salary_avg']).copy()
print(f"清洗后剩余 {len(df_clean)} 条")

def extract_experience(title):
    # 匹配 x-y年 或 x年以上
    patterns = [r'(\d+)-(\d+)年', r'(\d+)年以上', r'(\d+)年经验']
    for p in patterns:
        match = re.search(p, title)
        if match:
            if len(match.groups()) == 2:
                return int(match.group(1))  # 取下限
            else:
                return int(match.group(1))
    return 0  # 默认无经验

def extract_education(title):
    edu_keywords = ['本科', '硕士', '博士', '大专', '高中']
    for edu in edu_keywords:
        if edu in title:
            return edu
    return '不限'

df_clean['experience'] = df_clean['title'].apply(extract_experience)
df_clean['education'] = df_clean['title'].apply(extract_education)

df_clean['city'] = df_clean['location'].apply(lambda x: x.split('-')[0] if x else '')

import matplotlib.pyplot as plt
import seaborn as sns

sns.set_style('whitegrid')
plt.figure(figsize=(10,6))
sns.histplot(df_clean['salary_avg'], bins=30, kde=True)
plt.title('月薪分布（千元）')
plt.xlabel('月薪（千元）')
plt.show()

plt.figure(figsize=(10,6))
sns.boxplot(x='experience', y='salary_avg', data=df_clean)
plt.title('不同经验年限对应的薪资')
plt.xlabel('经验要求（年）')
plt.ylabel('月薪（千元）')
plt.show()

top_companies = df_clean['company'].value_counts().head(10)
plt.figure(figsize=(10,6))
sns.barplot(x=top_companies.values, y=top_companies.index)
plt.title('招聘数据分析岗位最多的公司')
plt.xlabel('职位数量')
plt.show()

import jieba
from wordcloud import WordCloud
import collections

# 添加自定义词典
jieba.add_word('数据分析')
jieba.add_word('数据挖掘')
jieba.add_word('机器学习')
jieba.add_word('深度学习')

text = ' '.join(df_clean['title'].tolist())
words = jieba.cut(text, cut_all=False)
# 过滤单字和无关词
stopwords = set(['的', '与', '或', '等', '有', '在', '和', '及', '之', '者', '、'])
word_list = [w for w in words if len(w) > 1 and w not in stopwords]
word_freq = collections.Counter(word_list)

# 生成词云
wordcloud = WordCloud(font_path='simhei.ttf', background_color='white', 
                      width=800, height=400).generate_from_frequencies(word_freq)
plt.figure(figsize=(12,6))
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis('off')
plt.show()

from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import LabelEncoder, OneHotEncoder
from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline
from sklearn.metrics import mean_absolute_error, r2_score

# 特征：experience, education
X = df_clean[['experience', 'education']]
y = df_clean['salary_avg']

# 划分训练测试
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# 预处理：experience 标准化，education 独热
preprocessor = ColumnTransformer(
    transformers=[
        ('num', 'passthrough', ['experience']),
        ('cat', OneHotEncoder(), ['education'])
    ])

model = Pipeline(steps=[
    ('preprocessor', preprocessor),
    ('regressor', LinearRegression())
])

model.fit(X_train, y_train)
y_pred = model.predict(X_test)

print(f"MAE: {mean_absolute_error(y_test, y_pred):.2f} 千元")
print(f"R2: {r2_score(y_test, y_pred):.3f}")

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.cluster import KMeans

# 使用标题作为文本
corpus = df_clean['title'].tolist()
vectorizer = TfidfVectorizer(max_features=100, stop_words='english')
X_tfidf = vectorizer.fit_transform(corpus)

# 肘部法则确定K
from sklearn.metrics import silhouette_score
scores = []
for k in range(2, 8):
    km = KMeans(n_clusters=k, random_state=42, n_init=10)
    labels = km.fit_predict(X_tfidf)
    scores.append(silhouette_score(X_tfidf, labels))
plt.plot(range(2,8), scores)
plt.xlabel('K')
plt.ylabel('Silhouette Score')
plt.show()
# 选择 K=4
kmeans = KMeans(n_clusters=4, random_state=42, n_init=10)
df_clean['cluster'] = kmeans.fit_predict(X_tfidf)

# 查看每个簇的关键词
for i in range(4):
    cluster_center = kmeans.cluster_centers_[i]
    top_indices = cluster_center.argsort()[-5:][::-1]
    top_words = [vectorizer.get_feature_names_out()[idx] for idx in top_indices]
    print(f"簇{i}: {top_words}")

from gensim import corpora, models
import jieba.posseg as pseg

# 分词并过滤词性（保留名词、动词等）
def tokenize(text):
    words = []
    for word, flag in pseg.cut(text):
        if flag in ['n', 'v', 'a', 'nr', 'ns'] and len(word) > 1:
            words.append(word)
    return words

texts = [tokenize(t) for t in df_clean['title']]
dictionary = corpora.Dictionary(texts)
dictionary.filter_extremes(no_below=5, no_above=0.5)
corpus = [dictionary.doc2bow(text) for text in texts]

lda = models.LdaModel(corpus, id2word=dictionary, num_topics=3, passes=10)
for idx, topic in lda.print_topics(-1):
    print(f"Topic {idx}: {topic}")