AI数据工程是炼油的“管道系统”，AI量化（AI Quant）就是那台精密的“发动机

import pandas as pd
import numpy as np
from sklearn.model_selection import GroupKFold
from sklearn.preprocessing import StandardScaler
import lightgbm as lgb
from sklearn.metrics import rank_correlation

# --- 1. 模拟数据加载 (实际中可从Wind/聚宽获取) ---
def load_data():
    # 假设数据包含: date, stock_code, 以及一系列技术/财务因子
    # 这里生成随机数据仅作演示结构
    np.random.seed(42)
    dates = pd.date_range('2020-01-01', periods=500, freq='D')
    stocks = [f'000001.SZ', '600000.SH', '300001.SZ']  # 仅3只股票示例
    idx = pd.MultiIndex.from_product([dates, stocks], names=['date', 'stock'])
    df = pd.DataFrame(index=idx).reset_index()
    # 特征: 比如过去5日收益、成交量比等 (实际中需计算)
    df['feature_1'] = np.random.randn(len(df))
    df['feature_2'] = np.random.randn(len(df))
    df['feature_3'] = np.random.randn(len(df))
    # 标签: 未来5日的收益率 (目标变量)
    df['label'] = np.random.randn(len(df)) * 0.01  # 模拟弱信号
    return df

data = load_data()

# --- 2. 特征工程与数据预处理 ---
# 注意: 必须按时间顺序分割, 防止未来信息泄露!
data['date'] = pd.to_datetime(data['date'])
data = data.sort_values(['date', 'stock'])

# 定义特征列和标签列
feature_cols = ['feature_1', 'feature_2', 'feature_3']
X = data[feature_cols].values
y = data['label'].values

# 时间序列分组 (用于交叉验证)
groups = data['date'].astype(str).values

# 标准化: 注意要在训练集上fit, 然后transform验证/测试集
# 这里为了演示清晰, 我们手动按时间分割

# --- 3. 时间序列分割 (避免数据泄露) ---
split_date = '2021-06-01'
train_mask = data['date'] < split_date
val_mask = (data['date'] >= split_date) & (data['date'] < '2021-12-01')
test_mask = data['date'] >= '2021-12-01'

# 标准化处理
scaler = StandardScaler()
X_train = scaler.fit_transform(X[train_mask])
X_val = scaler.transform(X[val_mask])
X_test = scaler.transform(X[test_mask])

y_train = y[train_mask]
y_val = y[val_mask]
y_test = y[test_mask]

# --- 4. 模型训练 (LightGBM) ---
model = lgb.LGBMRegressor(
    n_estimators=100,
    learning_rate=0.1,
    num_leaves=31,
    subsample=0.8,
    colsample_bytree=0.8,
    random_state=42,
    n_jobs=-1
)

model.fit(
    X_train, y_train,
    eval_set=[(X_train, y_train), (X_val, y_val)],
    eval_metric='l2',
    callbacks=[lgb.early_stopping(10), lgb.log_evaluation(0)]
)

# --- 5. 评估指标: 关注Rank IC (秩相关系数) ---
# 在金融中, 我们更关心预测排序与实际排序的相关性
pred_val = model.predict(X_val)
pred_test = model.predict(X_test)

# 计算日度Rank IC (按日期分组计算)
def daily_rank_ic(dates, preds, labels):
    df_tmp = pd.DataFrame({'date': dates, 'pred': preds, 'label': labels})
    ic_list = []
    for d in df_tmp['date'].unique():
        sub = df_tmp[df_tmp['date'] == d]
        if len(sub) > 1:  # 至少2只股票才能算排序
            ic = sub['pred'].corr(sub['label'], method='spearman')
            ic_list.append(ic)
    return np.mean(ic_list)

val_dates = data['date'][val_mask].values
test_dates = data['date'][test_mask].values

print(f"验证集平均Rank IC: {daily_rank_ic(val_dates, pred_val, y_val):.4f}")
print(f"测试集平均Rank IC: {daily_rank_ic(test_dates, pred_test, y_test):.4f}")

# 如果IC为正且稳定(比如>0.02), 则说明模型有微弱预测能力