本文目录导读：
我来介绍几种使用Pandas进行数据分组重标准化的方法。
基础方法：按组标准化
import pandas as pd
import numpy as np
from scipy import stats
# 创建示例数据
df = pd.DataFrame({
'group': ['A', 'A', 'A', 'B', 'B', 'B', 'C', 'C'],
'value': [10, 20, 30, 100, 200, 300, 50, 150]
})
print("原始数据：")
print(df)
Z-score 标准化（均值为0，标准差为1）
# 按组进行Z-score标准化
def zscore_standardize(x):
return (x - x.mean()) / x.std()
df['zscore_standardized'] = df.groupby('group')['value'].transform(zscore_standardize)
print("\nZ-score标准化后：")
print(df)
Min-Max 标准化（缩放到[0,1]区间）
def minmax_standardize(x):
return (x - x.min()) / (x.max() - x.min())
df['minmax_standardized'] = df.groupby('group')['value'].transform(minmax_standardize)
print("\nMin-Max标准化后：")
print(df)
使用apply方法的完整示例
def group_standardize(group_df):
"""对每个组进行多种标准化"""
result = group_df.copy()
# Z-score标准化
result['zscore'] = (group_df['value'] - group_df['value'].mean()) / group_df['value'].std()
# Min-Max标准化
result['minmax'] = (group_df['value'] - group_df['value'].min()) / (group_df['value'].max() - group_df['value'].min())
# 鲁棒标准化（使用中位数和四分位距）
result['robust'] = (group_df['value'] - group_df['value'].median()) / (group_df['value'].quantile(0.75) - group_df['value'].quantile(0.25))
# 均值归一化
result['mean_norm'] = group_df['value'] / group_df['value'].mean()
return result
# 应用标准化
standardized_df = df.groupby('group', group_keys=False).apply(group_standardize)
print("\n多种标准化方法结果：")
print(standardized_df)
实际案例：金融数据分组标准化
# 创建模拟金融数据
np.random.seed(42)
dates = pd.date_range('2023-01-01', periods=20, freq='D')
# 创建多个股票的数据
stocks_data = []
for stock in ['AAPL', 'GOOGL', 'MSFT']:
for date in dates:
stocks_data.append({
'stock': stock,
'date': date,
'price': 100 + np.random.randn() * 20 + (0 if stock == 'AAPL' else 50 if stock == 'GOOGL' else 25),
'volume': np.random.randint(1000000, 5000000)
})
df_stocks = pd.DataFrame(stocks_data)
print("模拟股票数据：")
print(df_stocks.head(10))
# 按股票分组标准化价格和成交量
def normalize_stock_data(group):
group = group.copy()
# 计算滚动均值和标准差（每个股票自己的）
group['price_zscore'] = (group['price'] - group['price'].rolling(5, min_periods=1).mean()) / \
group['price'].rolling(5, min_periods=1).std()
# 成交量标准化
group['volume_normalized'] = (group['volume'] - group['volume'].min()) / \
(group['volume'].max() - group['volume'].min())
return group
# 应用分组标准化
normalized_stocks = df_stocks.groupby('stock', group_keys=False).apply(normalize_stock_data)
print("\n标准化后的股票数据：")
print(normalized_stocks[normalized_stocks['stock'] == 'AAPL'].head())
高级用法：多列同时标准化
# 创建包含多个特征的数据
df_multi = pd.DataFrame({
'group': ['A', 'A', 'A', 'B', 'B', 'B'],
'feature1': [10, 20, 30, 100, 200, 300],
'feature2': [1, 2, 3, 10, 20, 30],
'feature3': [0.1, 0.2, 0.3, 1.0, 2.0, 3.0]
})
def standardize_multiple_features(group_df, features=['feature1', 'feature2', 'feature3']):
"""对多个特征进行分组标准化"""
result = group_df.copy()
for feature in features:
# Z-score标准化
result[f'{feature}_zscore'] = (group_df[feature] - group_df[feature].mean()) / group_df[feature].std()
# Min-Max标准化
result[f'{feature}_minmax'] = (group_df[feature] - group_df[feature].min()) / \
(group_df[feature].max() - group_df[feature].min())
return result
# 应用多特征标准化
standardized_multi = df_multi.groupby('group', group_keys=False).apply(standardize_multiple_features)
print("\n多特征标准化结果：")
print(standardized_multi)
使用transform的高效写法
# 高效的多列分组标准化
def efficient_group_standardize(df, group_col, value_cols):
"""高效的组内标准化方法"""
df_standardized = df.copy()
for col in value_cols:
# 计算分组统计量
group_means = df.groupby(group_col)[col].transform('mean')
group_stds = df.groupby(group_col)[col].transform('std')
group_mins = df.groupby(group_col)[col].transform('min')
group_maxs = df.groupby(group_col)[col].transform('max')
# Z-score标准化
df_standardized[f'{col}_zscore'] = (df[col] - group_means) / group_stds
# Min-Max标准化
df_standardized[f'{col}_minmax'] = (df[col] - group_mins) / (group_maxs - group_mins)
return df_standardized
# 测试高效方法
result_efficient = efficient_group_standardize(df_multi, 'group', ['feature1', 'feature2'])
print("\n高效标准化方法结果：")
print(result_efficient)
处理异常值的高级标准化
def robust_group_standardize(group_df, threshold=3):
"""在分组标准化前处理异常值"""
result = group_df.copy()
# 计算Z-score并标记异常值
z_scores = np.abs(stats.zscore(group_df['value']))
outliers = z_scores > threshold
# 将异常值替换为组中位数
median_val = group_df['value'].median()
result.loc[outliers, 'value'] = median_val
# 然后进行标准化
result['value_standardized'] = (result['value'] - result['value'].mean()) / result['value'].std()
# 标记原始异常值
result['was_outlier'] = outliers
return result
# 应用鲁棒标准化
df_robust = df.groupby('group', group_keys=False).apply(robust_group_standardize)
print("\n处理异常值后的标准化结果：")
print(df_robust)
关键要点
- 使用transform：比apply更高效，避免创建中间对象
- 处理缺失值：标准化前先处理NaN值
- 保持分组属性：确保每个组的标准化基于该组自己的统计量
- 异常值处理：考虑使用鲁棒标准化方法
选择哪种标准化方法取决于你的具体需求和数据特点。
