本文目录导读：
我来介绍几种使用Pandas进行数据分组洗牌的方法。
方法1：使用groupby和sample
import pandas as pd
import numpy as np
# 创建示例数据
df = pd.DataFrame({
'group': ['A', 'A', 'A', 'B', 'B', 'B', 'C', 'C'],
'value': [1, 2, 3, 4, 5, 6, 7, 8]
})
print("原始数据:")
print(df)
# 方法1：在每个组内随机洗牌
def shuffle_group(group):
return group.sample(frac=1, random_state=42)
shuffled_df = df.groupby('group', group_keys=False).apply(shuffle_group)
print("\n分组内洗牌结果:")
print(shuffled_df)
方法2：使用groupby和sample保持组结构
# 保持组内结构，只打乱组顺序
def shuffle_groups(df, group_col):
"""打乱组的顺序，但保持组内数据不变"""
# 获取唯一组列表
unique_groups = df[group_col].unique()
# 随机打乱组顺序
shuffled_groups = np.random.permutation(unique_groups)
# 创建新的DataFrame
result = pd.concat([df[df[group_col] == g] for g in shuffled_groups], ignore_index=True)
return result
# 示例
df_shuffled_groups = shuffle_groups(df, 'group')
print("组顺序打乱结果:")
print(df_shuffled_groups)
方法3：加权随机洗牌
# 基于权重的组内洗牌
def weighted_shuffle(group, weight_col=None):
"""基于权重进行组内洗牌"""
if weight_col:
weights = group[weight_col]
return group.sample(frac=1, weights=weights)
return group.sample(frac=1)
# 创建带权重的数据
df_weight = pd.DataFrame({
'group': ['A', 'A', 'A', 'B', 'B', 'B'],
'value': [1, 2, 3, 4, 5, 6],
'weight': [0.1, 0.2, 0.7, 0.3, 0.3, 0.4]
})
# 按权重洗牌
weighted_shuffled = df_weight.groupby('group', group_keys=False).apply(
lambda x: weighted_shuffle(x, 'weight')
)
print("\n加权洗牌结果:")
print(weighted_shuffled)
方法4：保持行数平衡的分组洗牌
def balanced_group_shuffle(df, group_col, seed=None):
"""均衡的分组洗牌，确保每个组有相似的样本数"""
if seed is not None:
np.random.seed(seed)
result_dfs = []
groups = df.groupby(group_col)
# 找到最小组的样本数
min_size = groups.size().min()
for name, group in groups:
# 从每个组中随机选择相同数量的样本
sampled = group.sample(n=min_size, random_state=seed)
result_dfs.append(sampled)
return pd.concat(result_dfs, ignore_index=True)
# 创建不均衡的数据
df_imbalanced = pd.DataFrame({
'group': ['A', 'A', 'A', 'A', 'B', 'B', 'C', 'C', 'C'],
'value': [1, 2, 3, 4, 5, 6, 7, 8, 9]
})
balanced = balanced_group_shuffle(df_imbalanced, 'group', seed=42)
print("\n均衡洗牌结果:")
print(balanced)
方法5：交叉验证的分组洗牌
def group_kfold_split(df, group_col, n_folds=5, shuffle=True, random_state=None):
"""分组K折交叉验证的数据分割"""
from sklearn.model_selection import GroupKFold
groups = df[group_col]
gkf = GroupKFold(n_splits=n_folds)
folds = []
for train_idx, test_idx in gkf.split(df, groups=groups):
train_df = df.iloc[train_idx]
test_df = df.iloc[test_idx]
folds.append((train_df, test_df))
return folds
# 使用示例
df_large = pd.DataFrame({
'group': ['A']*10 + ['B']*10 + ['C']*10,
'feature1': np.random.randn(30),
'feature2': np.random.randn(30),
'target': np.random.randint(0, 2, 30)
})
folds = group_kfold_split(df_large, 'group', n_folds=3)
print("\n分组K折交叉验证:")
for i, (train, test) in enumerate(folds):
print(f"Fold {i+1}: 训练集 {len(train)} 样本, 测试集 {len(test)} 样本")
print(f"训练集分组: {train['group'].unique()}")
print(f"测试集分组: {test['group'].unique()}")
实际应用示例
# 销售数据的分组分析
def analyze_sales_shuffle(sales_df):
"""对销售数据进行分组洗牌分析"""
# 按地区分组洗牌
shuffled_by_region = sales_df.groupby('region', group_keys=False).apply(
lambda x: x.sample(frac=1)
)
# 计算洗牌前后的统计
original_stats = sales_df.groupby('region')['sales'].agg(['mean', 'std'])
shuffled_stats = shuffled_by_region.groupby('region')['sales'].agg(['mean', 'std'])
# 比较差异
comparison = pd.DataFrame({
'原始均值': original_stats['mean'],
'洗牌后均值': shuffled_stats['mean'],
'差异': original_stats['mean'] - shuffled_stats['mean']
})
return comparison
# 示例数据
sales_data = pd.DataFrame({
'region': ['North']*5 + ['South']*5 + ['East']*5,
'product': ['A', 'B', 'C', 'D', 'E']*3,
'sales': [100, 150, 200, 120, 180, 90, 130, 160, 110, 140, 80, 120, 170, 95, 155]
})
result = analyze_sales_shuffle(sales_data)
print("\n销售数据分析结果:")
print(result)
这些方法可以根据你的具体需求选择使用：
- 方法1：组内完全随机化
- 方法2：保持组结构，只打乱顺序
- 方法3：带权重的随机化
- 方法4：均衡采样
- 方法5：交叉验证分割
选择哪种方法取决于你的具体应用场景和数据分析需求。
