本文目录导读：
我将为您详细介绍如何使用Pandas进行数据分组相似度分析,这里提供几个实用案例：
基础准备
import pandas as pd
import numpy as np
from scipy.spatial.distance import pdist, squareform
from scipy.stats import pearsonr
from sklearn.metrics.pairwise import cosine_similarity
import warnings
warnings.filterwarnings('ignore')
# 创建示例数据
np.random.seed(42)
data = pd.DataFrame({
'group': ['A']*4 + ['B']*4 + ['C']*4,
'feature1': np.random.randn(12) * 10,
'feature2': np.random.randn(12) * 10,
'feature3': np.random.randn(12) * 10,
'feature4': np.random.randn(12) * 10
})
print("示例数据预览：")
print(data.head(8))
计算组内相似度
1 欧氏距离相似度
def euclidean_similarity(matrix):
"""将欧氏距离转换为相似度"""
distances = pdist(matrix, metric='euclidean')
distances_matrix = squareform(distances)
# 归一化为相似度（0-1之间）
max_dist = distances_matrix.max()
similarity = 1 - (distances_matrix / max_dist)
return similarity
def group_euclidean_similarity(df, group_col, feature_cols):
"""计算每个组内的欧氏距离相似度"""
results = {}
for group, group_data in df.groupby(group_col):
features = group_data[feature_cols].values
if len(features) > 1:
sim_matrix = euclidean_similarity(features)
avg_similarity = (sim_matrix.sum() - len(features)) / (len(features) * (len(features) - 1))
results[group] = {
'avg_similarity': avg_similarity,
'similarity_matrix': pd.DataFrame(
sim_matrix,
index=group_data.index,
columns=group_data.index
)
}
return results
# 计算组内欧氏距离相似度
euclidean_results = group_euclidean_similarity(data, 'group', ['feature1', 'feature2', 'feature3', 'feature4'])
print("\n各组欧氏距离平均相似度：")
for group, result in euclidean_results.items():
print(f"组 {group}: {result['avg_similarity']:.4f}")
2 余弦相似度
def group_cosine_similarity(df, group_col, feature_cols):
"""计算每个组内的余弦相似度"""
results = {}
for group, group_data in df.groupby(group_col):
features = group_data[feature_cols].values
if len(features) > 1:
cos_sim = cosine_similarity(features)
avg_similarity = (cos_sim.sum() - len(features)) / (len(features) * (len(features) - 1))
results[group] = {
'avg_similarity': avg_similarity,
'similarity_matrix': pd.DataFrame(
cos_sim,
index=group_data.index,
columns=group_data.index
)
}
return results
cosine_results = group_cosine_similarity(data, 'group', ['feature1', 'feature2', 'feature3', 'feature4'])
print("\n各组余弦相似度结果：")
for group, result in cosine_results.items():
print(f"组 {group}: 平均余弦相似度 = {result['avg_similarity']:.4f}")
组间相似度比较
def between_group_similarity(df, group_col, feature_cols):
"""计算不同组之间的平均相似度"""
groups = df.groupby(group_col)
group_means = groups[feature_cols].mean()
# 计算各组均值之间的余弦相似度
similarity_matrix = cosine_similarity(group_means.values)
similarity_df = pd.DataFrame(
similarity_matrix,
index=group_means.index,
columns=group_means.index
)
return similarity_df
between_group_sim = between_group_similarity(data, 'group', ['feature1', 'feature2', 'feature3', 'feature4'])
print("\n组间相似度矩阵（基于均值）：")
print(between_group_sim)
综合相似度分析案例
def comprehensive_similarity_analysis(df, group_col, feature_cols, method='cosine'):
"""综合相似度分析"""
# 1. 组内相似度
within_group_sim = {}
for group, group_data in df.groupby(group_col):
features = group_data[feature_cols].values
if len(features) > 1:
if method == 'cosine':
sim_matrix = cosine_similarity(features)
else: # euclidean
sim_matrix = euclidean_similarity(features)
# 统计信息
upper_tri = sim_matrix[np.triu_indices_from(sim_matrix, k=1)]
within_group_sim[group] = {
'mean': np.mean(upper_tri),
'std': np.std(upper_tri),
'min': np.min(upper_tri),
'max': np.max(upper_tri),
'median': np.median(upper_tri)
}
within_df = pd.DataFrame(within_group_sim).T
within_df.index.name = group_col
# 2. 组间相似度
group_means = df.groupby(group_col)[feature_cols].mean()
if method == 'cosine':
between_sim = cosine_similarity(group_means.values)
else:
between_sim = euclidean_similarity(group_means.values)
between_df = pd.DataFrame(
between_sim,
index=group_means.index,
columns=group_means.index
)
return {
'within_group': within_df,
'between_group': between_df,
'group_means': group_means
}
# 运行综合分析
results = comprehensive_similarity_analysis(data, 'group',
['feature1', 'feature2', 'feature3', 'feature4'],
method='cosine')
print("\n=== 组内相似度统计 ===")
print(results['within_group'])
print("\n=== 组间相似度矩阵 ===")
print(results['between_group'])
print("\n=== 各组特征均值 ===")
print(results['group_means'])
可视化相似度结果
import matplotlib.pyplot as plt
import seaborn as sns
def plot_similarity_analysis(results):
"""可视化相似度分析结果"""
fig, axes = plt.subplots(1, 3, figsize=(15, 5))
# 1. 组内相似度分布
within_data = results['within_group']
axes[0].bar(within_data.index, within_data['mean'], yerr=within_data['std'])
axes[0].set_title('组内平均相似度')
axes[0].set_xlabel('Group')
axes[0].set_ylabel('相似度')
# 2. 组间相似度热力图
sns.heatmap(results['between_group'], annot=True, fmt='.3f',
cmap='YlOrRd', ax=axes[1])
axes[1].set_title('组间相似度热力图')
# 3. 特征均值雷达图（简化版）
means = results['group_means'].T
for i, group in enumerate(means.columns):
axes[2].plot(means.index, means[group], marker='o', label=group)
axes[2].set_title('各组特征均值比较')
axes[2].set_xlabel('Features')
axes[2].set_ylabel('Mean Value')
axes[2].legend()
plt.tight_layout()
plt.show()
plot_similarity_analysis(results)
实际应用案例：客户分组相似度
# 创建客户数据示例
customers = pd.DataFrame({
'customer_id': range(1, 13),
'segment': ['高端']*4 + ['中端']*4 + ['低端']*4,
'annual_spending': np.random.randint(1000, 50000, 12),
'purchase_frequency': np.random.randint(1, 20, 12),
'avg_order_value': np.random.randint(50, 500, 12),
'tenure_months': np.random.randint(1, 36, 12)
})
print("客户数据预览：")
print(customers)
# 标准化处理
from sklearn.preprocessing import StandardScaler
features = ['annual_spending', 'purchase_frequency', 'avg_order_value', 'tenure_months']
scaler = StandardScaler()
customers_scaled = customers.copy()
customers_scaled[features] = scaler.fit_transform(customers[features])
# 分析各客户分组的相似度
segments_analysis = comprehensive_similarity_analysis(
customers_scaled, 'segment', features, method='cosine'
)
print("\n不同客户分组的相似度分析：")
print("="*50)
print(f"\n组内相似度统计：")
print(segments_analysis['within_group'])
print(f"\n不同分组间的相似度：")
print(segments_analysis['between_group'])
# 找出最相似和最不相似的客户对
def find_most_similar_customers(df, feature_cols, top_n=3):
"""找出最相似的客户对"""
features = df[feature_cols].values
sim_matrix = cosine_similarity(features)
np.fill_diagonal(sim_matrix, 0) # 忽略自身
similar_pairs = []
for i in range(len(sim_matrix)):
for j in range(i+1, len(sim_matrix)):
similar_pairs.append({
'customer1': df.iloc[i]['customer_id'],
'customer2': df.iloc[j]['customer_id'],
'similarity': sim_matrix[i][j],
'segment1': df.iloc[i]['segment'],
'segment2': df.iloc[j]['segment']
})
similar_df = pd.DataFrame(similar_pairs)
return similar_df.sort_values('similarity', ascending=False).head(top_n)
top_similar = find_most_similar_customers(customers_scaled, features)
print("\n最相似的前3对客户：")
print(top_similar)
进阶：时间序列分组相似度
# 创建时间序列数据
dates = pd.date_range('2024-01-01', periods=30, freq='D')
time_series_data = []
for group in ['A', 'B', 'C']:
for date in dates:
time_series_data.append({
'date': date,
'group': group,
'value': np.random.randn() * 10 + 50 + (ord(group) - ord('A')) * 5
})
ts_df = pd.DataFrame(time_series_data)
ts_df.set_index('date', inplace=True)
def time_series_group_similarity(df, group_col, value_col, window=7):
"""计算时间序列分组相似度"""
# 计算滚动窗口内的统计量
results = {}
for group in df[group_col].unique():
group_data = df[df[group_col] == group][value_col]
results[group] = {
'rolling_mean': group_data.rolling(window=window).mean(),
'rolling_std': group_data.rolling(window=window).std()
}
# 计算皮尔逊相关系数
mean_series = pd.DataFrame({
group: results[group]['rolling_mean']
for group in results.keys()
}).dropna()
corr_matrix = mean_series.corr(method='pearson')
return corr_matrix, mean_series
# 运行时间序列分析
time_corr, time_means = time_series_group_similarity(ts_df, 'group', 'value')
print("\n时间序列分组相关性：")
print(time_corr)
# 可视化时间序列相似度
plt.figure(figsize=(12, 5))
plt.plot(time_means.index, time_means.values)'各组的滚动均值趋势')
plt.xlabel('Date')
plt.ylabel('Rolling Mean')
plt.legend(time_means.columns)
plt.xticks(rotation=45)
plt.show()
这些案例展示了Pandas在数据分组相似度分析中的主要应用：
- 多种相似度度量：欧氏距离、余弦相似度等
- 组内与组间分析：分析组内一致性和组间差异性
- 标准化处理：确保不同量纲的特征可比
- 可视化：通过图表直观展示相似度结果
- 实际应用：客户分群、时间序列分析等场景
根据具体需求,您可以选择合适的相似度度量方法和分析策略。
