""" 关联分析：将repo-level指标与repos_searched元信息join 生成关联分析图和分组对比图 """ import pandas as pd import numpy as np from pathlib import Path import json import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt import matplotlib.font_manager as fm import seaborn as sns import time # Nature风格设置 - 使用字体回退机制（与visualization.py保持一致） font_families_to_try = ['Arial', 'DejaVu Sans', 'Liberation Sans', 'sans-serif'] available_fonts = [f.name for f in fm.fontManager.ttflist] font_found = None for font_family in font_families_to_try: font_lower = font_family.lower() if any(f.lower() == font_lower for f in available_fonts): font_found = font_family break if font_found is None: font_found = 'sans-serif' plt.rcParams['font.family'] = font_found plt.rcParams['font.size'] = 20 plt.rcParams['axes.labelsize'] = 28 # Increased from 18 plt.rcParams['axes.titlesize'] = 28 # Increased from 20 plt.rcParams['xtick.labelsize'] = 24 # Increased from 15 plt.rcParams['ytick.labelsize'] = 24 # Increased from 15 plt.rcParams['legend.fontsize'] = 20 # Increased from 16 plt.rcParams['figure.titlesize'] = 32 # Increased from 22 plt.rcParams['axes.linewidth'] = 1.5 plt.rcParams['axes.spines.top'] = False plt.rcParams['axes.spines.right'] = False plt.rcParams['axes.grid'] = True plt.rcParams['grid.alpha'] = 0.3 plt.rcParams['grid.linewidth'] = 0.5 # Nature配色 NATURE_COLORS = { 'primary': '#2E5090', 'secondary': '#1A5490', 'accent': '#4A90E2', 'success': '#2E7D32', 'warning': '#F57C00', 'error': '#C62828', } def apply_nature_style(ax): """应用Nature风格""" ax.spines['top'].set_visible(False) ax.spines['right'].set_visible(False) ax.spines['left'].set_linewidth(1.5) ax.spines['bottom'].set_linewidth(1.5) ax.grid(True, alpha=0.3, linestyle='--', linewidth=0.5) ax.tick_params(width=1.5, length=5) class JoinInsights: def __init__(self, repos_searched_csv, repo_level_csv, check_history_csv, output_dir): self.repos_searched_csv = repos_searched_csv self.repo_level_csv = repo_level_csv self.check_history_csv = check_history_csv self.output_dir = Path(output_dir) self.output_dir.mkdir(parents=True, exist_ok=True) self.df_joined = None def load_and_join(self): """加载数据并join""" print("Loading data...") # 读取repo-level统计 df_repo = pd.read_csv(self.repo_level_csv) df_repo['full_name'] = df_repo['full_name'].fillna( df_repo['repo_name'].str.replace('___', '/') ) # 读取repos_searched（只读取需要的列以节省内存） print("Loading repos_searched.csv...") df_searched = pd.read_csv( self.repos_searched_csv, usecols=['full_name', 'keyword', 'stars', 'forks', 'open_issues', 'created_at', 'pushed_at', 'language', 'license', 'archived'], dtype={'stars': 'float64', 'forks': 'float64', 'open_issues': 'float64'} ) # 读取check_history（获取is_relevant） print("Loading repos_check_history.csv...") df_history = pd.read_csv( self.check_history_csv, usecols=['full_name', 'keyword', 'is_relevant'] ) # Join: 先join check_history获取is_relevant，再join searched获取元信息 print("Joining data...") df_joined = df_repo.merge(df_history, on='full_name', how='left') df_joined = df_joined.merge(df_searched, on='full_name', how='left', suffixes=('', '_searched')) # 处理重复列 if 'keyword_searched' in df_joined.columns: df_joined['keyword'] = df_joined['keyword'].fillna(df_joined['keyword_searched']) if 'language_searched' in df_joined.columns: df_joined['language_searched'] = df_joined['language_searched'].fillna(df_joined.get('primary_language', '')) # 清理 df_joined = df_joined.dropna(subset=['full_name']) self.df_joined = df_joined print(f"Joined data: {len(df_joined)} rows") # 保存join后的数据 df_joined.to_csv(self.output_dir / 'joined_data.csv', index=False) print(f"Saved joined data to {self.output_dir / 'joined_data.csv'}") def analyze_correlations(self): """分析关联性""" if self.df_joined is None: self.load_and_join() df = self.df_joined.copy() # 数值列相关性分析 numeric_cols = ['stars', 'forks', 'open_issues', 'total_code_lines', 'total_tokens', 'total_functions', 'total_files', 'comment_ratio', 'language_entropy'] numeric_cols = [c for c in numeric_cols if c in df.columns] df_numeric = df[numeric_cols].dropna() if len(df_numeric) > 0: corr_matrix = df_numeric.corr() # 保存相关性矩阵 corr_matrix.to_csv(self.output_dir / 'correlation_matrix.csv') # 重点相关性 insights = {} if 'stars' in df_numeric.columns and 'total_code_lines' in df_numeric.columns: corr = df_numeric['stars'].corr(df_numeric['total_code_lines']) insights['stars_vs_loc'] = float(corr) if 'stars' in df_numeric.columns and 'total_functions' in df_numeric.columns: corr = df_numeric['stars'].corr(df_numeric['total_functions']) insights['stars_vs_functions'] = float(corr) if 'stars' in df_numeric.columns and 'comment_ratio' in df_numeric.columns: corr = df_numeric['stars'].corr(df_numeric['comment_ratio']) insights['stars_vs_comment_ratio'] = float(corr) with open(self.output_dir / 'correlation_insights.json', 'w', encoding='utf-8') as f: json.dump(insights, f, indent=2) print(f"Correlation insights saved") def plot_stars_vs_metrics(self): """绘制stars与多个指标的关系""" if self.df_joined is None: self.load_and_join() df = self.df_joined.copy() df = df[df['stars'].notna() & (df['stars'] > 0)] if len(df) == 0: print("No data for stars vs metrics plot") return fig, axes = plt.subplots(2, 2, figsize=(19.2, 10.8)) colors_list = [NATURE_COLORS['primary'], NATURE_COLORS['accent'], NATURE_COLORS['success'], NATURE_COLORS['secondary']] # 1. stars vs total_code_lines ax = axes[0, 0] apply_nature_style(ax) df_plot = df[df['total_code_lines'] > 0] if len(df_plot) > 0: ax.scatter(df_plot['total_code_lines'], df_plot['stars'], alpha=0.4, s=30, color=colors_list[0], edgecolors='white', linewidth=0.5) ax.set_xscale('log') ax.set_yscale('log') ax.set_xlabel('Lines of Code (LOC, log scale)', fontsize=28, fontweight='bold') ax.set_ylabel('Stars (log scale)', fontsize=28, fontweight='bold') ax.set_title('Stars vs Lines of Code', fontsize=28, fontweight='bold') corr = np.corrcoef(np.log10(df_plot['total_code_lines']), np.log10(df_plot['stars']))[0, 1] ax.text(0.05, 0.95, f'r = {corr:.3f}', transform=ax.transAxes, fontsize=24, fontweight='bold', verticalalignment='top', bbox=dict(boxstyle='round', facecolor='white', alpha=0.8, edgecolor=NATURE_COLORS['primary'], linewidth=2)) # 2. stars vs total_functions ax = axes[0, 1] apply_nature_style(ax) df_plot = df[df['total_functions'] > 0] if len(df_plot) > 0: ax.scatter(df_plot['total_functions'], df_plot['stars'], alpha=0.4, s=30, color=colors_list[1], edgecolors='white', linewidth=0.5) ax.set_xscale('log') ax.set_yscale('log') ax.set_xlabel('Number of Functions (log scale)', fontsize=28, fontweight='bold') ax.set_ylabel('Stars (log scale)', fontsize=28, fontweight='bold') ax.set_title('Stars vs Number of Functions', fontsize=28, fontweight='bold') corr = np.corrcoef(np.log10(df_plot['total_functions']), np.log10(df_plot['stars']))[0, 1] ax.text(0.05, 0.95, f'r = {corr:.3f}', transform=ax.transAxes, fontsize=18, fontweight='bold', verticalalignment='top', bbox=dict(boxstyle='round', facecolor='white', alpha=0.8, edgecolor=NATURE_COLORS['accent'], linewidth=2)) # 3. stars vs comment_ratio ax = axes[1, 0] apply_nature_style(ax) df_plot = df[df['comment_ratio'].notna() & (df['comment_ratio'] >= 0)] if len(df_plot) > 0: ax.scatter(df_plot['comment_ratio'], df_plot['stars'], alpha=0.4, s=30, color=colors_list[2], edgecolors='white', linewidth=0.5) ax.set_yscale('log') ax.set_xlabel('Comment Ratio', fontsize=28, fontweight='bold') ax.set_ylabel('Stars (log scale)', fontsize=28, fontweight='bold') ax.set_title('Stars vs Comment Ratio', fontsize=28, fontweight='bold') corr = df_plot['comment_ratio'].corr(np.log10(df_plot['stars'])) ax.text(0.05, 0.95, f'r = {corr:.3f}', transform=ax.transAxes, fontsize=18, fontweight='bold', verticalalignment='top', bbox=dict(boxstyle='round', facecolor='white', alpha=0.8, edgecolor=NATURE_COLORS['success'], linewidth=2)) # 4. stars vs language_entropy ax = axes[1, 1] apply_nature_style(ax) df_plot = df[df['language_entropy'].notna() & (df['language_entropy'] >= 0)] if len(df_plot) > 0: ax.scatter(df_plot['language_entropy'], df_plot['stars'], alpha=0.4, s=30, color=colors_list[3], edgecolors='white', linewidth=0.5) ax.set_yscale('log') ax.set_xlabel('Language Diversity (Entropy)', fontsize=28, fontweight='bold') ax.set_ylabel('Stars (log scale)', fontsize=28, fontweight='bold') ax.set_title('Stars vs Language Diversity', fontsize=28, fontweight='bold') corr = df_plot['language_entropy'].corr(np.log10(df_plot['stars'])) ax.text(0.05, 0.95, f'r = {corr:.3f}', transform=ax.transAxes, fontsize=18, fontweight='bold', verticalalignment='top', bbox=dict(boxstyle='round', facecolor='white', alpha=0.8, edgecolor=NATURE_COLORS['secondary'], linewidth=2)) plt.suptitle('Correlation Analysis: Stars vs Code Metrics (Top 15K Repositories)', fontsize=32, fontweight='bold', y=0.995) plt.tight_layout(rect=[0, 0, 1, 0.96]) fig_path = self.output_dir / 'fig_insights_stars_vs_metrics.png' plt.savefig(fig_path, dpi=150, bbox_inches='tight', facecolor='white') plt.close() print(f"Saved: {fig_path}") def plot_by_keyword_comparison(self): """按keyword分组对比代码指标""" if self.df_joined is None: self.load_and_join() df = self.df_joined.copy() df = df[df['keyword'].notna()] # Top keywords (increased to 15 for better comparison) top_keywords = df['keyword'].value_counts().head(15).index df = df[df['keyword'].isin(top_keywords)] if len(df) == 0: print("No data for keyword comparison") return fig, axes = plt.subplots(2, 2, figsize=(19.2, 10.8)) colors_list = [NATURE_COLORS['primary'], NATURE_COLORS['success'], NATURE_COLORS['warning'], NATURE_COLORS['secondary']] # 1. 平均代码行数 ax = axes[0, 0] apply_nature_style(ax) stats = df.groupby('keyword')['total_code_lines'].mean().sort_values(ascending=False) stats.plot(kind='bar', ax=ax, color=colors_list[0], alpha=0.85, edgecolor='white', linewidth=1.5) ax.set_title('Average Lines of Code', fontsize=28, fontweight='bold') ax.set_xlabel('') ax.set_ylabel('Average LOC', fontsize=28) ax.tick_params(axis='x', rotation=45, labelsize=24) # Increased font size ax.tick_params(axis='y', labelsize=24) # 2. 平均注释率 ax = axes[0, 1] apply_nature_style(ax) stats = df.groupby('keyword')['comment_ratio'].mean().sort_values(ascending=False) stats.plot(kind='bar', ax=ax, color=colors_list[1], alpha=0.85, edgecolor='white', linewidth=1.5) ax.set_title('Average Comment Ratio', fontsize=28, fontweight='bold') ax.set_xlabel('') ax.set_ylabel('Comment Ratio', fontsize=28) ax.tick_params(axis='x', rotation=45, labelsize=24) # Increased font size ax.tick_params(axis='y', labelsize=24) # 3. 平均stars（如果有） ax = axes[1, 0] apply_nature_style(ax) if 'stars' in df.columns: stats = df.groupby('keyword')['stars'].mean().sort_values(ascending=False) stats.plot(kind='bar', ax=ax, color=colors_list[2], alpha=0.85, edgecolor='white', linewidth=1.5) ax.set_title('Average Stars', fontsize=28, fontweight='bold') ax.set_xlabel('') ax.set_ylabel('Average Stars', fontsize=28) ax.tick_params(axis='x', rotation=45, labelsize=24) # Increased font size ax.tick_params(axis='y', labelsize=24) # 4. 语言多样性 ax = axes[1, 1] apply_nature_style(ax) stats = df.groupby('keyword')['language_entropy'].mean().sort_values(ascending=False) stats.plot(kind='bar', ax=ax, color=colors_list[3], alpha=0.85, edgecolor='white', linewidth=1.5) ax.set_title('Average Language Diversity', fontsize=28, fontweight='bold') ax.set_xlabel('') ax.set_ylabel('Language Entropy', fontsize=28) ax.tick_params(axis='x', rotation=45, labelsize=24) # Increased font size ax.tick_params(axis='y', labelsize=24) plt.suptitle('Code Metrics Comparison by Keyword (Top 15K Repositories)', fontsize=32, fontweight='bold', y=0.995) plt.tight_layout(rect=[0, 0, 1, 0.96]) fig_path = self.output_dir / 'fig_insights_by_keyword.png' plt.savefig(fig_path, dpi=150, bbox_inches='tight', facecolor='white') plt.close() print(f"Saved: {fig_path}") def plot_archived_vs_active(self): """对比archived与active仓库的代码特征""" if self.df_joined is None: self.load_and_join() df = self.df_joined.copy() if 'archived' not in df.columns: print("No archived column in data") return df['is_archived'] = df['archived'].fillna(False) fig, axes = plt.subplots(2, 2, figsize=(19.2, 10.8)) # 1. 代码行数对比 ax = axes[0, 0] apply_nature_style(ax) df_plot = df[df['total_code_lines'] > 0] if len(df_plot) > 0: bp = df_plot.boxplot(column='total_code_lines', by='is_archived', ax=ax, widths=0.6, patch_artist=True, boxprops=dict(facecolor=NATURE_COLORS['primary'], alpha=0.7, linewidth=2), medianprops=dict(color='white', linewidth=3), whiskerprops=dict(linewidth=2), capprops=dict(linewidth=2)) ax.set_title('Lines of Code: Archived vs Active', fontsize=28, fontweight='bold') ax.set_xlabel('') ax.set_ylabel('Lines of Code', fontsize=28) ax.set_yscale('log') ax.set_xticklabels(['Active', 'Archived'], fontsize=24) plt.setp(ax.xaxis.get_majorticklabels(), rotation=0) # 2. 注释率对比 ax = axes[0, 1] apply_nature_style(ax) df_plot = df[df['comment_ratio'].notna()] if len(df_plot) > 0: bp = df_plot.boxplot(column='comment_ratio', by='is_archived', ax=ax, widths=0.6, patch_artist=True, boxprops=dict(facecolor=NATURE_COLORS['success'], alpha=0.7, linewidth=2), medianprops=dict(color='white', linewidth=3), whiskerprops=dict(linewidth=2), capprops=dict(linewidth=2)) ax.set_title('Comment Ratio: Archived vs Active', fontsize=28, fontweight='bold') ax.set_xlabel('') ax.set_ylabel('Comment Ratio', fontsize=28) ax.set_xticklabels(['Active', 'Archived'], fontsize=24) plt.setp(ax.xaxis.get_majorticklabels(), rotation=0) # 3. 函数数对比 ax = axes[1, 0] apply_nature_style(ax) df_plot = df[df['total_functions'] > 0] if len(df_plot) > 0: bp = df_plot.boxplot(column='total_functions', by='is_archived', ax=ax, widths=0.6, patch_artist=True, boxprops=dict(facecolor=NATURE_COLORS['accent'], alpha=0.7, linewidth=2), medianprops=dict(color='white', linewidth=3), whiskerprops=dict(linewidth=2), capprops=dict(linewidth=2)) ax.set_title('Number of Functions: Archived vs Active', fontsize=28, fontweight='bold') ax.set_xlabel('') ax.set_ylabel('Number of Functions', fontsize=28) ax.set_yscale('log') ax.set_xticklabels(['Active', 'Archived'], fontsize=24) plt.setp(ax.xaxis.get_majorticklabels(), rotation=0) # 4. 文件数对比 ax = axes[1, 1] apply_nature_style(ax) df_plot = df[df['total_files'] > 0] if len(df_plot) > 0: bp = df_plot.boxplot(column='total_files', by='is_archived', ax=ax, widths=0.6, patch_artist=True, boxprops=dict(facecolor=NATURE_COLORS['secondary'], alpha=0.7, linewidth=2), medianprops=dict(color='white', linewidth=3), whiskerprops=dict(linewidth=2), capprops=dict(linewidth=2)) ax.set_title('Number of Files: Archived vs Active', fontsize=28, fontweight='bold') ax.set_xlabel('') ax.set_ylabel('Number of Files', fontsize=28) ax.set_yscale('log') ax.set_xticklabels(['Active', 'Archived'], fontsize=24) plt.setp(ax.xaxis.get_majorticklabels(), rotation=0) plt.suptitle('Code Characteristics Comparison: Archived vs Active (Top 15K Repositories)', fontsize=32, fontweight='bold', y=0.995) plt.tight_layout(rect=[0, 0, 1, 0.96]) fig_path = self.output_dir / 'fig_insights_archived_vs_active.png' plt.savefig(fig_path, dpi=150, bbox_inches='tight', facecolor='white') plt.close() print(f"Saved: {fig_path}") def run(self): """执行完整分析""" print("=" * 80) print("关联分析与洞察") print("=" * 80) self.load_and_join() self.analyze_correlations() self.plot_stars_vs_metrics() self.plot_by_keyword_comparison() self.plot_archived_vs_active() print(f"\n关联分析完成！结果保存在: {self.output_dir}") if __name__ == "__main__": repos_searched_csv = "/home/weifengsun/tangou1/domain_code/src/workdir/repos_searched.csv" repo_level_csv = "/home/weifengsun/tangou1/domain_code/src/workdir/reporting/code_stats/repo_level_metrics_top15000.csv" check_history_csv = "/home/weifengsun/tangou1/domain_code/src/workdir/repos_check_history.csv" output_dir = "/home/weifengsun/tangou1/domain_code/src/workdir/reporting/insights" insights = JoinInsights(repos_searched_csv, repo_level_csv, check_history_csv, output_dir) insights.run()