data_raw/src.py

import pandas as pd
import os
from collections import set

def load_news_data(filepath):
    """Load news.tsv file with proper column names"""
    try:
        df = pd.read_csv(filepath, sep='\t', header=None, 
                        names=['NewsID', 'Category', 'SubCategory', 'Title', 'Abstract', 'URL', 'TitleEntities', 'AbstractEntities'])
        return df
    except Exception as e:
        print(f"Error loading {filepath}: {e}")
        return None

def load_behaviors_data(filepath):
    """Load behaviors.tsv file with proper column names"""
    try:
        df = pd.read_csv(filepath, sep='\t', header=None,
                        names=['ImpressionID', 'UserID', 'Time', 'History', 'Impressions'])
        return df
    except Exception as e:
        print(f"Error loading {filepath}: {e}")
        return None

def calculate_avg_length(text_series, unit='words'):
    """Calculate average length of text in words or characters"""
    if unit == 'words':
        lengths = text_series.dropna().apply(lambda x: len(str(x).split()))
    else:
        lengths = text_series.dropna().apply(lambda x: len(str(x)))
    return lengths.mean()

def print_mind_statistics():
    """Print comprehensive statistics for MIND dataset"""
    
    # Define data paths
    train_news_path = 'MINDsmall_train/news.tsv'
    dev_news_path = 'MINDsmall_dev/news.tsv'
    train_behaviors_path = 'MINDsmall_train/behaviors.tsv'
    dev_behaviors_path = 'MINDsmall_dev/behaviors.tsv'
    
    # Load all data
    print("Loading MIND dataset...")
    train_news = load_news_data(train_news_path)
    dev_news = load_news_data(dev_news_path)
    train_behaviors = load_behaviors_data(train_behaviors_path)
    dev_behaviors = load_behaviors_data(dev_behaviors_path)
    
    # Combine train and dev data for overall statistics
    all_news = pd.concat([train_news, dev_news], ignore_index=True) if train_news is not None and dev_news is not None else None
    all_behaviors = pd.concat([train_behaviors, dev_behaviors], ignore_index=True) if train_behaviors is not None and dev_behaviors is not None else None
    
    if all_news is None or all_behaviors is None:
        print("Error: Could not load dataset properly")
        return
    
    # Calculate statistics
    print("\n" + "="*50)
    print("MIND Dataset Statistics")
    print("="*50)
    
    # Number of unique news articles
    unique_news = all_news['NewsID'].nunique()
    print(f"#News {unique_news}")
    
    # Number of unique categories
    unique_categories = all_news['Category'].nunique()
    print(f"#Category {unique_categories}")
    
    # Number of unique sub-categories
    unique_subcategories = all_news['SubCategory'].nunique()
    print(f"#Sub-Category {unique_subcategories}")
    
    # Number of unique users
    unique_users = all_behaviors['UserID'].nunique()
    print(f"#User {unique_users}")
    
    # Average abstract length (in words)
    avg_abstract_len = calculate_avg_length(all_news['Abstract'], 'words')
    print(f"Avg. Abstract len. {avg_abstract_len:.2f}")
    
    # Average title length (in words)
    avg_title_len = calculate_avg_length(all_news['Title'], 'words')
    print(f"Avg. Title len. {avg_title_len:.2f}")
    
    print("\nBảng 1: Thống kê chi tiết của bộ dữ liệu MINDsmall.")
    
    # Additional detailed statistics
    print("\n" + "="*50)
    print("Detailed Breakdown")
    print("="*50)
    
    print(f"Training set news: {len(train_news) if train_news is not None else 0}")
    print(f"Development set news: {len(dev_news) if dev_news is not None else 0}")
    print(f"Training set users: {train_behaviors['UserID'].nunique() if train_behaviors is not None else 0}")
    print(f"Development set users: {dev_behaviors['UserID'].nunique() if dev_behaviors is not None else 0}")
    
    # Show category distribution
    print(f"\nTop 10 Categories:")
    category_counts = all_news['Category'].value_counts().head(10)
    for category, count in category_counts.items():
        print(f"  {category}: {count}")
    
    # Show sub-category distribution  
    print(f"\nTop 10 Sub-Categories:")
    subcategory_counts = all_news['SubCategory'].value_counts().head(10)
    for subcategory, count in subcategory_counts.items():
        print(f"  {subcategory}: {count}")

if __name__ == "__main__":
    print_mind_statistics()