File size: 1,165 Bytes
88da18c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 | import pandas as pd
import numpy as np
import re
from utils.text_processing import clean_text
def check_data_quality(df):
# Check for missing values
missing_values = df.isnull().sum()
# Check text length
df['text_length'] = df['text'].apply(len)
min_length = df['text_length'].min()
max_length = df['text_length'].max()
avg_length = df['text_length'].mean()
# Check special characters
def count_special_chars(text):
return len(re.findall(r'[^\w\s]', text))
df['special_chars'] = df['text'].apply(count_special_chars)
avg_special_chars = df['special_chars'].mean()
return {
"missing_values": missing_values.to_dict(),
"text_length_stats": {
"min": min_length,
"max": max_length,
"average": avg_length
},
"avg_special_chars": avg_special_chars
}
def clean_dataset(df):
# Remove rows with missing text
df = df.dropna(subset=['text'])
# Clean text
df['cleaned_text'] = df['text'].apply(clean_text)
# Remove duplicates
df = df.drop_duplicates(subset=['cleaned_text'])
return df |