Create preprocessing.py

modules/preprocessing.py

@@ -0,0 +1,39 @@
+# modules/preprocessing.py
+
+import re
+import nltk
+
+# Download stopwords if not already present
+nltk.download("stopwords", quiet=True)
+from nltk.corpus import stopwords
+
+STOPWORDS = set(stopwords.words("english"))
+
+def clean_text(text: str) -> str:
+    """
+    Basic text cleaning:
+    - Lowercase
+    - Remove URLs, mentions, hashtags
+    - Remove numbers, punctuation, and stopwords
+    """
+    text = text.lower()
+
+    # Remove URLs
+    text = re.sub(r"http\S+|www\S+|https\S+", "", text)
+
+    # Remove mentions and hashtags
+    text = re.sub(r"@\w+|#\w+", "", text)
+
+    # Remove numbers and special characters
+    text = re.sub(r"[^a-z\s]", "", text)
+
+    # Remove stopwords
+    tokens = [word for word in text.split() if word not in STOPWORDS]
+
+    return " ".join(tokens)
+
+def preprocess_texts(texts: list) -> list:
+    """
+    Clean and preprocess a list of texts.
+    """
+    return [clean_text(t) for t in texts if t.strip()]