feat: implement project embedding engine and feature similarity calculation modules

src/similarity_model/embedding_engine.py

@@ -11,7 +11,7 @@ from Data.database.sql_connector import load_preprocessed_projects
 logging.basicConfig(level=logging.INFO, format="%(asctime)s | %(levelname)s | %(message)s")
 logger = logging.getLogger(__name__)
 
-DEFAULT_MODEL = "all-MiniLM-L6-v2"
+DEFAULT_MODEL = "all-mpnet-base-v2"
 TEXT_COL = "clean_text"
 TITLE_COL = "project_title"
 TECH_COL = "technologies"

src/similarity_model/feature_similarity.py

@@ -1,5 +1,6 @@
 from typing import List, Dict, Any
 import pandas as pd
+import numpy as np
 from sentence_transformers import SentenceTransformer
 from scipy.optimize import linear_sum_assignment
 from sklearn.metrics.pairwise import cosine_similarity
@@ -109,6 +110,17 @@ def empty_result(unique_a=None, unique_b=None):
         "unique_b": unique_b or []
     }
 
+@lru_cache(maxsize=10000)
+def encode_single_feature(feature: str) -> np.ndarray:
+    import numpy as np
+    model = load_feature_model()
+    return model.encode(
+        [feature],
+        convert_to_numpy=True,
+        normalize_embeddings=True,
+        show_progress_bar=False
+    )[0].astype("float32")
+
 def encode_features(
     features: List[str],
     model
@@ -117,12 +129,10 @@ def encode_features(
     if not features:
         return np.array([])
 
-    vectors = model.encode(
-        features,
-        convert_to_numpy=True,
-        normalize_embeddings=True
-    )
-    return vectors.astype("float32")
+    embeddings = []
+    for feat in features:
+        embeddings.append(encode_single_feature(feat))
+    return np.array(embeddings)
 
 def compute_feature_similarity(
     features_a,