update

Data/database/sql_connector.py

@@ -40,35 +40,55 @@ except Exception as e:
 
 
 def load_preprocessed_projects():
+    try:
+        query = """
+        SELECT *
+        FROM PreProcessed_Projects
+        """
 
-    query = """
-    SELECT *
-    FROM PreProcessed_Projects
-    """
+        df = pd.read_sql(
+            query,
+            engine
+        )
 
-    df = pd.read_sql(
-        query,
-        engine
-    )
+        if "features" in df.columns:
 
-    if "features" in df.columns:
+            def parse_features(x):
 
-        def parse_features(x):
+                if not isinstance(x, str):
+                    return x
 
-            if not isinstance(x, str):
-                return x
-
-            try:
-                x = json.loads(x)
-
-                if isinstance(x, str):
+                try:
                     x = json.loads(x)
 
-                return x
-
-            except Exception:
-                return []
-
-        df["features"] = df["features"].apply(parse_features)
-
-    return df
+                    if isinstance(x, str):
+                        x = json.loads(x)
+
+                    return x
+
+                except Exception:
+                    return []
+
+            df["features"] = df["features"].apply(parse_features)
+
+        return df
+
+    except Exception as e:
+        print(f"Database connection failed, falling back to local metadata file. Error: {e}")
+        import os
+        from pathlib import Path
+        possible_paths = [
+            Path("models/metadata.parquet"),
+            Path(__file__).resolve().parent.parent.parent / "models" / "metadata.parquet",
+            Path("../models/metadata.parquet"),
+        ]
+        for path in possible_paths:
+            if path.exists():
+                print(f"Loading local metadata from {path}")
+                df = pd.read_parquet(path)
+                if "features" in df.columns:
+                    df["features"] = df["features"].apply(
+                        lambda x: x.tolist() if hasattr(x, "tolist") else (list(x) if isinstance(x, (list, tuple, set)) else [])
+                    )
+                return df
+        raise FileNotFoundError(f"Could not connect to database and local metadata.parquet was not found. Looked in: {[str(p) for p in possible_paths]}")

src/similarity_model/llm_feature_extractor.py

@@ -114,17 +114,57 @@ def extract_features_llm(text: str):
 
     prompt = build_feature_prompt(text)
 
-    time.sleep(4)
+    # Short delay to prevent hitting rate limits
+    time.sleep(1)
 
-    response = generate_text(
-        prompt,
-        task="feature",
-        temperature = 0
-    )
+    try:
+        response = generate_text(
+            prompt,
+            task="feature",
+            temperature = 0
+        )
 
-    return parse_features_response(
-        response
-    )
+        return parse_features_response(
+            response
+        )
+    except Exception as e:
+        print(f"Gemini feature extraction failed, falling back to local dictionary-based keyword matching. Error: {e}")
+        try:
+            import pandas as pd
+            import numpy as np
+            from pathlib import Path
+            
+            possible_paths = [
+                Path("models/metadata.parquet"),
+                Path(__file__).resolve().parent.parent.parent / "models" / "metadata.parquet",
+                Path("../models/metadata.parquet"),
+            ]
+            
+            features_dict = set()
+            for path in possible_paths:
+                if path.exists():
+                    df = pd.read_parquet(path)
+                    for f_list in df['features']:
+                        if isinstance(f_list, (list, np.ndarray, set, tuple)):
+                            features_dict.update([str(f).strip().lower() for f in f_list])
+                    break
+            
+            text_lower = text.lower()
+            matched = []
+            
+            # Sort by length descending to match longer keywords first (e.g. 'deep learning' before 'learning')
+            for f in sorted(list(features_dict), key=len, reverse=True):
+                if len(f) > 2:
+                    pattern = r'\b' + re.escape(f) + r'\b'
+                    if re.search(pattern, text_lower):
+                        matched.append(f)
+                        if len(matched) >= 10:
+                            break
+            print(f"Local fallback matched features: {matched}")
+            return matched
+        except Exception as local_err:
+            print(f"Local feature extraction fallback failed: {local_err}")
+            return []
 
 
 def parse_features_response(response: str):