feat: implement sync worker service for automated FAISS index rebuilding and project preprocessing

src/similarity_model/embedding_engine.py

@@ -96,7 +96,7 @@ def train_embedding_engine():
     
     engine = ProjectEmbedder()
     engine.build_index(df)
-    engine.save_artifacts()
+    engine.save_artifacts(str(MODEL_DIR))
     
     logger.info("Embedding engine completed successfully.")
     return engine

src/similarity_model/sync_worker.py

@@ -40,18 +40,18 @@ class RebuildManager:
         self.accumulated_changes += 1
         self.pending_rebuild = True
 
-    def check_and_rebuild(self):
+    def check_and_rebuild(self, force=False):
         if not self.pending_rebuild:
             return False
 
         now = time.time()
         time_elapsed = now - self.last_rebuild_time
         
-        # Trigger rebuild if we hit the change threshold, OR if the cooldown has passed
-        if self.accumulated_changes >= self.rebuild_threshold or time_elapsed >= self.rebuild_cooldown:
+        # Trigger rebuild if we hit the change threshold, OR if the cooldown has passed, OR if forced
+        if force or self.accumulated_changes >= self.rebuild_threshold or time_elapsed >= self.rebuild_cooldown:
             logger.info(
                 f"Triggering FAISS index rebuild. "
-                f"Accumulated changes: {self.accumulated_changes}, time elapsed: {time_elapsed:.1f}s"
+                f"Accumulated changes: {self.accumulated_changes}, time elapsed: {time_elapsed:.1f}s, force: {force}"
             )
             try:
                 train_embedding_engine()
@@ -199,8 +199,8 @@ def run_worker():
                 batch = result.mappings().all()
             
             if not batch:
-                # Idle period: check if we have any pending delayed rebuilds
-                rebuild_manager.check_and_rebuild()
+                # Idle period: force rebuild of any pending changes since queue is empty
+                rebuild_manager.check_and_rebuild(force=True)
                 time.sleep(POLL_INTERVAL)
                 continue
             
@@ -210,8 +210,18 @@ def run_worker():
                 if changed:
                     rebuild_manager.record_change()
             
-            # Post-batch check for rebuilding FAISS index
-            rebuild_manager.check_and_rebuild()
+            # Check if there are any remaining unprocessed items in the queue
+            with engine.connect() as conn:
+                any_left = conn.execute(text("""
+                    SELECT TOP 1 QueueId FROM SyncQueue 
+                    WHERE Processed = 0 AND RetryCount < :max_retries
+                """), {"max_retries": MAX_RETRIES}).fetchone()
+            
+            # If no more items are left in the queue, we can rebuild immediately!
+            if not any_left:
+                rebuild_manager.check_and_rebuild(force=True)
+            else:
+                rebuild_manager.check_and_rebuild()
             
         except Exception as e:
             logger.error(f"Error in Sync Worker loop: {e}", exc_info=True)