Dmitry Beresnev commited on
Commit ·
df070dd
1
Parent(s): 97980d5
fix resources consumption
Browse files- app/pages/05_Dashboard.py +15 -5
- app/utils/ai_summary_worker.py +14 -8
app/pages/05_Dashboard.py
CHANGED
|
@@ -28,7 +28,7 @@ from components.news import (
|
|
| 28 |
)
|
| 29 |
from utils.breaking_news_scorer import get_breaking_news_scorer
|
| 30 |
from utils.ai_summary_store import init_storage, enqueue_items, fetch_summaries, get_status
|
| 31 |
-
from utils.ai_summary_worker import start_worker_if_needed
|
| 32 |
|
| 33 |
# Import news scrapers
|
| 34 |
try:
|
|
@@ -196,6 +196,17 @@ with st.sidebar:
|
|
| 196 |
if auto_refresh:
|
| 197 |
st.info("⏱️ Auto-refresh enabled (3 min)")
|
| 198 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 199 |
st.markdown("---")
|
| 200 |
st.markdown("### 📊 Feed Statistics")
|
| 201 |
|
|
@@ -280,9 +291,8 @@ with st.sidebar:
|
|
| 280 |
# Check for forced refresh (don't clear yet - wait until after fetching)
|
| 281 |
force_refresh = st.session_state.get('force_refresh', False)
|
| 282 |
|
| 283 |
-
# Initialize AI summary store
|
| 284 |
init_storage()
|
| 285 |
-
start_worker_if_needed()
|
| 286 |
|
| 287 |
# Fetch news from all sources IN PARALLEL for maximum performance
|
| 288 |
import pandas as pd
|
|
@@ -523,7 +533,7 @@ with st.spinner(f"Loading news from {_TOTAL_SOURCES} sources..."):
|
|
| 523 |
with ThreadPoolExecutor(max_workers=4) as executor:
|
| 524 |
futures_map = {executor.submit(fn): name for fn, name in _fetch_tasks}
|
| 525 |
try:
|
| 526 |
-
for future in as_completed(futures_map, timeout=
|
| 527 |
source_name = futures_map[future]
|
| 528 |
try:
|
| 529 |
result, error = future.result()
|
|
@@ -535,7 +545,7 @@ with st.spinner(f"Loading news from {_TOTAL_SOURCES} sources..."):
|
|
| 535 |
completed_sources.append(f"{source_name} (error)")
|
| 536 |
status_placeholder.warning(f"⚠️ {source_name} failed, continuing with other sources...")
|
| 537 |
except TimeoutError:
|
| 538 |
-
fetch_errors.append("⏱️ Some sources timed out after
|
| 539 |
status_placeholder.warning(f"⚠️ {len(completed_sources)}/{_TOTAL_SOURCES} sources loaded (some timed out)")
|
| 540 |
all_sources = set(futures_map.values())
|
| 541 |
for source in all_sources - set(completed_sources):
|
|
|
|
| 28 |
)
|
| 29 |
from utils.breaking_news_scorer import get_breaking_news_scorer
|
| 30 |
from utils.ai_summary_store import init_storage, enqueue_items, fetch_summaries, get_status
|
| 31 |
+
from utils.ai_summary_worker import start_worker_if_needed, is_worker_running
|
| 32 |
|
| 33 |
# Import news scrapers
|
| 34 |
try:
|
|
|
|
| 196 |
if auto_refresh:
|
| 197 |
st.info("⏱️ Auto-refresh enabled (3 min)")
|
| 198 |
|
| 199 |
+
st.markdown("---")
|
| 200 |
+
st.markdown("### 🤖 AI Summary Worker")
|
| 201 |
+
worker_running = is_worker_running()
|
| 202 |
+
if worker_running:
|
| 203 |
+
st.success("● Worker running")
|
| 204 |
+
else:
|
| 205 |
+
st.warning("● Worker stopped")
|
| 206 |
+
if st.button("▶ Start Worker" if not worker_running else "↺ Restart Worker", use_container_width=True):
|
| 207 |
+
start_worker_if_needed()
|
| 208 |
+
st.rerun()
|
| 209 |
+
|
| 210 |
st.markdown("---")
|
| 211 |
st.markdown("### 📊 Feed Statistics")
|
| 212 |
|
|
|
|
| 291 |
# Check for forced refresh (don't clear yet - wait until after fetching)
|
| 292 |
force_refresh = st.session_state.get('force_refresh', False)
|
| 293 |
|
| 294 |
+
# Initialize AI summary store (shared across sessions/processes)
|
| 295 |
init_storage()
|
|
|
|
| 296 |
|
| 297 |
# Fetch news from all sources IN PARALLEL for maximum performance
|
| 298 |
import pandas as pd
|
|
|
|
| 533 |
with ThreadPoolExecutor(max_workers=4) as executor:
|
| 534 |
futures_map = {executor.submit(fn): name for fn, name in _fetch_tasks}
|
| 535 |
try:
|
| 536 |
+
for future in as_completed(futures_map, timeout=15):
|
| 537 |
source_name = futures_map[future]
|
| 538 |
try:
|
| 539 |
result, error = future.result()
|
|
|
|
| 545 |
completed_sources.append(f"{source_name} (error)")
|
| 546 |
status_placeholder.warning(f"⚠️ {source_name} failed, continuing with other sources...")
|
| 547 |
except TimeoutError:
|
| 548 |
+
fetch_errors.append("⏱️ Some sources timed out after 15 seconds - displaying available results")
|
| 549 |
status_placeholder.warning(f"⚠️ {len(completed_sources)}/{_TOTAL_SOURCES} sources loaded (some timed out)")
|
| 550 |
all_sources = set(futures_map.values())
|
| 551 |
for source in all_sources - set(completed_sources):
|
app/utils/ai_summary_worker.py
CHANGED
|
@@ -87,15 +87,21 @@ def _pid_running(pid: int) -> bool:
|
|
| 87 |
return False
|
| 88 |
|
| 89 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 90 |
def start_worker_if_needed():
|
| 91 |
-
if
|
| 92 |
-
|
| 93 |
-
with open(PID_FILE, "r", encoding="utf-8") as f:
|
| 94 |
-
pid = int(f.read().strip() or 0)
|
| 95 |
-
if pid and _pid_running(pid):
|
| 96 |
-
return
|
| 97 |
-
except Exception:
|
| 98 |
-
pass
|
| 99 |
|
| 100 |
pid = os.fork()
|
| 101 |
if pid != 0:
|
|
|
|
| 87 |
return False
|
| 88 |
|
| 89 |
|
| 90 |
+
def is_worker_running() -> bool:
|
| 91 |
+
"""Return True if the worker process is currently running."""
|
| 92 |
+
if not os.path.exists(PID_FILE):
|
| 93 |
+
return False
|
| 94 |
+
try:
|
| 95 |
+
with open(PID_FILE, "r", encoding="utf-8") as f:
|
| 96 |
+
pid = int(f.read().strip() or 0)
|
| 97 |
+
return bool(pid and _pid_running(pid))
|
| 98 |
+
except Exception:
|
| 99 |
+
return False
|
| 100 |
+
|
| 101 |
+
|
| 102 |
def start_worker_if_needed():
|
| 103 |
+
if is_worker_running():
|
| 104 |
+
return
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 105 |
|
| 106 |
pid = os.fork()
|
| 107 |
if pid != 0:
|