Spaces:
Sleeping
Sleeping
Upload 20 files
Browse files- Dockerfile +36 -0
- agent.py +229 -0
- ai_report_assistant.py +205 -0
- calgary_rules.py +302 -0
- calgary_rules.yaml +22 -0
- model_pipeline.py +319 -0
- preliminary_design_assistant.py +370 -0
- report_engine.py +1417 -0
- requirements.txt +9 -0
- results_db.py +333 -0
- rpt_reconciliation.py +464 -0
- scenario_manager.py +600 -0
- server.py +231 -0
- sessions.py +98 -0
- smoke_test.py +46 -0
- sql_agent.py +390 -0
- swmm_core.py +106 -0
- swmm_worker.py +207 -0
- tools.py +404 -0
- worker-requirements.txt +2 -0
Dockerfile
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
FROM python:3.11-slim
|
| 2 |
+
|
| 3 |
+
ENV PYTHONUNBUFFERED=1 \
|
| 4 |
+
PIP_NO_CACHE_DIR=1 \
|
| 5 |
+
SWMM_WORKER_PYTHON=/opt/swmm-venv/bin/python \
|
| 6 |
+
SWMM_SESSION_ROOT=/tmp/swmm_sessions \
|
| 7 |
+
PYTHONFAULTHANDLER=1
|
| 8 |
+
|
| 9 |
+
WORKDIR /app
|
| 10 |
+
|
| 11 |
+
RUN apt-get update \
|
| 12 |
+
&& apt-get install -y --no-install-recommends build-essential gcc g++ \
|
| 13 |
+
&& rm -rf /var/lib/apt/lists/*
|
| 14 |
+
|
| 15 |
+
COPY requirements.txt worker-requirements.txt /app/
|
| 16 |
+
|
| 17 |
+
# Main server environment: MCP + FastAPI. OpenSWMM is intentionally absent.
|
| 18 |
+
RUN python -m pip install --upgrade pip setuptools wheel \
|
| 19 |
+
&& python -m pip install -r /app/requirements.txt
|
| 20 |
+
|
| 21 |
+
# Native SWMM environment: used only by the one-shot crash-isolated worker.
|
| 22 |
+
RUN python -m venv /opt/swmm-venv \
|
| 23 |
+
&& /opt/swmm-venv/bin/python -m pip install --upgrade pip setuptools wheel \
|
| 24 |
+
&& /opt/swmm-venv/bin/python -m pip install -r /app/worker-requirements.txt \
|
| 25 |
+
&& /opt/swmm-venv/bin/python -c "from openswmm.engine import Solver; print('Isolated OpenSWMM worker verified')"
|
| 26 |
+
|
| 27 |
+
COPY . /app
|
| 28 |
+
|
| 29 |
+
RUN useradd --create-home --uid 1000 appuser \
|
| 30 |
+
&& mkdir -p /tmp/swmm_sessions \
|
| 31 |
+
&& chown -R appuser:appuser /app /opt/swmm-venv /tmp/swmm_sessions
|
| 32 |
+
|
| 33 |
+
USER appuser
|
| 34 |
+
EXPOSE 7860
|
| 35 |
+
|
| 36 |
+
CMD ["python", "-X", "faulthandler", "-m", "uvicorn", "server:app", "--host", "0.0.0.0", "--port", "7860"]
|
agent.py
ADDED
|
@@ -0,0 +1,229 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Built-in agent: an LLM tool-use loop over the SWMM tool registry.
|
| 2 |
+
|
| 3 |
+
For platforms that are not MCP clients (plain REST callers, n8n HTTP nodes,
|
| 4 |
+
Custom GPT Actions, simple webhooks), this provides a single "ask the agent"
|
| 5 |
+
endpoint. MCP-native clients (Claude Desktop/web, Gemini, LangChain,
|
| 6 |
+
Flowise, Langflow) should normally drive the tools directly instead — their
|
| 7 |
+
own model is the agent.
|
| 8 |
+
|
| 9 |
+
Providers (two wire dialects, both via httpx, no SDK dependencies):
|
| 10 |
+
anthropic -> Anthropic Messages API (ANTHROPIC_API_KEY)
|
| 11 |
+
openai -> OpenAI chat completions (OPENAI_API_KEY)
|
| 12 |
+
gemini -> Gemini OpenAI-compatible endpoint (GEMINI_API_KEY)
|
| 13 |
+
groq -> Groq OpenAI-compatible endpoint (GROQ_API_KEY)
|
| 14 |
+
mistral -> Mistral OpenAI-compatible endpoint (MISTRAL_API_KEY)
|
| 15 |
+
local -> any OpenAI-compatible server (Ollama, LM Studio, vLLM) via
|
| 16 |
+
base_url; api_key optional
|
| 17 |
+
|
| 18 |
+
Keys come from environment (HF Space secrets) or per-request overrides.
|
| 19 |
+
Every response includes the full tool-call audit trail.
|
| 20 |
+
"""
|
| 21 |
+
from __future__ import annotations
|
| 22 |
+
|
| 23 |
+
import inspect
|
| 24 |
+
import json
|
| 25 |
+
import os
|
| 26 |
+
import time
|
| 27 |
+
from typing import Any
|
| 28 |
+
|
| 29 |
+
import httpx
|
| 30 |
+
|
| 31 |
+
from tools import TOOL_REGISTRY
|
| 32 |
+
|
| 33 |
+
MAX_STEPS = 8
|
| 34 |
+
TOOL_RESULT_CHAR_LIMIT = 14000
|
| 35 |
+
|
| 36 |
+
SYSTEM_PROMPT = """You are a stormwater modelling analysis agent operating deterministic SWMM tools.
|
| 37 |
+
|
| 38 |
+
Rules of practice:
|
| 39 |
+
- Work from tool results only; never invent numbers. If output is unavailable, say so — do not report zero.
|
| 40 |
+
- Distinguish SCREENING results from CRITERIA: thresholds (e.g. Calgary 3.0/4.0 m/s velocity screens) require confirmation by the responsible engineer; say "screens above/below" not "fails/passes" unless a criterion is confirmed.
|
| 41 |
+
- Typical workflow: upload_model -> run_simulation -> targeted result/screening tools. Reuse an existing session_id when the user provides one.
|
| 42 |
+
- If the rpt_reconciliation verdict flags links, note that .rpt values are authoritative for those links.
|
| 43 |
+
- State clearly that outputs are preliminary engineering screening, not a professional determination.
|
| 44 |
+
Answer concisely with the key numbers and their provenance (which tool produced them)."""
|
| 45 |
+
|
| 46 |
+
PROVIDER_PRESETS: dict[str, dict[str, str]] = {
|
| 47 |
+
"anthropic": {"dialect": "anthropic", "base_url": "https://api.anthropic.com",
|
| 48 |
+
"env": "ANTHROPIC_API_KEY", "default_model": "claude-sonnet-4-5"},
|
| 49 |
+
"openai": {"dialect": "openai", "base_url": "https://api.openai.com/v1",
|
| 50 |
+
"env": "OPENAI_API_KEY", "default_model": "gpt-4o"},
|
| 51 |
+
"gemini": {"dialect": "openai", "base_url": "https://generativelanguage.googleapis.com/v1beta/openai",
|
| 52 |
+
"env": "GEMINI_API_KEY", "default_model": "gemini-2.0-flash"},
|
| 53 |
+
"groq": {"dialect": "openai", "base_url": "https://api.groq.com/openai/v1",
|
| 54 |
+
"env": "GROQ_API_KEY", "default_model": "llama-3.3-70b-versatile"},
|
| 55 |
+
"mistral": {"dialect": "openai", "base_url": "https://api.mistral.ai/v1",
|
| 56 |
+
"env": "MISTRAL_API_KEY", "default_model": "mistral-large-latest"},
|
| 57 |
+
"local": {"dialect": "openai", "base_url": os.environ.get("LOCAL_LLM_BASE_URL", "http://localhost:11434/v1"),
|
| 58 |
+
"env": "LOCAL_LLM_API_KEY", "default_model": os.environ.get("LOCAL_LLM_MODEL", "llama3.1")},
|
| 59 |
+
}
|
| 60 |
+
|
| 61 |
+
# Tools the agent may call. upload_model is included so callers can pass INP
|
| 62 |
+
# content inline; generate_report excluded by default (large side effects)
|
| 63 |
+
# unless allow_report=True.
|
| 64 |
+
AGENT_TOOLS_DEFAULT = [
|
| 65 |
+
"upload_model", "run_simulation", "list_sessions", "get_node_results",
|
| 66 |
+
"get_link_results", "get_subcatchment_results", "get_timeseries",
|
| 67 |
+
"query_results", "get_table_catalog", "calgary_screening",
|
| 68 |
+
"preliminary_design_review", "get_reconciliation", "run_scenario",
|
| 69 |
+
]
|
| 70 |
+
|
| 71 |
+
_JSON_TYPES = {str: "string", int: "integer", float: "number", bool: "boolean",
|
| 72 |
+
dict: "object", list: "array"}
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def _tool_schemas(names: list[str]) -> list[dict[str, Any]]:
|
| 76 |
+
schemas = []
|
| 77 |
+
for name in names:
|
| 78 |
+
fn = TOOL_REGISTRY.get(name)
|
| 79 |
+
if fn is None:
|
| 80 |
+
continue
|
| 81 |
+
sig = inspect.signature(fn)
|
| 82 |
+
props, required = {}, []
|
| 83 |
+
for pname, param in sig.parameters.items():
|
| 84 |
+
ann = param.annotation
|
| 85 |
+
jtype = "string"
|
| 86 |
+
for py, js in _JSON_TYPES.items():
|
| 87 |
+
if ann is py:
|
| 88 |
+
jtype = js
|
| 89 |
+
break
|
| 90 |
+
if ann in (dict | str | None, dict | str):
|
| 91 |
+
jtype = "object"
|
| 92 |
+
props[pname] = {"type": jtype}
|
| 93 |
+
if param.default is inspect.Parameter.empty:
|
| 94 |
+
required.append(pname)
|
| 95 |
+
schemas.append({"name": name,
|
| 96 |
+
"description": (fn.__doc__ or name).strip()[:900],
|
| 97 |
+
"input_schema": {"type": "object", "properties": props, "required": required}})
|
| 98 |
+
return schemas
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def _execute(name: str, arguments: dict[str, Any]) -> str:
|
| 102 |
+
fn = TOOL_REGISTRY.get(name)
|
| 103 |
+
if fn is None:
|
| 104 |
+
return json.dumps({"error": f"unknown tool {name}"})
|
| 105 |
+
try:
|
| 106 |
+
result = fn(**(arguments or {}))
|
| 107 |
+
text = json.dumps(result, default=str)
|
| 108 |
+
except Exception as exc: # deterministic error surface for the model
|
| 109 |
+
text = json.dumps({"error": f"{type(exc).__name__}: {exc}"})
|
| 110 |
+
if len(text) > TOOL_RESULT_CHAR_LIMIT:
|
| 111 |
+
text = text[:TOOL_RESULT_CHAR_LIMIT] + '... (truncated — request a smaller limit or use query_results)"}'
|
| 112 |
+
return text
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
class LLMClient:
|
| 116 |
+
"""Minimal two-dialect chat client. `transport` is injectable for tests."""
|
| 117 |
+
|
| 118 |
+
def __init__(self, provider: str, model: str | None = None, api_key: str | None = None,
|
| 119 |
+
base_url: str | None = None, transport: Any | None = None):
|
| 120 |
+
preset = PROVIDER_PRESETS.get(provider)
|
| 121 |
+
if preset is None:
|
| 122 |
+
raise ValueError(f"Unknown provider '{provider}'. Choose from {sorted(PROVIDER_PRESETS)}.")
|
| 123 |
+
self.provider = provider
|
| 124 |
+
self.dialect = preset["dialect"]
|
| 125 |
+
self.base_url = (base_url or preset["base_url"]).rstrip("/")
|
| 126 |
+
self.model = model or preset["default_model"]
|
| 127 |
+
self.api_key = api_key or os.environ.get(preset["env"], "")
|
| 128 |
+
if not self.api_key and provider != "local":
|
| 129 |
+
raise ValueError(
|
| 130 |
+
f"No API key for provider '{provider}'. Set the {preset['env']} Space secret "
|
| 131 |
+
"or pass api_key in the request.")
|
| 132 |
+
self._transport = transport
|
| 133 |
+
|
| 134 |
+
def chat(self, messages: list[dict], tools: list[dict]) -> dict:
|
| 135 |
+
if self._transport is not None:
|
| 136 |
+
return self._transport(self, messages, tools)
|
| 137 |
+
if self.dialect == "anthropic":
|
| 138 |
+
return self._chat_anthropic(messages, tools)
|
| 139 |
+
return self._chat_openai(messages, tools)
|
| 140 |
+
|
| 141 |
+
def _chat_anthropic(self, messages: list[dict], tools: list[dict]) -> dict:
|
| 142 |
+
resp = httpx.post(
|
| 143 |
+
f"{self.base_url}/v1/messages",
|
| 144 |
+
headers={"x-api-key": self.api_key, "anthropic-version": "2023-06-01"},
|
| 145 |
+
json={"model": self.model, "max_tokens": 2000, "system": SYSTEM_PROMPT,
|
| 146 |
+
"messages": messages, "tools": tools},
|
| 147 |
+
timeout=120.0)
|
| 148 |
+
resp.raise_for_status()
|
| 149 |
+
data = resp.json()
|
| 150 |
+
calls = [{"id": b["id"], "name": b["name"], "arguments": b["input"]}
|
| 151 |
+
for b in data.get("content", []) if b.get("type") == "tool_use"]
|
| 152 |
+
text = "".join(b.get("text", "") for b in data.get("content", []) if b.get("type") == "text")
|
| 153 |
+
return {"text": text, "tool_calls": calls, "raw_content": data.get("content", []),
|
| 154 |
+
"stop": data.get("stop_reason")}
|
| 155 |
+
|
| 156 |
+
def _chat_openai(self, messages: list[dict], tools: list[dict]) -> dict:
|
| 157 |
+
oai_tools = [{"type": "function",
|
| 158 |
+
"function": {"name": t["name"], "description": t["description"],
|
| 159 |
+
"parameters": t["input_schema"]}} for t in tools]
|
| 160 |
+
oai_messages = [{"role": "system", "content": SYSTEM_PROMPT}] + messages
|
| 161 |
+
headers = {"Content-Type": "application/json"}
|
| 162 |
+
if self.api_key:
|
| 163 |
+
headers["Authorization"] = f"Bearer {self.api_key}"
|
| 164 |
+
resp = httpx.post(f"{self.base_url}/chat/completions", headers=headers,
|
| 165 |
+
json={"model": self.model, "messages": oai_messages,
|
| 166 |
+
"tools": oai_tools or None}, timeout=120.0)
|
| 167 |
+
resp.raise_for_status()
|
| 168 |
+
msg = resp.json()["choices"][0]["message"]
|
| 169 |
+
calls = [{"id": c["id"], "name": c["function"]["name"],
|
| 170 |
+
"arguments": json.loads(c["function"]["arguments"] or "{}")}
|
| 171 |
+
for c in (msg.get("tool_calls") or [])]
|
| 172 |
+
return {"text": msg.get("content") or "", "tool_calls": calls,
|
| 173 |
+
"raw_message": msg, "stop": "tool_use" if calls else "end"}
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
def run_agent(question: str, provider: str = "anthropic", model: str | None = None,
|
| 177 |
+
api_key: str | None = None, base_url: str | None = None,
|
| 178 |
+
session_id: str | None = None, inp_content: str | None = None,
|
| 179 |
+
allow_report: bool = False, max_steps: int = MAX_STEPS,
|
| 180 |
+
transport: Any | None = None) -> dict:
|
| 181 |
+
"""Run the tool-use loop and return {answer, tool_trace, steps, provider}."""
|
| 182 |
+
client = LLMClient(provider, model, api_key, base_url, transport)
|
| 183 |
+
tool_names = list(AGENT_TOOLS_DEFAULT) + (["generate_report", "close_session"] if allow_report else [])
|
| 184 |
+
tools = _tool_schemas(tool_names)
|
| 185 |
+
|
| 186 |
+
user_text = question
|
| 187 |
+
if session_id:
|
| 188 |
+
user_text += f"\n\n(Existing session_id: {session_id})"
|
| 189 |
+
if inp_content:
|
| 190 |
+
user_text += "\n\nA SWMM .inp model is provided below — upload it first.\n<inp_file>\n" + inp_content[:400000] + "\n</inp_file>"
|
| 191 |
+
|
| 192 |
+
trace: list[dict[str, Any]] = []
|
| 193 |
+
if client.dialect == "anthropic":
|
| 194 |
+
messages: list[dict] = [{"role": "user", "content": user_text}]
|
| 195 |
+
for step in range(max_steps):
|
| 196 |
+
reply = client.chat(messages, tools)
|
| 197 |
+
if not reply["tool_calls"]:
|
| 198 |
+
return {"answer": reply["text"], "tool_trace": trace, "steps": step + 1,
|
| 199 |
+
"provider": provider, "model": client.model}
|
| 200 |
+
messages.append({"role": "assistant", "content": reply["raw_content"]})
|
| 201 |
+
results_content = []
|
| 202 |
+
for call in reply["tool_calls"]:
|
| 203 |
+
t0 = time.time()
|
| 204 |
+
output = _execute(call["name"], call["arguments"])
|
| 205 |
+
trace.append({"tool": call["name"], "arguments": call["arguments"],
|
| 206 |
+
"elapsed_s": round(time.time() - t0, 2),
|
| 207 |
+
"result_preview": output[:400]})
|
| 208 |
+
results_content.append({"type": "tool_result", "tool_use_id": call["id"],
|
| 209 |
+
"content": output})
|
| 210 |
+
messages.append({"role": "user", "content": results_content})
|
| 211 |
+
else:
|
| 212 |
+
messages = [{"role": "user", "content": user_text}]
|
| 213 |
+
for step in range(max_steps):
|
| 214 |
+
reply = client.chat(messages, tools)
|
| 215 |
+
if not reply["tool_calls"]:
|
| 216 |
+
return {"answer": reply["text"], "tool_trace": trace, "steps": step + 1,
|
| 217 |
+
"provider": provider, "model": client.model}
|
| 218 |
+
messages.append(reply["raw_message"])
|
| 219 |
+
for call in reply["tool_calls"]:
|
| 220 |
+
t0 = time.time()
|
| 221 |
+
output = _execute(call["name"], call["arguments"])
|
| 222 |
+
trace.append({"tool": call["name"], "arguments": call["arguments"],
|
| 223 |
+
"elapsed_s": round(time.time() - t0, 2),
|
| 224 |
+
"result_preview": output[:400]})
|
| 225 |
+
messages.append({"role": "tool", "tool_call_id": call["id"], "content": output})
|
| 226 |
+
|
| 227 |
+
return {"answer": "Agent reached the maximum number of steps without a final answer. "
|
| 228 |
+
"Partial evidence is in tool_trace.",
|
| 229 |
+
"tool_trace": trace, "steps": max_steps, "provider": provider, "model": client.model}
|
ai_report_assistant.py
ADDED
|
@@ -0,0 +1,205 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Optional AI assistance for Calgary SWMR drafting and consistency review.
|
| 2 |
+
|
| 3 |
+
This module is intentionally separate from the general model-analysis agent.
|
| 4 |
+
The deterministic report engine remains the source of all engineering values,
|
| 5 |
+
criteria statuses, checklist statuses, and permitted conclusions.
|
| 6 |
+
"""
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import json
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
from typing import Any, Mapping, Sequence
|
| 12 |
+
|
| 13 |
+
import requests
|
| 14 |
+
|
| 15 |
+
PROMPT_DIR = Path(__file__).resolve().parent / "prompts"
|
| 16 |
+
|
| 17 |
+
PROVIDERS: dict[str, dict[str, Any]] = {
|
| 18 |
+
"Claude (Anthropic)": {
|
| 19 |
+
"models": ["claude-opus-4-5", "claude-sonnet-4-5", "claude-haiku-4-5"],
|
| 20 |
+
"format": "anthropic",
|
| 21 |
+
"url": "https://api.anthropic.com/v1/messages",
|
| 22 |
+
},
|
| 23 |
+
"GPT (OpenAI)": {
|
| 24 |
+
"models": ["gpt-4o", "gpt-4o-mini", "gpt-4-turbo"],
|
| 25 |
+
"format": "openai",
|
| 26 |
+
"url": "https://api.openai.com/v1/chat/completions",
|
| 27 |
+
},
|
| 28 |
+
"Gemini (Google)": {
|
| 29 |
+
"models": ["gemini-2.0-flash", "gemini-1.5-pro", "gemini-1.5-flash"],
|
| 30 |
+
"format": "gemini",
|
| 31 |
+
"url": "https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent",
|
| 32 |
+
},
|
| 33 |
+
"Groq (Free tier)": {
|
| 34 |
+
"models": ["llama-3.3-70b-versatile", "llama-3.1-8b-instant", "mixtral-8x7b-32768"],
|
| 35 |
+
"format": "openai",
|
| 36 |
+
"url": "https://api.groq.com/openai/v1/chat/completions",
|
| 37 |
+
},
|
| 38 |
+
"Mistral": {
|
| 39 |
+
"models": ["mistral-large-latest", "mistral-small-latest", "open-mistral-7b"],
|
| 40 |
+
"format": "openai",
|
| 41 |
+
"url": "https://api.mistral.ai/v1/chat/completions",
|
| 42 |
+
},
|
| 43 |
+
}
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def load_prompt(name: str) -> str:
|
| 47 |
+
path = PROMPT_DIR / name
|
| 48 |
+
if not path.exists():
|
| 49 |
+
raise FileNotFoundError(f"Prompt file not found: {path}")
|
| 50 |
+
return path.read_text(encoding="utf-8").strip()
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
DRAFT_SYSTEM_PROMPT = load_prompt("calgary_report_drafting_system.txt")
|
| 54 |
+
REVIEW_SYSTEM_PROMPT = load_prompt("calgary_report_review_system.txt")
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def _call_provider(
|
| 58 |
+
provider_name: str,
|
| 59 |
+
api_key: str,
|
| 60 |
+
model: str,
|
| 61 |
+
messages: Sequence[Mapping[str, str]],
|
| 62 |
+
system_prompt: str,
|
| 63 |
+
max_tokens: int = 3200,
|
| 64 |
+
timeout: int = 90,
|
| 65 |
+
) -> str:
|
| 66 |
+
if provider_name not in PROVIDERS:
|
| 67 |
+
raise ValueError(f"Unsupported provider: {provider_name}")
|
| 68 |
+
if not api_key.strip():
|
| 69 |
+
raise ValueError("An API key is required for optional AI assistance.")
|
| 70 |
+
|
| 71 |
+
provider = PROVIDERS[provider_name]
|
| 72 |
+
fmt = provider["format"]
|
| 73 |
+
|
| 74 |
+
if fmt == "anthropic":
|
| 75 |
+
response = requests.post(
|
| 76 |
+
provider["url"],
|
| 77 |
+
headers={
|
| 78 |
+
"Content-Type": "application/json",
|
| 79 |
+
"x-api-key": api_key,
|
| 80 |
+
"anthropic-version": "2023-06-01",
|
| 81 |
+
},
|
| 82 |
+
json={
|
| 83 |
+
"model": model,
|
| 84 |
+
"max_tokens": max_tokens,
|
| 85 |
+
"system": system_prompt,
|
| 86 |
+
"messages": list(messages),
|
| 87 |
+
},
|
| 88 |
+
timeout=timeout,
|
| 89 |
+
)
|
| 90 |
+
response.raise_for_status()
|
| 91 |
+
return response.json()["content"][0]["text"].strip()
|
| 92 |
+
|
| 93 |
+
if fmt == "openai":
|
| 94 |
+
response = requests.post(
|
| 95 |
+
provider["url"],
|
| 96 |
+
headers={
|
| 97 |
+
"Content-Type": "application/json",
|
| 98 |
+
"Authorization": f"Bearer {api_key}",
|
| 99 |
+
},
|
| 100 |
+
json={
|
| 101 |
+
"model": model,
|
| 102 |
+
"max_tokens": max_tokens,
|
| 103 |
+
"messages": [{"role": "system", "content": system_prompt}] + list(messages),
|
| 104 |
+
},
|
| 105 |
+
timeout=timeout,
|
| 106 |
+
)
|
| 107 |
+
response.raise_for_status()
|
| 108 |
+
return response.json()["choices"][0]["message"]["content"].strip()
|
| 109 |
+
|
| 110 |
+
if fmt == "gemini":
|
| 111 |
+
url = provider["url"].replace("{model}", model) + f"?key={api_key}"
|
| 112 |
+
contents = []
|
| 113 |
+
for message in messages:
|
| 114 |
+
role = "model" if message["role"] == "assistant" else "user"
|
| 115 |
+
contents.append({"role": role, "parts": [{"text": message["content"]}]})
|
| 116 |
+
response = requests.post(
|
| 117 |
+
url,
|
| 118 |
+
json={
|
| 119 |
+
"system_instruction": {"parts": [{"text": system_prompt}]},
|
| 120 |
+
"contents": contents,
|
| 121 |
+
"generationConfig": {"maxOutputTokens": max_tokens},
|
| 122 |
+
},
|
| 123 |
+
timeout=timeout,
|
| 124 |
+
)
|
| 125 |
+
response.raise_for_status()
|
| 126 |
+
return response.json()["candidates"][0]["content"]["parts"][0]["text"].strip()
|
| 127 |
+
|
| 128 |
+
raise ValueError(f"Provider format is not implemented: {fmt}")
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def _safe_json(data: Mapping[str, Any]) -> str:
|
| 132 |
+
return json.dumps(data, ensure_ascii=False, default=str, separators=(",", ":"))
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
def draft_report_section(
|
| 136 |
+
*,
|
| 137 |
+
provider_name: str,
|
| 138 |
+
api_key: str,
|
| 139 |
+
model: str,
|
| 140 |
+
section: str,
|
| 141 |
+
report_context: Mapping[str, Any],
|
| 142 |
+
additional_instruction: str = "",
|
| 143 |
+
) -> str:
|
| 144 |
+
"""Draft narrative from deterministic report context only."""
|
| 145 |
+
request = f"""Prepare this Calgary SWMR draft component: {section}.
|
| 146 |
+
|
| 147 |
+
Use only the VERIFIED_REPORT_CONTEXT JSON below. Follow every conclusion-control,
|
| 148 |
+
checklist, storage, outfall, minor-system, major-system, and model-input/output rule
|
| 149 |
+
in the system prompt. Preserve all values and units exactly.
|
| 150 |
+
|
| 151 |
+
VERIFIED_REPORT_CONTEXT:
|
| 152 |
+
{_safe_json(report_context)}
|
| 153 |
+
|
| 154 |
+
ADDITIONAL_USER_INSTRUCTION:
|
| 155 |
+
{additional_instruction.strip() or 'None'}
|
| 156 |
+
|
| 157 |
+
Return report-ready prose with clear headings. Do not include a preamble about being an AI.
|
| 158 |
+
"""
|
| 159 |
+
return _call_provider(
|
| 160 |
+
provider_name,
|
| 161 |
+
api_key,
|
| 162 |
+
model,
|
| 163 |
+
[{"role": "user", "content": request}],
|
| 164 |
+
DRAFT_SYSTEM_PROMPT,
|
| 165 |
+
)
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
def review_report_narrative(
|
| 169 |
+
*,
|
| 170 |
+
provider_name: str,
|
| 171 |
+
api_key: str,
|
| 172 |
+
model: str,
|
| 173 |
+
narrative: str,
|
| 174 |
+
report_context: Mapping[str, Any],
|
| 175 |
+
) -> str:
|
| 176 |
+
"""Review narrative against deterministic facts without changing model results."""
|
| 177 |
+
request = f"""Review the DRAFT_NARRATIVE against VERIFIED_REPORT_CONTEXT.
|
| 178 |
+
Identify unsupported claims, value or unit discrepancies, checklist omissions,
|
| 179 |
+
misuse of criteria, overstatements, and contradictions. Then provide corrected
|
| 180 |
+
replacement wording for each material issue.
|
| 181 |
+
|
| 182 |
+
VERIFIED_REPORT_CONTEXT:
|
| 183 |
+
{_safe_json(report_context)}
|
| 184 |
+
|
| 185 |
+
DRAFT_NARRATIVE:
|
| 186 |
+
{narrative}
|
| 187 |
+
"""
|
| 188 |
+
return _call_provider(
|
| 189 |
+
provider_name,
|
| 190 |
+
api_key,
|
| 191 |
+
model,
|
| 192 |
+
[{"role": "user", "content": request}],
|
| 193 |
+
REVIEW_SYSTEM_PROMPT,
|
| 194 |
+
)
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
def draft_multiple_report_sections(*, provider_name: str, api_key: str, model: str, sections: Sequence[str], report_context: Mapping[str, Any], additional_instruction: str = "") -> dict[str, str]:
|
| 198 |
+
"""Generate independent section drafts so each can be reviewed and approved."""
|
| 199 |
+
drafts: dict[str, str] = {}
|
| 200 |
+
for section in sections:
|
| 201 |
+
drafts[section] = draft_report_section(
|
| 202 |
+
provider_name=provider_name, api_key=api_key, model=model, section=section,
|
| 203 |
+
report_context=report_context, additional_instruction=additional_instruction,
|
| 204 |
+
)
|
| 205 |
+
return drafts
|
calgary_rules.py
ADDED
|
@@ -0,0 +1,302 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Deterministic Calgary SWMR criteria helpers.
|
| 2 |
+
|
| 3 |
+
The rules in this module are deliberately transparent and configurable. They are
|
| 4 |
+
screening/default values, not a substitute for current City direction, an
|
| 5 |
+
approved SMDP/MDP, or professional engineering judgement.
|
| 6 |
+
"""
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import json
|
| 10 |
+
import math
|
| 11 |
+
import re
|
| 12 |
+
from dataclasses import asdict, dataclass, field
|
| 13 |
+
from typing import Any
|
| 14 |
+
|
| 15 |
+
import pandas as pd
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
@dataclass
|
| 19 |
+
class CalgaryCriteria:
|
| 20 |
+
profile_name: str = "City of Calgary SWMR"
|
| 21 |
+
source_manual: str = "City of Calgary Stormwater Management & Design Manual (2011)"
|
| 22 |
+
manual_status: str = "Historical baseline - verify current amendments"
|
| 23 |
+
minor_release_rate_lps_ha: float | None = None
|
| 24 |
+
trap_low_max_depth_m: float = 0.50
|
| 25 |
+
entrance_grade_margin_m: float = 0.30
|
| 26 |
+
pipe_advisory_velocity_mps: float = 3.0
|
| 27 |
+
pipe_critical_velocity_mps: float = 4.0
|
| 28 |
+
conduit_capacity_review_ratio: float = 0.80
|
| 29 |
+
conduit_capacity_warning_ratio: float = 0.95
|
| 30 |
+
continuity_review_pct: float = 0.50
|
| 31 |
+
continuity_warning_pct: float = 1.00
|
| 32 |
+
depth_velocity_curve: tuple[tuple[float, float], ...] = (
|
| 33 |
+
(0.5, 0.80), (1.0, 0.32), (2.0, 0.21), (3.0, 0.09)
|
| 34 |
+
)
|
| 35 |
+
special_link_limits: dict[str, float] = field(default_factory=dict)
|
| 36 |
+
storage_classification: dict[str, str] = field(default_factory=dict)
|
| 37 |
+
outfall_classification: dict[str, str] = field(default_factory=dict)
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def infer_design_event(rain_gages: list[str], fallback: str = "Model design event") -> tuple[str, str]:
|
| 41 |
+
names = [str(x) for x in rain_gages if str(x).strip()]
|
| 42 |
+
for name in names:
|
| 43 |
+
normalized = name.replace("_", " ").replace("-", " ")
|
| 44 |
+
duration = re.search(r"(\d+(?:\.\d+)?)\s*h", normalized, re.I)
|
| 45 |
+
return_period = re.search(r"(?:1\s*[:/]\s*)?(\d+)\s*y", normalized, re.I)
|
| 46 |
+
if duration and return_period:
|
| 47 |
+
return (
|
| 48 |
+
f"Calgary {duration.group(1)}-hour, 1:{return_period.group(1)}-year design storm",
|
| 49 |
+
f"Inferred from rain-gage name '{name}'; confirm before issue.",
|
| 50 |
+
)
|
| 51 |
+
return fallback, "Entered by user or retained from model metadata; confirm before issue."
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def permissible_overland_depth(velocity_mps: float, curve: tuple[tuple[float, float], ...]) -> float | None:
|
| 55 |
+
"""Linearly interpolate the Alberta/Calgary street depth-velocity envelope."""
|
| 56 |
+
try:
|
| 57 |
+
v = float(velocity_mps)
|
| 58 |
+
except (TypeError, ValueError):
|
| 59 |
+
return None
|
| 60 |
+
pts = sorted((float(x), float(y)) for x, y in curve)
|
| 61 |
+
if not pts:
|
| 62 |
+
return None
|
| 63 |
+
if v <= pts[0][0]:
|
| 64 |
+
return pts[0][1]
|
| 65 |
+
if v > pts[-1][0]:
|
| 66 |
+
return pts[-1][1]
|
| 67 |
+
for (v1, d1), (v2, d2) in zip(pts[:-1], pts[1:]):
|
| 68 |
+
if v1 <= v <= v2:
|
| 69 |
+
f = (v - v1) / (v2 - v1)
|
| 70 |
+
return d1 + f * (d2 - d1)
|
| 71 |
+
return None
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def classify_overland(depth_m: float, velocity_mps: float, curve: tuple[tuple[float, float], ...]) -> tuple[str, float | None]:
|
| 75 |
+
allowed = permissible_overland_depth(velocity_mps, curve)
|
| 76 |
+
if allowed is None:
|
| 77 |
+
return "Review - criterion unavailable", None
|
| 78 |
+
d = float(depth_m or 0.0)
|
| 79 |
+
ratio = d / allowed if allowed > 0 else math.inf
|
| 80 |
+
if ratio <= 0.90:
|
| 81 |
+
return "Pass", allowed
|
| 82 |
+
if ratio <= 1.00:
|
| 83 |
+
return "Pass - near limit", allowed
|
| 84 |
+
return "Exceeds depth-velocity criterion", allowed
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def manning_full_capacity_circular(diameter: float, slope: float, n: float, unit_system: str) -> float | None:
|
| 88 |
+
"""Full-flow Manning capacity for a circular conduit in model-native flow units.
|
| 89 |
+
|
| 90 |
+
SI result: m3/s. US customary result: cfs.
|
| 91 |
+
"""
|
| 92 |
+
try:
|
| 93 |
+
d, s, rough = float(diameter), float(slope), float(n)
|
| 94 |
+
except (TypeError, ValueError):
|
| 95 |
+
return None
|
| 96 |
+
if d <= 0 or s <= 0 or rough <= 0:
|
| 97 |
+
return None
|
| 98 |
+
area = math.pi * d * d / 4.0
|
| 99 |
+
radius = d / 4.0
|
| 100 |
+
coefficient = 1.0 if unit_system.upper().startswith("SI") else 1.486
|
| 101 |
+
return coefficient / rough * area * radius ** (2.0 / 3.0) * math.sqrt(s)
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def build_minor_system_capacity_table(link_table: pd.DataFrame, unit_system: str, flow_unit: str, length_unit: str, criteria: CalgaryCriteria) -> pd.DataFrame:
|
| 105 |
+
if link_table is None or link_table.empty:
|
| 106 |
+
return pd.DataFrame()
|
| 107 |
+
df = link_table.copy()
|
| 108 |
+
if "Model Type" in df:
|
| 109 |
+
df = df[df["Model Type"].astype(str).str.lower().eq("conduit")]
|
| 110 |
+
if "Shape" in df:
|
| 111 |
+
df = df[df["Shape"].astype(str).str.upper().eq("CIRCULAR")]
|
| 112 |
+
diam_col = next((c for c in df.columns if c.startswith("Diameter / Geom1") or c.startswith("Diameter (")), None)
|
| 113 |
+
flow_col = next((c for c in df.columns if c.startswith("Peak Flow (")), None)
|
| 114 |
+
length_col = next((c for c in df.columns if c.startswith("Length (")), None)
|
| 115 |
+
if not all([diam_col, flow_col]):
|
| 116 |
+
return pd.DataFrame()
|
| 117 |
+
rows = []
|
| 118 |
+
for _, r in df.iterrows():
|
| 119 |
+
slope = pd.to_numeric(r.get("Slope (ft/ft)", r.get("Slope", math.nan)), errors="coerce")
|
| 120 |
+
# The model table may not include slope. Calculate from offsets/inverts only when explicitly available.
|
| 121 |
+
cap = manning_full_capacity_circular(r.get(diam_col), slope, r.get("Manning n"), unit_system) if pd.notna(slope) else None
|
| 122 |
+
peak = pd.to_numeric(r.get(flow_col), errors="coerce")
|
| 123 |
+
modelled_depth_ratio = pd.to_numeric(r.get("Depth Ratio"), errors="coerce")
|
| 124 |
+
capacity_ratio = float(peak / cap) if cap and pd.notna(peak) else math.nan
|
| 125 |
+
if pd.notna(capacity_ratio):
|
| 126 |
+
if capacity_ratio >= criteria.conduit_capacity_warning_ratio:
|
| 127 |
+
status = "Warning - limited calculated capacity"
|
| 128 |
+
elif capacity_ratio >= criteria.conduit_capacity_review_ratio:
|
| 129 |
+
status = "Review calculated capacity"
|
| 130 |
+
else:
|
| 131 |
+
status = "Pass calculated capacity screening"
|
| 132 |
+
capacity_basis = "Manning full-flow capacity"
|
| 133 |
+
elif pd.notna(modelled_depth_ratio):
|
| 134 |
+
if modelled_depth_ratio >= criteria.conduit_capacity_warning_ratio:
|
| 135 |
+
status = "Warning - high modelled depth ratio"
|
| 136 |
+
elif modelled_depth_ratio >= criteria.conduit_capacity_review_ratio:
|
| 137 |
+
status = "Review modelled depth ratio"
|
| 138 |
+
else:
|
| 139 |
+
status = "Below depth-ratio screening threshold"
|
| 140 |
+
capacity_basis = "Full-flow capacity not calculated"
|
| 141 |
+
else:
|
| 142 |
+
status = "Not assessed - missing capacity and depth ratio"
|
| 143 |
+
capacity_basis = "Insufficient data"
|
| 144 |
+
rows.append({
|
| 145 |
+
"Segment": r.get("Link ID"), "From": r.get("From Node"), "To": r.get("To Node"),
|
| 146 |
+
f"Diameter ({length_unit})": r.get(diam_col), f"Length ({length_unit})": r.get(length_col) if length_col else None,
|
| 147 |
+
"Manning n": r.get("Manning n"), f"Routed Flow ({flow_unit})": peak,
|
| 148 |
+
f"Full-Flow Capacity ({flow_unit})": cap,
|
| 149 |
+
"Calculated Capacity Ratio": capacity_ratio if pd.notna(capacity_ratio) else None,
|
| 150 |
+
"Modelled Depth Ratio": modelled_depth_ratio if pd.notna(modelled_depth_ratio) else None,
|
| 151 |
+
f"Spare Capacity ({flow_unit})": (cap - peak) if cap and pd.notna(peak) else None,
|
| 152 |
+
"Assessment Basis": capacity_basis,
|
| 153 |
+
"Status": status,
|
| 154 |
+
})
|
| 155 |
+
return pd.DataFrame(rows)
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
def build_overland_compliance_table(overland: pd.DataFrame, criteria: CalgaryCriteria, flow_unit: str, length_unit: str, velocity_unit: str) -> pd.DataFrame:
|
| 159 |
+
if overland is None or overland.empty:
|
| 160 |
+
return pd.DataFrame()
|
| 161 |
+
flow_col = next((c for c in overland.columns if c.startswith("Peak Flow (")), None)
|
| 162 |
+
depth_col = next((c for c in overland.columns if c.startswith("Peak Depth (")), None)
|
| 163 |
+
vel_col = next((c for c in overland.columns if c.startswith("Peak Velocity (")), None)
|
| 164 |
+
rows = []
|
| 165 |
+
for _, r in overland.iterrows():
|
| 166 |
+
d = pd.to_numeric(r.get(depth_col), errors="coerce") if depth_col else math.nan
|
| 167 |
+
v = pd.to_numeric(r.get(vel_col), errors="coerce") if vel_col else math.nan
|
| 168 |
+
status, permitted = classify_overland(d, v, criteria.depth_velocity_curve) if pd.notna(d) and pd.notna(v) else ("Review - missing depth/velocity", None)
|
| 169 |
+
link_id = str(r.get("Link ID", ""))
|
| 170 |
+
special_limit = criteria.special_link_limits.get(link_id)
|
| 171 |
+
special_status = ""
|
| 172 |
+
if special_limit is not None and flow_col:
|
| 173 |
+
q = pd.to_numeric(r.get(flow_col), errors="coerce")
|
| 174 |
+
special_status = "Pass" if pd.notna(q) and q <= special_limit else "Exceeds project-specific flow limit"
|
| 175 |
+
rows.append({
|
| 176 |
+
"Segment": link_id, "From": r.get("From Node"), "To": r.get("To Node"),
|
| 177 |
+
f"Peak Flow ({flow_unit})": r.get(flow_col) if flow_col else None,
|
| 178 |
+
f"Peak Depth ({length_unit})": d, f"Peak Velocity ({velocity_unit})": v,
|
| 179 |
+
f"Permissible Depth ({length_unit})": permitted, "Depth-Velocity Status": status,
|
| 180 |
+
f"Special Flow Limit ({flow_unit})": special_limit, "Special Limit Status": special_status,
|
| 181 |
+
"Spill Active": "Yes" if ("spill" in link_id.lower() and pd.to_numeric(r.get(flow_col), errors="coerce") > 0) else "No",
|
| 182 |
+
})
|
| 183 |
+
return pd.DataFrame(rows)
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
def apply_storage_classification(storage_table: pd.DataFrame, criteria: CalgaryCriteria, length_unit: str) -> pd.DataFrame:
|
| 187 |
+
if storage_table is None or storage_table.empty:
|
| 188 |
+
return pd.DataFrame()
|
| 189 |
+
df = storage_table.copy()
|
| 190 |
+
depth_col = next((c for c in df.columns if c.startswith("Time-Series Peak Depth") or c.startswith("Reported Peak Depth")), None)
|
| 191 |
+
max_col = next((c for c in df.columns if c.startswith("Maximum Depth")), None)
|
| 192 |
+
classes, statuses, margins = [], [], []
|
| 193 |
+
for _, r in df.iterrows():
|
| 194 |
+
sid = str(r.get("Storage ID", ""))
|
| 195 |
+
cls = criteria.storage_classification.get(sid)
|
| 196 |
+
if not cls:
|
| 197 |
+
s = sid.lower()
|
| 198 |
+
if "storage" in s:
|
| 199 |
+
cls = "Street trap low / surface storage"
|
| 200 |
+
elif s.startswith("cb"):
|
| 201 |
+
cls = "Catchbasin ponding storage"
|
| 202 |
+
elif s.startswith("sub"):
|
| 203 |
+
cls = "Private-site / routing storage"
|
| 204 |
+
else:
|
| 205 |
+
cls = "Unclassified storage"
|
| 206 |
+
peak = pd.to_numeric(r.get(depth_col), errors="coerce") if depth_col else math.nan
|
| 207 |
+
maxd = pd.to_numeric(r.get(max_col), errors="coerce") if max_col else math.nan
|
| 208 |
+
margin = maxd - peak if pd.notna(maxd) and pd.notna(peak) else math.nan
|
| 209 |
+
ratio = peak / maxd if pd.notna(maxd) and maxd > 0 and pd.notna(peak) else math.nan
|
| 210 |
+
if cls.startswith("Street trap"):
|
| 211 |
+
if pd.isna(peak): status = "Review - missing peak depth"
|
| 212 |
+
elif peak > criteria.trap_low_max_depth_m: status = "Exceeds trap-low depth criterion"
|
| 213 |
+
elif peak >= 0.95 * criteria.trap_low_max_depth_m: status = "Pass - limited margin"
|
| 214 |
+
else: status = "Pass"
|
| 215 |
+
else:
|
| 216 |
+
if pd.isna(ratio): status = "Review - unclassified or incomplete"
|
| 217 |
+
elif ratio >= 1.0: status = "Exceeded"
|
| 218 |
+
elif ratio >= 0.90: status = "Near capacity"
|
| 219 |
+
else: status = "Pass"
|
| 220 |
+
classes.append(cls); statuses.append(status); margins.append(margin)
|
| 221 |
+
df.insert(1, "Calgary Storage Classification", classes)
|
| 222 |
+
df[f"Remaining Depth Margin ({length_unit})"] = margins
|
| 223 |
+
df["Calgary Status"] = statuses
|
| 224 |
+
return df
|
| 225 |
+
|
| 226 |
+
|
| 227 |
+
def criteria_register(criteria: CalgaryCriteria) -> pd.DataFrame:
|
| 228 |
+
rows = [
|
| 229 |
+
("CAL-GEN-001", "Source manual", criteria.source_manual, criteria.manual_status, "Verify current City amendments"),
|
| 230 |
+
("CAL-MIN-001", "Minor-system release rate", criteria.minor_release_rate_lps_ha, "Project-specific", "L/s/ha"),
|
| 231 |
+
("CAL-MAJ-001", "Street depth-velocity envelope", json.dumps(criteria.depth_velocity_curve), "Calgary/Alberta reference curve", "m/s vs m"),
|
| 232 |
+
("CAL-TRL-001", "Trap-low maximum ponding depth", criteria.trap_low_max_depth_m, "Configurable default", "m"),
|
| 233 |
+
("CAL-TRL-002", "Entrance grade margin", criteria.entrance_grade_margin_m, "Configurable default", "m"),
|
| 234 |
+
("CAL-MIN-002", "Conduit capacity review ratio", criteria.conduit_capacity_review_ratio, "Screening", "fraction"),
|
| 235 |
+
("CAL-MIN-003", "Conduit capacity warning ratio", criteria.conduit_capacity_warning_ratio, "Screening", "fraction"),
|
| 236 |
+
("CAL-QA-001", "Continuity review threshold", criteria.continuity_review_pct, "Screening", "%"),
|
| 237 |
+
("CAL-QA-002", "Continuity warning threshold", criteria.continuity_warning_pct, "Screening", "%"),
|
| 238 |
+
]
|
| 239 |
+
return pd.DataFrame(rows, columns=["Rule ID", "Criterion", "Value", "Rule Status", "Units / Note"])
|
| 240 |
+
|
| 241 |
+
|
| 242 |
+
def build_llm_report_context(*, metadata: dict[str, Any], criteria: CalgaryCriteria, findings: list[str], tables: dict[str, pd.DataFrame]) -> dict[str, Any]:
|
| 243 |
+
compact_tables = {}
|
| 244 |
+
for name, df in tables.items():
|
| 245 |
+
if df is not None and not df.empty:
|
| 246 |
+
compact_tables[name] = df.head(30).where(pd.notna(df), None).to_dict(orient="records")
|
| 247 |
+
return {
|
| 248 |
+
"task": "Draft a City of Calgary stormwater management report narrative for professional review.",
|
| 249 |
+
"metadata": metadata,
|
| 250 |
+
"approved_criteria": asdict(criteria),
|
| 251 |
+
"deterministic_findings": findings,
|
| 252 |
+
"verified_tables": compact_tables,
|
| 253 |
+
"interpretation_controls": {
|
| 254 |
+
"minor_system_primary_metric": "modelled depth ratio unless a calculated full-flow capacity is present",
|
| 255 |
+
"full_flow_capacity_available": bool("minor_system" in compact_tables and any(r.get("Full-Flow Capacity (m³/s)") is not None or r.get("Full-Flow Capacity (cfs)") is not None for r in compact_tables.get("minor_system", []))),
|
| 256 |
+
"generic_storage_freeboard_screening_allowed": False,
|
| 257 |
+
"storm_name_verified": False,
|
| 258 |
+
"model_input_output_overall_status": next((r.get("Status") for r in compact_tables.get("swmr_checklist", []) if r.get("Item") == "SWMR-17"), "Not assessed"),
|
| 259 |
+
"allowed_conclusions": {"adequate": False, "compliant": False, "safe": False, "effective": False, "approved": False},
|
| 260 |
+
},
|
| 261 |
+
"preferred_wording": {
|
| 262 |
+
"minor_system": "Use 'modelled depth ratio' when full-flow capacity is unavailable.",
|
| 263 |
+
"storage": "Use 'remaining depth margin' for maximum ponding-depth criteria.",
|
| 264 |
+
"event": "State that the design-event name is inferred when not independently verified.",
|
| 265 |
+
"outfalls": "Report modelled flows and state that downstream capacity requires confirmation.",
|
| 266 |
+
"model_documentation": "Distinguish digital package completeness from report appendix, schematic, drawing reconciliation, and authenticated-file completeness.",
|
| 267 |
+
},
|
| 268 |
+
"constraints": [
|
| 269 |
+
"Do not change or recalculate numerical results.",
|
| 270 |
+
"Do not claim City approval or professional certification.",
|
| 271 |
+
"Identify inferred, project-specific, historical, and unverified criteria.",
|
| 272 |
+
"Use 'not provided' where project facts are missing.",
|
| 273 |
+
"Distinguish pass, warning, information gap, and professional judgement.",
|
| 274 |
+
"Do not use adequate, compliant, safe, effective, acceptable, or approved unless deterministic verified criteria authorize that wording.",
|
| 275 |
+
"Use the Calgary SWMR completeness register and state all missing or partial mandatory items.",
|
| 276 |
+
"Never call modelled depth ratio capacity used unless calculated full-flow capacity is present.",
|
| 277 |
+
"Do not apply generic junction freeboard screening to storage nodes.",
|
| 278 |
+
],
|
| 279 |
+
}
|
| 280 |
+
|
| 281 |
+
|
| 282 |
+
CALGARY_LLM_SYSTEM_PROMPT = """You are a Calgary stormwater report drafting assistant.
|
| 283 |
+
Use only the verified structured data, deterministic findings, approved Project Criteria Register, and Calgary SWMR Completeness Register supplied by the application.
|
| 284 |
+
Never calculate or modify hydraulic values. Never invent project facts. Never describe a screening threshold as a current City requirement unless its rule status is verified.
|
| 285 |
+
|
| 286 |
+
ENGINEERING INTERPRETATION RULES
|
| 287 |
+
1. Do not describe the model, system, infrastructure, design, or results as adequate, compliant, safe, effective, acceptable, or approved unless the deterministic compliance engine explicitly returns that conclusion for the applicable verified criterion.
|
| 288 |
+
2. Do not apply generic junction freeboard or depth-ratio criteria to storage nodes. Use the Calgary storage classification, adopted maximum ponding depth, remaining depth margin, utilization, spill elevation, and Calgary status fields.
|
| 289 |
+
3. For active spill links, report the modeled peak flow, depth, velocity, and deterministic depth-velocity screening result. Also require confirmation of grading continuity, route containment, public safety, erosion, downstream impacts, and receiving-system capacity.
|
| 290 |
+
4. Do not state that outfalls manage flow effectively or have adequate receiving capacity. Report only modeled flow, depth, flooding, boundary type, and verified downstream criteria.
|
| 291 |
+
5. When full-flow pipe capacity is unavailable, do not describe conduit depth ratio as full capacity utilization, spare capacity, or hydraulic capacity. Call it modeled depth ratio.
|
| 292 |
+
6. Prioritize approved project-specific criteria, applicable MDP/SMDP/pond-report/prior-SWMR criteria, verified current City requirements, historical manual criteria, then user-defined screening thresholds.
|
| 293 |
+
7. Clearly distinguish deterministic results, screening results, project-specific requirements, historical reference criteria, inferred information, missing information, and professional judgement.
|
| 294 |
+
8. Do not invent project facts, capacities, material properties, downstream conditions, grading information, or compliance conclusions.
|
| 295 |
+
9. Where criteria are missing or unverified, use wording such as 'requires confirmation', 'screening only', 'not established from the available data', or 'cannot be concluded from the model results alone'.
|
| 296 |
+
10. The generated text is an engineering draft for professional review and must not be presented as final certification or municipal approval.
|
| 297 |
+
|
| 298 |
+
CALGARY SWMR CHECKLIST RULES
|
| 299 |
+
Use the completeness register when drafting. Do not omit a checklist subject merely because data are unavailable; state what is missing and what must be provided. For each material conclusion, identify supporting report tables, figures, model results, approved criteria, or source documents. Do not state that the SWMR is complete when mandatory items are Missing, Partially complete, or Require professional confirmation. Highlight unresolved issues, departures, missing project data, drawing gaps, and unverified downstream conditions in the executive summary and Outstanding Information and Actions section. Distinguish model-derived information from drawing-, survey-, project-, and professionally-confirmed information.
|
| 300 |
+
|
| 301 |
+
Draft clear consultant-quality narrative for professional review. Do not certify compliance or approval.
|
| 302 |
+
"""
|
calgary_rules.yaml
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
profile: City of Calgary SWMR
|
| 2 |
+
source_manual:
|
| 3 |
+
title: City of Calgary Stormwater Management & Design Manual
|
| 4 |
+
edition: 2011
|
| 5 |
+
status: historical_baseline_verify_current_amendments
|
| 6 |
+
criteria:
|
| 7 |
+
trap_low_max_depth_m: 0.50
|
| 8 |
+
entrance_grade_margin_m: 0.30
|
| 9 |
+
conduit_capacity_review_ratio: 0.80
|
| 10 |
+
conduit_capacity_warning_ratio: 0.95
|
| 11 |
+
pipe_advisory_velocity_mps: 3.0
|
| 12 |
+
pipe_critical_velocity_mps: 4.0
|
| 13 |
+
continuity_review_pct: 0.50
|
| 14 |
+
continuity_warning_pct: 1.00
|
| 15 |
+
depth_velocity_curve:
|
| 16 |
+
- [0.5, 0.80]
|
| 17 |
+
- [1.0, 0.32]
|
| 18 |
+
- [2.0, 0.21]
|
| 19 |
+
- [3.0, 0.09]
|
| 20 |
+
notes:
|
| 21 |
+
- Project-specific SMDP, MDP, pond report, City direction, and current amendments take precedence.
|
| 22 |
+
- Values marked as screening criteria are not automatic compliance limits.
|
model_pipeline.py
ADDED
|
@@ -0,0 +1,319 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Headless SWMM model pipeline: INP parsing and result-summary builders.
|
| 2 |
+
|
| 3 |
+
Extracted verbatim from the SWMM6 GIS Tool (Rev 23.2) Streamlit app so the
|
| 4 |
+
same deterministic logic serves the MCP/REST server without a Streamlit
|
| 5 |
+
dependency. Includes the Rev 23.2 fix resolving IRREGULAR-section full depth
|
| 6 |
+
from [TRANSECTS] GR data.
|
| 7 |
+
"""
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import pandas as pd
|
| 11 |
+
import numpy as np
|
| 12 |
+
|
| 13 |
+
def parse_inp_sections(inp_path):
|
| 14 |
+
sections = {}
|
| 15 |
+
current = None
|
| 16 |
+
with open(inp_path, encoding="utf-8", errors="ignore") as f:
|
| 17 |
+
for line in f:
|
| 18 |
+
line = line.strip()
|
| 19 |
+
if not line or line.startswith(";"):
|
| 20 |
+
continue
|
| 21 |
+
if line.startswith("["):
|
| 22 |
+
try:
|
| 23 |
+
current = line[1:line.index("]")]
|
| 24 |
+
sections[current] = []
|
| 25 |
+
except ValueError:
|
| 26 |
+
pass
|
| 27 |
+
elif current is not None:
|
| 28 |
+
sections[current].append(line.split())
|
| 29 |
+
return sections
|
| 30 |
+
|
| 31 |
+
def parse_node_types(sections):
|
| 32 |
+
types = {}
|
| 33 |
+
for sname, ntype in [("JUNCTIONS", "junction"), ("OUTFALLS", "outfall"),
|
| 34 |
+
("STORAGE", "storage"), ("DIVIDERS", "divider")]:
|
| 35 |
+
for row in sections.get(sname, []):
|
| 36 |
+
if row:
|
| 37 |
+
types[row[0]] = ntype
|
| 38 |
+
return types
|
| 39 |
+
|
| 40 |
+
def parse_link_topology(sections):
|
| 41 |
+
"""Return {link_id: (from_node, to_node, type)} dict."""
|
| 42 |
+
topo = {}
|
| 43 |
+
for sname, ltype in [("CONDUITS", "conduit"), ("PUMPS", "pump"),
|
| 44 |
+
("ORIFICES", "orifice"), ("WEIRS", "weir"), ("OUTLETS", "outlet")]:
|
| 45 |
+
for row in sections.get(sname, []):
|
| 46 |
+
if len(row) >= 3:
|
| 47 |
+
topo[row[0]] = (row[1], row[2], ltype)
|
| 48 |
+
return topo
|
| 49 |
+
|
| 50 |
+
def parse_conduit_geometry(sections):
|
| 51 |
+
"""Return {conduit_id: {length, roughness, xsect_params}} dict."""
|
| 52 |
+
geom = {}
|
| 53 |
+
for row in sections.get("CONDUITS", []):
|
| 54 |
+
if len(row) >= 6:
|
| 55 |
+
try:
|
| 56 |
+
geom[row[0]] = {"length": float(row[3]), "roughness": float(row[4])}
|
| 57 |
+
except ValueError:
|
| 58 |
+
pass
|
| 59 |
+
# Full depth of IRREGULAR sections comes from the referenced transect's
|
| 60 |
+
# GR rows (max station elevation - min station elevation), matching the
|
| 61 |
+
# engine's Cross Section Summary "Full Depth". Previously float() failed
|
| 62 |
+
# on the transect NAME in geom1 and the depth silently defaulted to
|
| 63 |
+
# 1.0 m downstream, distorting depth ratios for street/overland links.
|
| 64 |
+
transect_full_depth = {}
|
| 65 |
+
current_transect = None
|
| 66 |
+
for row in sections.get("TRANSECTS", []):
|
| 67 |
+
tag = str(row[0]).upper()
|
| 68 |
+
if tag == "X1" and len(row) >= 2:
|
| 69 |
+
current_transect = row[1]
|
| 70 |
+
transect_full_depth.setdefault(current_transect, [])
|
| 71 |
+
elif tag == "GR" and current_transect is not None:
|
| 72 |
+
# GR rows are (elev, station) pairs.
|
| 73 |
+
for i in range(1, len(row) - 1, 2):
|
| 74 |
+
try:
|
| 75 |
+
transect_full_depth[current_transect].append(float(row[i]))
|
| 76 |
+
except ValueError:
|
| 77 |
+
pass
|
| 78 |
+
transect_full_depth = {
|
| 79 |
+
name: (max(elevs) - min(elevs)) for name, elevs in transect_full_depth.items() if elevs
|
| 80 |
+
}
|
| 81 |
+
for row in sections.get("XSECTIONS", []):
|
| 82 |
+
if len(row) >= 3 and row[0] in geom:
|
| 83 |
+
geom[row[0]]["shape"] = row[1]
|
| 84 |
+
if str(row[1]).upper() == "IRREGULAR":
|
| 85 |
+
geom[row[0]]["transect"] = row[2]
|
| 86 |
+
full = transect_full_depth.get(row[2])
|
| 87 |
+
if full and full > 0:
|
| 88 |
+
geom[row[0]]["diameter"] = full
|
| 89 |
+
else:
|
| 90 |
+
try:
|
| 91 |
+
geom[row[0]]["diameter"] = float(row[2])
|
| 92 |
+
except (ValueError, IndexError):
|
| 93 |
+
pass
|
| 94 |
+
return geom
|
| 95 |
+
|
| 96 |
+
def parse_subcatchment_attrs(sections):
|
| 97 |
+
attrs = {}
|
| 98 |
+
for row in sections.get("SUBCATCHMENTS", []):
|
| 99 |
+
if len(row) >= 6:
|
| 100 |
+
try:
|
| 101 |
+
attrs[row[0]] = {
|
| 102 |
+
"gage": row[1],
|
| 103 |
+
"outlet": row[2],
|
| 104 |
+
"area": float(row[3]),
|
| 105 |
+
"pct_imp": float(row[4]),
|
| 106 |
+
"width": float(row[5]) if len(row) > 5 else 0.0,
|
| 107 |
+
"slope": float(row[6]) if len(row) > 6 else 0.0,
|
| 108 |
+
}
|
| 109 |
+
except (ValueError, IndexError):
|
| 110 |
+
pass
|
| 111 |
+
return attrs
|
| 112 |
+
|
| 113 |
+
def parse_gis(sections):
|
| 114 |
+
"""Extract node coords, link vertices, sub polygons from INP."""
|
| 115 |
+
dims = get_map_dimensions(sections)
|
| 116 |
+
|
| 117 |
+
# Node coordinates
|
| 118 |
+
raw_coords = {}
|
| 119 |
+
for row in sections.get("COORDINATES", []):
|
| 120 |
+
if len(row) >= 3:
|
| 121 |
+
try:
|
| 122 |
+
raw_coords[row[0]] = (float(row[1]), float(row[2]))
|
| 123 |
+
except ValueError:
|
| 124 |
+
pass
|
| 125 |
+
|
| 126 |
+
node_coords = normalize_coords(raw_coords, dims)
|
| 127 |
+
|
| 128 |
+
# Link vertices
|
| 129 |
+
raw_verts = {}
|
| 130 |
+
for row in sections.get("VERTICES", []):
|
| 131 |
+
if len(row) >= 3:
|
| 132 |
+
try:
|
| 133 |
+
raw_verts.setdefault(row[0], []).append((float(row[1]), float(row[2])))
|
| 134 |
+
except ValueError:
|
| 135 |
+
pass
|
| 136 |
+
|
| 137 |
+
# Normalize vertices using same scale
|
| 138 |
+
if raw_coords and dims:
|
| 139 |
+
xmin, ymin, xmax, ymax = dims
|
| 140 |
+
cx = (xmin + xmax) / 2
|
| 141 |
+
cy = (ymin + ymax) / 2
|
| 142 |
+
rx = max(xmax - xmin, 1e-9)
|
| 143 |
+
ry = max(ymax - ymin, 1e-9)
|
| 144 |
+
scale = 0.005 / max(rx, ry)
|
| 145 |
+
elif raw_coords:
|
| 146 |
+
xs = [v[0] for v in raw_coords.values()]
|
| 147 |
+
ys = [v[1] for v in raw_coords.values()]
|
| 148 |
+
cx = (min(xs) + max(xs)) / 2
|
| 149 |
+
cy = (min(ys) + max(ys)) / 2
|
| 150 |
+
scale = 0.005 / max(max(xs) - min(xs), max(ys) - min(ys), 1e-9)
|
| 151 |
+
else:
|
| 152 |
+
cx, cy, scale = 0, 0, 1
|
| 153 |
+
|
| 154 |
+
link_vertices = {}
|
| 155 |
+
for lid, verts in raw_verts.items():
|
| 156 |
+
link_vertices[lid] = [((x - cx) * scale, (y - cy) * scale) for x, y in verts]
|
| 157 |
+
|
| 158 |
+
# Subcatchment polygons
|
| 159 |
+
raw_polys = {}
|
| 160 |
+
for row in sections.get("Polygons", []):
|
| 161 |
+
if len(row) >= 3:
|
| 162 |
+
try:
|
| 163 |
+
raw_polys.setdefault(row[0], []).append((float(row[1]), float(row[2])))
|
| 164 |
+
except ValueError:
|
| 165 |
+
pass
|
| 166 |
+
|
| 167 |
+
sub_polygons = {}
|
| 168 |
+
for sid, pts in raw_polys.items():
|
| 169 |
+
sub_polygons[sid] = [((x - cx) * scale, (y - cy) * scale) for x, y in pts]
|
| 170 |
+
|
| 171 |
+
return node_coords, link_vertices, sub_polygons
|
| 172 |
+
|
| 173 |
+
def build_node_summary(node_ts, node_types, flood_thresh, depth_ratio_thresh):
|
| 174 |
+
rows = []
|
| 175 |
+
for nid, d in node_ts.items():
|
| 176 |
+
depths = d.get("depth", [0])
|
| 177 |
+
floods = d.get("flooding", [0])
|
| 178 |
+
inflows = d.get("inflow", [0])
|
| 179 |
+
invert = d.get("invert_elevation", 0)
|
| 180 |
+
full_d = d.get("full_depth", 1) or 1
|
| 181 |
+
|
| 182 |
+
pk_depth = max(depths) if depths else 0
|
| 183 |
+
pk_flood = max(floods) if floods else 0
|
| 184 |
+
pk_inflow = max(inflows) if inflows else 0
|
| 185 |
+
depth_ratio = pk_depth / full_d
|
| 186 |
+
|
| 187 |
+
if pk_flood > flood_thresh:
|
| 188 |
+
status = "🚨 Flooded"
|
| 189 |
+
elif depth_ratio > depth_ratio_thresh:
|
| 190 |
+
status = "⚠️ Near Capacity"
|
| 191 |
+
else:
|
| 192 |
+
status = "✅ OK"
|
| 193 |
+
|
| 194 |
+
rows.append({
|
| 195 |
+
"Node ID": nid,
|
| 196 |
+
"Type": node_types.get(nid, "junction"),
|
| 197 |
+
"Invert (m)": round(invert, 3),
|
| 198 |
+
"Full Depth (m)": round(full_d, 3),
|
| 199 |
+
"Peak Depth (m)": round(pk_depth, 4),
|
| 200 |
+
"Depth Ratio": round(depth_ratio, 3),
|
| 201 |
+
"Peak Flooding (m³/s)": round(pk_flood, 6),
|
| 202 |
+
"Peak Inflow (m³/s)": round(pk_inflow, 6),
|
| 203 |
+
"Status": status,
|
| 204 |
+
})
|
| 205 |
+
return pd.DataFrame(rows)
|
| 206 |
+
|
| 207 |
+
def build_link_summary(link_ts, link_topo, conduit_geom, depth_ratio_thresh, vel_thresh):
|
| 208 |
+
rows = []
|
| 209 |
+
for lid, d in link_ts.items():
|
| 210 |
+
flows = d.get("flow", [0])
|
| 211 |
+
depths = d.get("depth", [0])
|
| 212 |
+
velocities = d.get("velocity", [0])
|
| 213 |
+
topo = link_topo.get(lid, ("?", "?", "conduit"))
|
| 214 |
+
geom = conduit_geom.get(lid, {})
|
| 215 |
+
|
| 216 |
+
pk_flow = max(flows) if flows else 0
|
| 217 |
+
pk_depth = max(depths) if depths else 0
|
| 218 |
+
pk_velocity = max((abs(v) for v in velocities), default=0)
|
| 219 |
+
diam = geom.get("diameter", 1) or 1
|
| 220 |
+
length = geom.get("length", 0)
|
| 221 |
+
depth_ratio = pk_depth / diam
|
| 222 |
+
|
| 223 |
+
if depth_ratio >= 1.0:
|
| 224 |
+
status = "Pressurised"
|
| 225 |
+
elif depth_ratio > depth_ratio_thresh:
|
| 226 |
+
status = "Surcharging"
|
| 227 |
+
elif pk_velocity > vel_thresh:
|
| 228 |
+
status = "High Velocity"
|
| 229 |
+
elif depth_ratio > 0.5:
|
| 230 |
+
status = "Filling"
|
| 231 |
+
else:
|
| 232 |
+
status = "Free-flow"
|
| 233 |
+
|
| 234 |
+
rows.append({
|
| 235 |
+
"Link ID": lid,
|
| 236 |
+
"Type": topo[2],
|
| 237 |
+
"From Node": topo[0],
|
| 238 |
+
"To Node": topo[1],
|
| 239 |
+
"Length (m)": round(length, 1),
|
| 240 |
+
"Diameter (m)": round(diam, 3),
|
| 241 |
+
"Peak Flow (m³/s)": round(pk_flow, 6),
|
| 242 |
+
"Peak Depth (m)": round(pk_depth, 4),
|
| 243 |
+
"Depth Ratio": round(depth_ratio, 3),
|
| 244 |
+
"Peak Velocity (m/s)": round(pk_velocity, 3),
|
| 245 |
+
"Status": status,
|
| 246 |
+
})
|
| 247 |
+
return pd.DataFrame(rows)
|
| 248 |
+
|
| 249 |
+
def build_sub_summary(sub_ts, sub_attrs, times=None, flow_units="CMS"):
|
| 250 |
+
"""Build subcatchment summary with time-integrated runoff volume.
|
| 251 |
+
|
| 252 |
+
Values remain in the SWMM model unit system. The legacy internal column name
|
| 253 |
+
``Total Runoff (m³)`` is retained for database compatibility, but its value is
|
| 254 |
+
flow integrated over time in the native flow-volume basis (e.g., ft³ for CFS,
|
| 255 |
+
m³ for CMS, litres for LPS). The report engine assigns the correct label.
|
| 256 |
+
"""
|
| 257 |
+
rows = []
|
| 258 |
+
flow_units = str(flow_units or "CMS").upper()
|
| 259 |
+
|
| 260 |
+
def _integrate(values, timestamps):
|
| 261 |
+
if not values:
|
| 262 |
+
return 0.0
|
| 263 |
+
if timestamps and len(timestamps) == len(values) and len(values) > 1:
|
| 264 |
+
total = 0.0
|
| 265 |
+
for i in range(1, len(values)):
|
| 266 |
+
try:
|
| 267 |
+
dt = (timestamps[i] - timestamps[i - 1]).total_seconds()
|
| 268 |
+
except Exception:
|
| 269 |
+
dt = 0.0
|
| 270 |
+
if dt > 0:
|
| 271 |
+
total += 0.5 * (float(values[i - 1]) + float(values[i])) * dt
|
| 272 |
+
return total
|
| 273 |
+
return float(sum(values))
|
| 274 |
+
|
| 275 |
+
def _rain_depth(values, timestamps):
|
| 276 |
+
# Rainfall is an intensity (in/hr for US, mm/hr for SI).
|
| 277 |
+
return _integrate(values, timestamps) / 3600.0
|
| 278 |
+
|
| 279 |
+
for sid, d in sub_ts.items():
|
| 280 |
+
runoffs = d.get("runoff", [0])
|
| 281 |
+
rainfalls = d.get("rainfall", [0])
|
| 282 |
+
attrs = sub_attrs.get(sid, {})
|
| 283 |
+
area = d.get("area", attrs.get("area", 0))
|
| 284 |
+
pct_imp = d.get("pct_imp", attrs.get("pct_imp", 0))
|
| 285 |
+
|
| 286 |
+
pk_runoff = max(runoffs) if runoffs else 0
|
| 287 |
+
pk_rainfall = max(rainfalls) if rainfalls else 0
|
| 288 |
+
integrated_flow_seconds = _integrate(runoffs, times)
|
| 289 |
+
total_rain_depth = _rain_depth(rainfalls, times)
|
| 290 |
+
|
| 291 |
+
# Convert integrated native flow to the native report volume basis.
|
| 292 |
+
if flow_units == "CFS":
|
| 293 |
+
total_runoff_vol = integrated_flow_seconds # ft³
|
| 294 |
+
runoff_depth = (total_runoff_vol / (area * 43560.0) * 12.0) if area > 0 else 0.0
|
| 295 |
+
elif flow_units == "CMS":
|
| 296 |
+
total_runoff_vol = integrated_flow_seconds # m³
|
| 297 |
+
runoff_depth = (total_runoff_vol / (area * 10000.0) * 1000.0) if area > 0 else 0.0
|
| 298 |
+
elif flow_units == "LPS":
|
| 299 |
+
total_runoff_vol = integrated_flow_seconds # litres
|
| 300 |
+
runoff_depth = ((total_runoff_vol / 1000.0) / (area * 10000.0) * 1000.0) if area > 0 else 0.0
|
| 301 |
+
else:
|
| 302 |
+
total_runoff_vol = integrated_flow_seconds
|
| 303 |
+
runoff_depth = 0.0
|
| 304 |
+
|
| 305 |
+
rc = (runoff_depth / total_rain_depth) if total_rain_depth > 0 else 0.0
|
| 306 |
+
|
| 307 |
+
rows.append({
|
| 308 |
+
"Sub ID": sid,
|
| 309 |
+
"Area (ha)": round(area, 3),
|
| 310 |
+
"% Impervious": round(pct_imp, 1),
|
| 311 |
+
"Connected To": attrs.get("outlet", "?"),
|
| 312 |
+
"Peak Runoff (m³/s)": round(pk_runoff, 6),
|
| 313 |
+
"Total Runoff (m³)": round(total_runoff_vol, 3),
|
| 314 |
+
"Peak Rainfall (mm/h)": round(pk_rainfall, 4),
|
| 315 |
+
"Total Rainfall Depth": round(total_rain_depth, 4),
|
| 316 |
+
"Runoff Depth": round(runoff_depth, 4),
|
| 317 |
+
"Runoff Coefficient": round(rc, 3),
|
| 318 |
+
})
|
| 319 |
+
return pd.DataFrame(rows)
|
preliminary_design_assistant.py
ADDED
|
@@ -0,0 +1,370 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Rev23 preliminary design review and controlled-correction workflow.
|
| 2 |
+
|
| 3 |
+
Deterministic code identifies traceable model findings and applies only
|
| 4 |
+
engineer-approved edits. The LLM explains and prioritises findings; it does not
|
| 5 |
+
silently alter the model or determine municipal compliance.
|
| 6 |
+
"""
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
from dataclasses import asdict, dataclass, field
|
| 10 |
+
from datetime import datetime
|
| 11 |
+
from typing import Any, Mapping, Sequence
|
| 12 |
+
import hashlib
|
| 13 |
+
import json
|
| 14 |
+
import re
|
| 15 |
+
|
| 16 |
+
import pandas as pd
|
| 17 |
+
|
| 18 |
+
from ai_report_assistant import PROVIDERS, _call_provider, load_prompt
|
| 19 |
+
from scenario_manager import _split_sections, _join_sections, _data_tokens, _replace_tokens, _set_option
|
| 20 |
+
|
| 21 |
+
REVIEW_SYSTEM_PROMPT = load_prompt("preliminary_design_review_system.txt")
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
@dataclass
|
| 25 |
+
class DesignFinding:
|
| 26 |
+
finding_id: str
|
| 27 |
+
category: str
|
| 28 |
+
severity: str
|
| 29 |
+
finding_type: str
|
| 30 |
+
object_type: str = "MODEL"
|
| 31 |
+
object_id: str = "MODEL"
|
| 32 |
+
rule_id: str = ""
|
| 33 |
+
criterion_status: str = "Screening"
|
| 34 |
+
deterministic_basis: str = ""
|
| 35 |
+
recommended_action: str = ""
|
| 36 |
+
proposed_parameter: str = ""
|
| 37 |
+
proposed_value: Any = None
|
| 38 |
+
units: str = ""
|
| 39 |
+
engineer_decision: str = "Defer"
|
| 40 |
+
resolution_status: str = "Open"
|
| 41 |
+
reviewer_comment: str = ""
|
| 42 |
+
ai_explanation: str = ""
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def _rows(sections: Mapping[str, Sequence[str]], name: str) -> list[list[str]]:
|
| 46 |
+
out: list[list[str]] = []
|
| 47 |
+
for line in sections.get(name, []):
|
| 48 |
+
parsed = _data_tokens(line)
|
| 49 |
+
if parsed:
|
| 50 |
+
out.append(parsed[0])
|
| 51 |
+
return out
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def _ids(sections: Mapping[str, Sequence[str]], names: Sequence[str]) -> set[str]:
|
| 55 |
+
values: set[str] = set()
|
| 56 |
+
for name in names:
|
| 57 |
+
values.update(row[0] for row in _rows(sections, name) if row)
|
| 58 |
+
return values
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def _first_numeric(row: Mapping[str, Any], aliases: Sequence[str]) -> float | None:
|
| 62 |
+
"""Return the first explicitly available numeric value without converting missing data to zero."""
|
| 63 |
+
for key in aliases:
|
| 64 |
+
if key not in row:
|
| 65 |
+
continue
|
| 66 |
+
value = row.get(key)
|
| 67 |
+
if value is None or value == "":
|
| 68 |
+
continue
|
| 69 |
+
parsed = pd.to_numeric(pd.Series([value]), errors="coerce").iloc[0]
|
| 70 |
+
if pd.notna(parsed):
|
| 71 |
+
return float(parsed)
|
| 72 |
+
return None
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def _records(df: pd.DataFrame | None, limit: int = 500) -> list[dict[str, Any]]:
|
| 76 |
+
if df is None or df.empty:
|
| 77 |
+
return []
|
| 78 |
+
clean = df.head(limit).copy()
|
| 79 |
+
clean = clean.where(pd.notna(clean), None)
|
| 80 |
+
return clean.to_dict(orient="records")
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def build_deterministic_findings(
|
| 84 |
+
*,
|
| 85 |
+
inp_text: str,
|
| 86 |
+
node_summary: pd.DataFrame | None = None,
|
| 87 |
+
link_summary: pd.DataFrame | None = None,
|
| 88 |
+
sub_summary: pd.DataFrame | None = None,
|
| 89 |
+
metadata: Mapping[str, Any] | None = None,
|
| 90 |
+
simulation_completed: bool = False,
|
| 91 |
+
output_results_available: bool = False,
|
| 92 |
+
flood_threshold: float = 0.0,
|
| 93 |
+
depth_ratio_threshold: float = 0.85,
|
| 94 |
+
velocity_threshold: float = 3.0,
|
| 95 |
+
) -> list[dict[str, Any]]:
|
| 96 |
+
"""Create a conservative, traceable preliminary-review register."""
|
| 97 |
+
_, sections = _split_sections(inp_text)
|
| 98 |
+
findings: list[DesignFinding] = []
|
| 99 |
+
counter = 1
|
| 100 |
+
|
| 101 |
+
def add(category: str, severity: str, finding_type: str, basis: str, action: str,
|
| 102 |
+
object_type: str = "MODEL", object_id: str = "MODEL", rule_id: str = "",
|
| 103 |
+
proposed_parameter: str = "", proposed_value: Any = None, units: str = "",
|
| 104 |
+
criterion_status: str = "Screening") -> None:
|
| 105 |
+
nonlocal counter
|
| 106 |
+
findings.append(DesignFinding(
|
| 107 |
+
finding_id=f"PDA-{counter:03d}", category=category, severity=severity,
|
| 108 |
+
finding_type=finding_type, object_type=object_type, object_id=str(object_id),
|
| 109 |
+
rule_id=rule_id, criterion_status=criterion_status,
|
| 110 |
+
deterministic_basis=basis, recommended_action=action,
|
| 111 |
+
proposed_parameter=proposed_parameter, proposed_value=proposed_value, units=units,
|
| 112 |
+
))
|
| 113 |
+
counter += 1
|
| 114 |
+
|
| 115 |
+
# Completeness and topology.
|
| 116 |
+
node_ids = _ids(sections, ["JUNCTIONS", "STORAGE", "OUTFALLS", "DIVIDERS"])
|
| 117 |
+
link_rows = []
|
| 118 |
+
for sec in ["CONDUITS", "PUMPS", "ORIFICES", "WEIRS", "OUTLETS"]:
|
| 119 |
+
for row in _rows(sections, sec):
|
| 120 |
+
if len(row) >= 3:
|
| 121 |
+
link_rows.append((sec, row[0], row[1], row[2]))
|
| 122 |
+
referenced_nodes = {x for _, _, a, b in link_rows for x in (a, b)}
|
| 123 |
+
# SWMM matches object IDs case-insensitively, so an endpoint that
|
| 124 |
+
# differs from a defined node only by letter case still routes in the
|
| 125 |
+
# engine. Treat exact-case misses with a case-insensitive fallback:
|
| 126 |
+
# genuine misses stay Critical; case-only mismatches are a Medium
|
| 127 |
+
# naming-consistency finding (they break exact-match post-processing
|
| 128 |
+
# and GIS joins even though the simulation runs).
|
| 129 |
+
node_ids_casefold = {str(n).casefold(): n for n in node_ids}
|
| 130 |
+
for nid in sorted(referenced_nodes - node_ids):
|
| 131 |
+
matched = node_ids_casefold.get(str(nid).casefold())
|
| 132 |
+
if matched is not None:
|
| 133 |
+
add("Topology", "Medium", "ID case inconsistency",
|
| 134 |
+
f"Link endpoint '{nid}' matches defined node '{matched}' only when letter case is ignored. "
|
| 135 |
+
"SWMM resolves the connection, but exact-match tools (GIS joins, scripts, result queries) will not.",
|
| 136 |
+
f"Rename the link endpoint or the node so both use '{matched}' consistently.", "NODE", nid)
|
| 137 |
+
else:
|
| 138 |
+
add("Topology", "Critical", "Invalid reference", f"Node '{nid}' is referenced by a link but is not defined.",
|
| 139 |
+
"Define the node or correct the link endpoint before design use.", "NODE", nid)
|
| 140 |
+
|
| 141 |
+
connected = referenced_nodes
|
| 142 |
+
for nid in sorted(node_ids - connected):
|
| 143 |
+
# Outfalls/storage-only models can legitimately have simple topology, so advisory.
|
| 144 |
+
add("Topology", "Advisory", "Connectivity review", f"Node '{nid}' is not referenced by a hydraulic link.",
|
| 145 |
+
"Confirm whether the node is intentionally isolated or the model connection is incomplete.", "NODE", nid)
|
| 146 |
+
|
| 147 |
+
sub_rows = _rows(sections, "SUBCATCHMENTS")
|
| 148 |
+
sub_names = {r[0] for r in sub_rows}
|
| 149 |
+
sub_names_casefold = {str(s).casefold(): s for s in sub_names}
|
| 150 |
+
for row in sub_rows:
|
| 151 |
+
if len(row) >= 3 and row[2] not in node_ids and row[2] not in sub_names:
|
| 152 |
+
outlet = str(row[2])
|
| 153 |
+
matched = node_ids_casefold.get(outlet.casefold()) or sub_names_casefold.get(outlet.casefold())
|
| 154 |
+
if matched is not None:
|
| 155 |
+
add("Hydrology", "Medium", "ID case inconsistency",
|
| 156 |
+
f"Subcatchment '{row[0]}' outlet '{outlet}' matches defined object '{matched}' only when letter case is ignored. "
|
| 157 |
+
"SWMM resolves the routing, but exact-match tools will not.",
|
| 158 |
+
f"Rename the outlet reference or the object so both use '{matched}' consistently.", "SUBCATCHMENT", row[0])
|
| 159 |
+
else:
|
| 160 |
+
add("Hydrology", "Critical", "Invalid outlet", f"Subcatchment '{row[0]}' routes to undefined outlet '{outlet}'.",
|
| 161 |
+
"Correct the outlet reference.", "SUBCATCHMENT", row[0])
|
| 162 |
+
if len(row) >= 7:
|
| 163 |
+
try:
|
| 164 |
+
area, imperv, width, slope = float(row[3]), float(row[4]), float(row[5]), float(row[6])
|
| 165 |
+
if area <= 0:
|
| 166 |
+
add("Hydrology", "High", "Input review", f"Area is {area}.", "Enter a positive drainage area.", "SUBCATCHMENT", row[0], proposed_parameter="area")
|
| 167 |
+
if imperv < 0 or imperv > 100:
|
| 168 |
+
add("Hydrology", "Critical", "Input range", f"Imperviousness is {imperv}%.", "Set imperviousness within 0–100%.", "SUBCATCHMENT", row[0], proposed_parameter="imperviousness", units="%")
|
| 169 |
+
elif imperv >= 95:
|
| 170 |
+
add("Hydrology", "Moderate", "Sensitivity review", f"Imperviousness is {imperv}%.", "Confirm land-use basis and test sensitivity.", "SUBCATCHMENT", row[0])
|
| 171 |
+
if width <= 0:
|
| 172 |
+
add("Hydrology", "High", "Input review", f"Width is {width}.", "Enter and document a representative subcatchment width.", "SUBCATCHMENT", row[0], proposed_parameter="width")
|
| 173 |
+
if slope <= 0:
|
| 174 |
+
add("Hydrology", "High", "Input review", f"Slope is {slope}%.", "Confirm grading and enter a positive slope.", "SUBCATCHMENT", row[0], proposed_parameter="slope", units="%")
|
| 175 |
+
except Exception:
|
| 176 |
+
pass
|
| 177 |
+
|
| 178 |
+
# Options and rainfall.
|
| 179 |
+
options = {r[0].upper(): r[1] for r in _rows(sections, "OPTIONS") if len(r) >= 2}
|
| 180 |
+
if "FLOW_UNITS" not in options:
|
| 181 |
+
add("Simulation setup", "High", "Missing option", "FLOW_UNITS is not explicitly defined.", "Define and verify the model unit system.")
|
| 182 |
+
report_step = options.get("REPORT_STEP", "")
|
| 183 |
+
wet_step = options.get("WET_STEP", "")
|
| 184 |
+
if not report_step:
|
| 185 |
+
add("Simulation setup", "Moderate", "Missing option", "REPORT_STEP is not explicitly defined.", "Set a reporting timestep appropriate for the event and control response.", proposed_parameter="REPORT_STEP")
|
| 186 |
+
if not wet_step:
|
| 187 |
+
add("Simulation setup", "Moderate", "Missing option", "WET_STEP is not explicitly defined.", "Set and document the wet-weather timestep.", proposed_parameter="WET_STEP")
|
| 188 |
+
rain_gages = _rows(sections, "RAINGAGES")
|
| 189 |
+
time_series = _rows(sections, "TIMESERIES")
|
| 190 |
+
if not rain_gages:
|
| 191 |
+
add("Rainfall", "Critical", "Missing rainfall", "No [RAINGAGES] records were identified.", "Add and verify the applicable design rainfall.")
|
| 192 |
+
if rain_gages and not time_series:
|
| 193 |
+
add("Rainfall", "High", "Rainfall source review", "Rain gages exist but no internal [TIMESERIES] records were identified.", "Confirm external rainfall files and package them with the model.")
|
| 194 |
+
|
| 195 |
+
# Deterministic result screening. Result-based findings are permitted only
|
| 196 |
+
# when a successful simulation and parsed output tables are explicitly available.
|
| 197 |
+
results_ready = bool(simulation_completed and output_results_available)
|
| 198 |
+
if results_ready and node_summary is not None and not node_summary.empty:
|
| 199 |
+
for _, r in node_summary.iterrows():
|
| 200 |
+
row = r.to_dict()
|
| 201 |
+
nid = str(row.get("Node ID", row.get("ID", "")))
|
| 202 |
+
flooding = _first_numeric(row, ["Peak Flooding (m³/s)", "Peak Flooding", "Max Flooding", "Maximum Flooding"] )
|
| 203 |
+
if flooding is not None and flooding > flood_threshold:
|
| 204 |
+
add("Hydraulics", "Critical", "Flooding", f"Peak model flooding is {flooding:.6g}.", "Review HGL, rim elevation, downstream boundary, and design alternatives.", "NODE", nid, "CAL-HYD-FLOOD", units=str(metadata.get("flow_units", "") if metadata else ""))
|
| 205 |
+
|
| 206 |
+
if results_ready and link_summary is not None and not link_summary.empty:
|
| 207 |
+
for _, r in link_summary.iterrows():
|
| 208 |
+
row = r.to_dict()
|
| 209 |
+
lid = str(row.get("Link ID", row.get("ID", "")))
|
| 210 |
+
vel = _first_numeric(row, ["Peak Velocity (m/s)", "Max Velocity", "Maximum Velocity", "Velocity"] )
|
| 211 |
+
ratio = _first_numeric(row, ["Depth Ratio", "Max/Full Depth", "Maximum Depth Ratio"] )
|
| 212 |
+
if vel is not None and vel > velocity_threshold:
|
| 213 |
+
add("Hydraulics", "High", "Velocity screening", f"Maximum modelled velocity is {vel:.4g}, above the configured {velocity_threshold:g} screening value.", "Confirm pipe/channel material, erosion protection, energy dissipation, and applicable criterion.", "LINK", lid, "CAL-HYD-VEL", units="model units")
|
| 214 |
+
if ratio is not None and ratio >= depth_ratio_threshold:
|
| 215 |
+
add("Hydraulics", "High", "Depth-ratio screening", f"Maximum modelled depth ratio is {ratio:.4g}, at or above the configured {depth_ratio_threshold:g} screening value.", "Review capacity, HGL, surcharge duration, and downstream boundary conditions.", "LINK", lid, "CAL-HYD-DEPTH")
|
| 216 |
+
|
| 217 |
+
if results_ready and sub_summary is not None and not sub_summary.empty:
|
| 218 |
+
zero_runoff_tolerance = 1e-9
|
| 219 |
+
for _, r in sub_summary.iterrows():
|
| 220 |
+
row = r.to_dict()
|
| 221 |
+
sid = str(row.get("Sub ID", row.get("Subcatchment", row.get("ID", ""))))
|
| 222 |
+
runoff = _first_numeric(row, ["Peak Runoff (m³/s)", "Peak Runoff", "Peak Runoff (cfs)", "Peak Runoff (L/s)"] )
|
| 223 |
+
if runoff is not None and abs(runoff) <= zero_runoff_tolerance:
|
| 224 |
+
add("Hydrology", "Moderate", "Zero runoff", "Peak runoff is explicitly reported as zero for the completed simulation.", "Confirm rainfall assignment, infiltration, routing, and subcatchment activation.", "SUBCATCHMENT", sid)
|
| 225 |
+
|
| 226 |
+
# Missing design criteria are never silently replaced by generic standards.
|
| 227 |
+
add("Criteria", "High", "Missing project criterion", "Project-specific allowable release rate is not established by the model alone.", "Enter the approved release criterion and source before evaluating outlet compliance.", rule_id="CAL-MIN-001", criterion_status="Not established")
|
| 228 |
+
if _rows(sections, "STORAGE"):
|
| 229 |
+
add("Storage", "High", "Missing project criterion", "Approved pond/storage HWL, freeboard, emergency spill elevation, and classification are not established by the model alone.", "Enter the approved storage criteria and source before concluding performance.", rule_id="CAL-POND-001", criterion_status="Not established")
|
| 230 |
+
|
| 231 |
+
return [asdict(x) for x in findings]
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
def findings_dataframe(findings: Sequence[Mapping[str, Any]]) -> pd.DataFrame:
|
| 235 |
+
columns = [f.name for f in DesignFinding.__dataclass_fields__.values()]
|
| 236 |
+
df = pd.DataFrame(list(findings))
|
| 237 |
+
for col in columns:
|
| 238 |
+
if col not in df.columns:
|
| 239 |
+
df[col] = ""
|
| 240 |
+
return df[columns]
|
| 241 |
+
|
| 242 |
+
|
| 243 |
+
def build_review_context(
|
| 244 |
+
*,
|
| 245 |
+
inp_name: str,
|
| 246 |
+
inp_text: str,
|
| 247 |
+
findings: Sequence[Mapping[str, Any]],
|
| 248 |
+
metadata: Mapping[str, Any],
|
| 249 |
+
criteria: Mapping[str, Any] | None = None,
|
| 250 |
+
node_summary: pd.DataFrame | None = None,
|
| 251 |
+
link_summary: pd.DataFrame | None = None,
|
| 252 |
+
sub_summary: pd.DataFrame | None = None,
|
| 253 |
+
simulation_completed: bool = False,
|
| 254 |
+
output_results_available: bool = False,
|
| 255 |
+
review_mode: str = "Input and simulation-output review",
|
| 256 |
+
) -> dict[str, Any]:
|
| 257 |
+
_, sections = _split_sections(inp_text)
|
| 258 |
+
output_ready = bool(simulation_completed and output_results_available)
|
| 259 |
+
return {
|
| 260 |
+
"review_mode": review_mode,
|
| 261 |
+
"availability": {
|
| 262 |
+
"model_input_available": bool(inp_text),
|
| 263 |
+
"simulation_completed": bool(simulation_completed),
|
| 264 |
+
"output_results_available": bool(output_results_available),
|
| 265 |
+
"result_based_review_authorized": output_ready,
|
| 266 |
+
},
|
| 267 |
+
"model": {
|
| 268 |
+
"name": inp_name,
|
| 269 |
+
"sha256": hashlib.sha256(inp_text.encode("utf-8")).hexdigest(),
|
| 270 |
+
"section_counts": {k: len(_rows(sections, k)) for k in sections},
|
| 271 |
+
},
|
| 272 |
+
"metadata": dict(metadata or {}),
|
| 273 |
+
"criteria": dict(criteria or {}),
|
| 274 |
+
"simulation_results": {
|
| 275 |
+
"node_summary": _records(node_summary) if output_ready else [],
|
| 276 |
+
"link_summary": _records(link_summary) if output_ready else [],
|
| 277 |
+
"subcatchment_summary": _records(sub_summary) if output_ready else [],
|
| 278 |
+
},
|
| 279 |
+
"result_table_counts": {
|
| 280 |
+
"nodes": 0 if node_summary is None else int(len(node_summary)),
|
| 281 |
+
"links": 0 if link_summary is None else int(len(link_summary)),
|
| 282 |
+
"subcatchments": 0 if sub_summary is None else int(len(sub_summary)),
|
| 283 |
+
},
|
| 284 |
+
"deterministic_findings": list(findings),
|
| 285 |
+
"allowed_ai_actions": ["explain", "prioritise", "identify missing information", "propose review steps", "propose structured edits for engineer approval"],
|
| 286 |
+
"prohibited_ai_actions": ["claim compliance", "silently modify the model", "invent criteria", "certify design", "approve a preferred design", "convert missing values to zero"],
|
| 287 |
+
}
|
| 288 |
+
|
| 289 |
+
|
| 290 |
+
def ai_review_findings(*, provider_name: str, api_key: str, model: str, review_context: Mapping[str, Any], user_request: str = "") -> str:
|
| 291 |
+
request = f"""Review the preliminary SWMM model using only the deterministic context below.
|
| 292 |
+
Prioritise issues, explain likely engineering implications, identify missing criteria,
|
| 293 |
+
and propose conservative next actions. Do not recalculate results or claim compliance.
|
| 294 |
+
|
| 295 |
+
USER_REQUEST:
|
| 296 |
+
{user_request or 'Provide a complete preliminary design review.'}
|
| 297 |
+
|
| 298 |
+
DETERMINISTIC_CONTEXT:
|
| 299 |
+
{json.dumps(review_context, ensure_ascii=False, default=str)}
|
| 300 |
+
|
| 301 |
+
Return concise professional Markdown with these headings:
|
| 302 |
+
1. Review summary
|
| 303 |
+
2. Priority findings
|
| 304 |
+
3. Modelling corrections for engineer consideration
|
| 305 |
+
4. Design-criteria confirmations required
|
| 306 |
+
5. Recommended path to scenario analysis
|
| 307 |
+
"""
|
| 308 |
+
return _call_provider(provider_name, api_key, model, [{"role": "user", "content": request}], REVIEW_SYSTEM_PROMPT, max_tokens=3600)
|
| 309 |
+
|
| 310 |
+
|
| 311 |
+
def apply_approved_changes(inp_text: str, findings: Sequence[Mapping[str, Any]], edited_values: Mapping[str, Any] | None = None) -> tuple[str, list[dict[str, Any]]]:
|
| 312 |
+
"""Apply a deliberately limited set of approved, structured edits."""
|
| 313 |
+
preamble, sections = _split_sections(inp_text)
|
| 314 |
+
log: list[dict[str, Any]] = []
|
| 315 |
+
edited_values = dict(edited_values or {})
|
| 316 |
+
|
| 317 |
+
def update_named(section: str, object_id: str, token_index: int, value: Any, finding_id: str, parameter: str):
|
| 318 |
+
lines = sections.get(section, [])
|
| 319 |
+
changed = False
|
| 320 |
+
old = None
|
| 321 |
+
new_lines = []
|
| 322 |
+
for line in lines:
|
| 323 |
+
parsed = _data_tokens(line)
|
| 324 |
+
if parsed and parsed[0][0].casefold() == object_id.casefold() and len(parsed[0]) > token_index:
|
| 325 |
+
tokens, comment = parsed
|
| 326 |
+
old = tokens[token_index]
|
| 327 |
+
tokens[token_index] = str(value)
|
| 328 |
+
line = _replace_tokens(line, tokens, comment)
|
| 329 |
+
changed = True
|
| 330 |
+
new_lines.append(line)
|
| 331 |
+
sections[section] = new_lines
|
| 332 |
+
log.append({"finding_id": finding_id, "object_id": object_id, "parameter": parameter, "old_value": old, "new_value": value, "applied": changed})
|
| 333 |
+
|
| 334 |
+
for f in findings:
|
| 335 |
+
if str(f.get("engineer_decision", "")).lower() != "accept":
|
| 336 |
+
continue
|
| 337 |
+
fid = str(f.get("finding_id", ""))
|
| 338 |
+
parameter = str(f.get("proposed_parameter", ""))
|
| 339 |
+
value = edited_values.get(fid, f.get("proposed_value"))
|
| 340 |
+
oid = str(f.get("object_id", ""))
|
| 341 |
+
otype = str(f.get("object_type", "")).upper()
|
| 342 |
+
if value in (None, "") or not parameter:
|
| 343 |
+
log.append({"finding_id": fid, "object_id": oid, "parameter": parameter, "applied": False, "note": "No structured value supplied; retained as an accepted review action only."})
|
| 344 |
+
continue
|
| 345 |
+
if otype == "SUBCATCHMENT":
|
| 346 |
+
idx = {"area": 3, "imperviousness": 4, "width": 5, "slope": 6}.get(parameter)
|
| 347 |
+
if idx is not None:
|
| 348 |
+
update_named("SUBCATCHMENTS", oid, idx, value, fid, parameter)
|
| 349 |
+
continue
|
| 350 |
+
if otype == "MODEL" and parameter.upper() in {"REPORT_STEP", "WET_STEP", "ROUTING_STEP", "END_DATE", "END_TIME"}:
|
| 351 |
+
sections["OPTIONS"] = _set_option(sections.get("OPTIONS", []), parameter.upper(), str(value))
|
| 352 |
+
log.append({"finding_id": fid, "object_id": "MODEL", "parameter": parameter.upper(), "new_value": value, "applied": True})
|
| 353 |
+
continue
|
| 354 |
+
log.append({"finding_id": fid, "object_id": oid, "parameter": parameter, "new_value": value, "applied": False, "note": "Parameter is not yet supported by the controlled correction engine."})
|
| 355 |
+
return _join_sections(preamble, sections), log
|
| 356 |
+
|
| 357 |
+
|
| 358 |
+
def review_manifest(*, original_name: str, reviewed_name: str, original_text: str, reviewed_text: str, findings: Sequence[Mapping[str, Any]], change_log: Sequence[Mapping[str, Any]], status: str) -> dict[str, Any]:
|
| 359 |
+
return {
|
| 360 |
+
"workflow": "Rev23 Preliminary Design Assistant",
|
| 361 |
+
"created_at": datetime.now().isoformat(timespec="seconds"),
|
| 362 |
+
"original_model": original_name,
|
| 363 |
+
"reviewed_model": reviewed_name,
|
| 364 |
+
"original_sha256": hashlib.sha256(original_text.encode("utf-8")).hexdigest(),
|
| 365 |
+
"reviewed_sha256": hashlib.sha256(reviewed_text.encode("utf-8")).hexdigest(),
|
| 366 |
+
"review_status": status,
|
| 367 |
+
"finding_counts": pd.Series([f.get("severity", "") for f in findings]).value_counts().to_dict() if findings else {},
|
| 368 |
+
"decision_counts": pd.Series([f.get("engineer_decision", "") for f in findings]).value_counts().to_dict() if findings else {},
|
| 369 |
+
"change_log": list(change_log),
|
| 370 |
+
}
|
report_engine.py
ADDED
|
@@ -0,0 +1,1417 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Municipal stormwater report generator for the SWMM6 GIS Space.
|
| 2 |
+
|
| 3 |
+
Creates an editable DOCX report and a ZIP package of supporting CSV/JSON data
|
| 4 |
+
from the deterministic simulation results already held in Streamlit session state.
|
| 5 |
+
"""
|
| 6 |
+
from __future__ import annotations
|
| 7 |
+
|
| 8 |
+
import io
|
| 9 |
+
import json
|
| 10 |
+
import re
|
| 11 |
+
import sqlite3
|
| 12 |
+
import tempfile
|
| 13 |
+
import zipfile
|
| 14 |
+
from dataclasses import dataclass, asdict
|
| 15 |
+
from datetime import datetime
|
| 16 |
+
from pathlib import Path
|
| 17 |
+
from typing import Any, Mapping
|
| 18 |
+
|
| 19 |
+
import pandas as pd
|
| 20 |
+
from docx import Document
|
| 21 |
+
from docx.enum.section import WD_ORIENT, WD_SECTION
|
| 22 |
+
from docx.enum.text import WD_ALIGN_PARAGRAPH
|
| 23 |
+
from docx.enum.table import WD_CELL_VERTICAL_ALIGNMENT, WD_TABLE_ALIGNMENT
|
| 24 |
+
from docx.shared import Inches, Pt, RGBColor
|
| 25 |
+
from docx.oxml import OxmlElement
|
| 26 |
+
from docx.oxml.ns import qn
|
| 27 |
+
|
| 28 |
+
from calgary_rules import (
|
| 29 |
+
CalgaryCriteria, apply_storage_classification, build_llm_report_context,
|
| 30 |
+
build_minor_system_capacity_table, build_overland_compliance_table,
|
| 31 |
+
criteria_register, infer_design_event,
|
| 32 |
+
)
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
@dataclass
|
| 36 |
+
class ReportCriteria:
|
| 37 |
+
node_depth_ratio: float = 0.80
|
| 38 |
+
minimum_freeboard: float = 0.50
|
| 39 |
+
conduit_depth_ratio: float = 0.80
|
| 40 |
+
velocity_threshold: float = 4.00
|
| 41 |
+
continuity_review: float = 0.50
|
| 42 |
+
continuity_warning: float = 1.00
|
| 43 |
+
suppress_empty_sections: bool = True
|
| 44 |
+
major_link_ids: tuple[str, ...] = ()
|
| 45 |
+
area_classification: dict[str, str] | None = None
|
| 46 |
+
calgary_enabled: bool = True
|
| 47 |
+
minor_release_rate_lps_ha: float | None = None
|
| 48 |
+
trap_low_max_depth_m: float = 0.50
|
| 49 |
+
entrance_grade_margin_m: float = 0.30
|
| 50 |
+
conduit_capacity_warning_ratio: float = 0.95
|
| 51 |
+
special_link_limits: dict[str, float] | None = None
|
| 52 |
+
storage_classification: dict[str, str] | None = None
|
| 53 |
+
outfall_classification: dict[str, str] | None = None
|
| 54 |
+
checklist_overrides: dict[str, str] | None = None
|
| 55 |
+
drawing_inventory: tuple[str, ...] = ()
|
| 56 |
+
applicable_reports: tuple[str, ...] = ()
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
@dataclass
|
| 60 |
+
class ReportMetadata:
|
| 61 |
+
project_name: str = "SWMM Project"
|
| 62 |
+
client: str = "Not provided"
|
| 63 |
+
consultant: str = "Not provided"
|
| 64 |
+
consultant_file_no: str = "Not provided"
|
| 65 |
+
subdivision_no: str = "Not provided"
|
| 66 |
+
outline_plan_no: str = "Not provided"
|
| 67 |
+
development_permit_no: str = "Not provided"
|
| 68 |
+
design_storm: str = "Model design event"
|
| 69 |
+
prepared_by: str = "Not provided"
|
| 70 |
+
checked_by: str = "Not provided"
|
| 71 |
+
report_date: str = ""
|
| 72 |
+
municipality: str = "City of Calgary-style"
|
| 73 |
+
contact_name: str = "Not provided"
|
| 74 |
+
contact_email: str = "Not provided"
|
| 75 |
+
legal_description: str = "Not provided"
|
| 76 |
+
submission_status: str = "Preliminary"
|
| 77 |
+
construction_drawing_no: str = "Not provided"
|
| 78 |
+
development_agreement_no: str = "Not provided"
|
| 79 |
+
|
| 80 |
+
def __post_init__(self) -> None:
|
| 81 |
+
if not self.report_date:
|
| 82 |
+
self.report_date = datetime.now().strftime("%B %d, %Y")
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def _safe_name(value: str) -> str:
|
| 86 |
+
value = re.sub(r"[^A-Za-z0-9._-]+", "_", str(value).strip())
|
| 87 |
+
return value.strip("_") or "SWMM_Project"
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def _set_cell_text(cell, value: Any, bold: bool = False, size: float = 8.0) -> None:
|
| 91 |
+
cell.text = ""
|
| 92 |
+
cell.vertical_alignment = WD_CELL_VERTICAL_ALIGNMENT.CENTER
|
| 93 |
+
p = cell.paragraphs[0]
|
| 94 |
+
p.alignment = WD_ALIGN_PARAGRAPH.CENTER
|
| 95 |
+
p.paragraph_format.space_after = Pt(0)
|
| 96 |
+
run = p.add_run("" if value is None else str(value))
|
| 97 |
+
run.bold = bold
|
| 98 |
+
run.font.size = Pt(size)
|
| 99 |
+
run.font.name = "Arial"
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def _repeat_table_header(row) -> None:
|
| 103 |
+
tr_pr = row._tr.get_or_add_trPr()
|
| 104 |
+
tbl_header = OxmlElement("w:tblHeader")
|
| 105 |
+
tbl_header.set(qn("w:val"), "true")
|
| 106 |
+
tr_pr.append(tbl_header)
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def _set_cell_shading(cell, fill: str = "D9EAD3") -> None:
|
| 110 |
+
tc_pr = cell._tc.get_or_add_tcPr()
|
| 111 |
+
shd = tc_pr.find(qn("w:shd"))
|
| 112 |
+
if shd is None:
|
| 113 |
+
shd = OxmlElement("w:shd")
|
| 114 |
+
tc_pr.append(shd)
|
| 115 |
+
shd.set(qn("w:fill"), fill)
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def _set_landscape(section) -> None:
|
| 119 |
+
section.orientation = WD_ORIENT.LANDSCAPE
|
| 120 |
+
section.page_width, section.page_height = section.page_height, section.page_width
|
| 121 |
+
section.top_margin = Inches(0.45)
|
| 122 |
+
section.bottom_margin = Inches(0.45)
|
| 123 |
+
section.left_margin = Inches(0.45)
|
| 124 |
+
section.right_margin = Inches(0.45)
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def _format_value(value: Any) -> str:
|
| 128 |
+
if value is None or (isinstance(value, float) and pd.isna(value)):
|
| 129 |
+
return ""
|
| 130 |
+
if isinstance(value, float):
|
| 131 |
+
if abs(value) >= 1000:
|
| 132 |
+
return f"{value:,.2f}"
|
| 133 |
+
if abs(value) >= 10:
|
| 134 |
+
return f"{value:,.3f}".rstrip("0").rstrip(".")
|
| 135 |
+
return f"{value:,.4f}".rstrip("0").rstrip(".")
|
| 136 |
+
return str(value)
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def _add_df_table(
|
| 140 |
+
doc: Document,
|
| 141 |
+
title: str,
|
| 142 |
+
df: pd.DataFrame,
|
| 143 |
+
max_rows: int = 250,
|
| 144 |
+
*,
|
| 145 |
+
font_size: float = 7.4,
|
| 146 |
+
landscape: bool = False,
|
| 147 |
+
) -> None:
|
| 148 |
+
if landscape:
|
| 149 |
+
section = doc.add_section(WD_SECTION.NEW_PAGE)
|
| 150 |
+
_set_landscape(section)
|
| 151 |
+
doc.add_heading(title, level=3)
|
| 152 |
+
if df is None or df.empty:
|
| 153 |
+
doc.add_paragraph("No applicable model records were identified.")
|
| 154 |
+
return
|
| 155 |
+
|
| 156 |
+
shown = df.head(max_rows).copy()
|
| 157 |
+
table = doc.add_table(rows=1, cols=len(shown.columns))
|
| 158 |
+
table.style = "Table Grid"
|
| 159 |
+
table.alignment = WD_TABLE_ALIGNMENT.CENTER
|
| 160 |
+
table.autofit = True
|
| 161 |
+
_repeat_table_header(table.rows[0])
|
| 162 |
+
for i, col in enumerate(shown.columns):
|
| 163 |
+
_set_cell_text(table.rows[0].cells[i], col, bold=True, size=max(6.3, font_size - 0.2))
|
| 164 |
+
_set_cell_shading(table.rows[0].cells[i])
|
| 165 |
+
for _, row in shown.iterrows():
|
| 166 |
+
cells = table.add_row().cells
|
| 167 |
+
for i, col in enumerate(shown.columns):
|
| 168 |
+
_set_cell_text(cells[i], _format_value(row[col]), size=font_size)
|
| 169 |
+
if len(df) > max_rows:
|
| 170 |
+
doc.add_paragraph(f"Table truncated in the report at {max_rows:,} rows. Complete data are included in the ZIP package.")
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
@dataclass(frozen=True)
|
| 174 |
+
class UnitContext:
|
| 175 |
+
flow_units: str
|
| 176 |
+
system: str
|
| 177 |
+
flow: str
|
| 178 |
+
length: str
|
| 179 |
+
area: str
|
| 180 |
+
velocity: str
|
| 181 |
+
rainfall: str
|
| 182 |
+
storage: str
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
def _unit_context(flow_units: str) -> UnitContext:
|
| 186 |
+
u = str(flow_units or "").upper()
|
| 187 |
+
if u in {"CFS", "GPM", "MGD", "IMGD", "AFD"}:
|
| 188 |
+
flow = {"CFS": "cfs", "GPM": "gpm", "MGD": "MGD", "IMGD": "IMGD", "AFD": "ac-ft/day"}.get(u, u)
|
| 189 |
+
return UnitContext(u, "US Customary", flow, "ft", "ac", "ft/s", "in/hr", "ft³")
|
| 190 |
+
flow = {"CMS": "m³/s", "LPS": "L/s", "MLD": "ML/day"}.get(u, u or "model units")
|
| 191 |
+
return UnitContext(u or "UNKNOWN", "SI", flow, "m", "ha", "m/s", "mm/hr", "m³")
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
def _rename_native_columns(df: pd.DataFrame, units: UnitContext) -> pd.DataFrame:
|
| 195 |
+
if df is None:
|
| 196 |
+
return pd.DataFrame()
|
| 197 |
+
out = df.copy()
|
| 198 |
+
replacements = {
|
| 199 |
+
"Area (ha)": f"Area ({units.area})",
|
| 200 |
+
"Width (m)": f"Width ({units.length})",
|
| 201 |
+
"Length (m)": f"Length ({units.length})",
|
| 202 |
+
"Diameter (m)": f"Diameter ({units.length})",
|
| 203 |
+
"Geom1 (m)": f"Geom1 ({units.length})",
|
| 204 |
+
"Inlet Offset (m)": f"Inlet Offset ({units.length})",
|
| 205 |
+
"Outlet Offset (m)": f"Outlet Offset ({units.length})",
|
| 206 |
+
"Invert (m)": f"Invert ({units.length})",
|
| 207 |
+
"Full Depth (m)": f"Full Depth ({units.length})",
|
| 208 |
+
"Peak Depth (m)": f"Peak Depth ({units.length})",
|
| 209 |
+
"Maximum HGL (m)": f"Maximum HGL ({units.length})",
|
| 210 |
+
"Ground/Rim (m)": f"Ground/Rim ({units.length})",
|
| 211 |
+
"Freeboard (m)": f"Freeboard ({units.length})",
|
| 212 |
+
"Peak Flow (m³/s)": f"Peak Flow ({units.flow})",
|
| 213 |
+
"Peak Runoff (m³/s)": f"Peak Runoff ({units.flow})",
|
| 214 |
+
"Peak Flooding (m³/s)": f"Peak Flooding ({units.flow})",
|
| 215 |
+
"Peak Inflow (m³/s)": f"Peak Inflow ({units.flow})",
|
| 216 |
+
"Peak Velocity (m/s)": f"Peak Velocity ({units.velocity})",
|
| 217 |
+
"Peak Rainfall (mm/h)": f"Peak Rainfall ({units.rainfall})",
|
| 218 |
+
}
|
| 219 |
+
if units.flow_units == "CFS":
|
| 220 |
+
replacements["Total Runoff (m³)"] = "Runoff Volume (ft³)"
|
| 221 |
+
replacements["Total Rainfall Depth"] = "Total Rainfall (in)"
|
| 222 |
+
replacements["Runoff Depth"] = "Runoff Depth (in)"
|
| 223 |
+
elif units.flow_units == "CMS":
|
| 224 |
+
replacements["Total Runoff (m³)"] = "Runoff Volume (m³)"
|
| 225 |
+
replacements["Total Rainfall Depth"] = "Total Rainfall (mm)"
|
| 226 |
+
replacements["Runoff Depth"] = "Runoff Depth (mm)"
|
| 227 |
+
elif units.flow_units == "LPS":
|
| 228 |
+
replacements["Total Runoff (m³)"] = "Runoff Volume (L)"
|
| 229 |
+
replacements["Total Rainfall Depth"] = "Total Rainfall (mm)"
|
| 230 |
+
replacements["Runoff Depth"] = "Runoff Depth (mm)"
|
| 231 |
+
else:
|
| 232 |
+
replacements["Total Runoff (m³)"] = f"Integrated Runoff ({units.flow}-s)"
|
| 233 |
+
return out.rename(columns={k: v for k, v in replacements.items() if k in out.columns})
|
| 234 |
+
|
| 235 |
+
def _inp_options(sections: dict[str, list[list[str]]]) -> dict[str, str]:
|
| 236 |
+
result: dict[str, str] = {}
|
| 237 |
+
for row in sections.get("OPTIONS", []):
|
| 238 |
+
if len(row) >= 2:
|
| 239 |
+
result[row[0].upper()] = " ".join(row[1:])
|
| 240 |
+
return result
|
| 241 |
+
|
| 242 |
+
|
| 243 |
+
def _subcatchment_model_table(sections: dict[str, list[list[str]]], sub_summary: pd.DataFrame) -> pd.DataFrame:
|
| 244 |
+
attrs: dict[str, dict[str, Any]] = {}
|
| 245 |
+
for row in sections.get("SUBCATCHMENTS", []):
|
| 246 |
+
if len(row) >= 7:
|
| 247 |
+
try:
|
| 248 |
+
attrs[row[0]] = {
|
| 249 |
+
"Rain Gage": row[1], "Outlet": row[2], "Area (ha)": float(row[3]),
|
| 250 |
+
"Impervious (%)": float(row[4]), "Width (m)": float(row[5]), "Slope (%)": float(row[6]),
|
| 251 |
+
}
|
| 252 |
+
except ValueError:
|
| 253 |
+
continue
|
| 254 |
+
for row in sections.get("SUBAREAS", []):
|
| 255 |
+
if len(row) >= 7 and row[0] in attrs:
|
| 256 |
+
try:
|
| 257 |
+
attrs[row[0]].update({
|
| 258 |
+
"n Imperv.": float(row[1]), "n Perv.": float(row[2]),
|
| 259 |
+
"Dstore Imperv.": float(row[3]), "Dstore Perv.": float(row[4]),
|
| 260 |
+
"Zero Imperv. (%)": float(row[5]), "Routing": row[6],
|
| 261 |
+
})
|
| 262 |
+
except ValueError:
|
| 263 |
+
pass
|
| 264 |
+
model_df = pd.DataFrame([{"Sub ID": sid, **vals} for sid, vals in attrs.items()])
|
| 265 |
+
if model_df.empty:
|
| 266 |
+
return sub_summary.copy()
|
| 267 |
+
if sub_summary is not None and not sub_summary.empty:
|
| 268 |
+
# Model input values are authoritative for shared fields; avoid duplicate area/impervious columns.
|
| 269 |
+
result_df = sub_summary.drop(columns=[c for c in ["Area (ha)", "% Impervious"] if c in sub_summary.columns], errors="ignore")
|
| 270 |
+
return model_df.merge(result_df, on="Sub ID", how="left")
|
| 271 |
+
return model_df
|
| 272 |
+
|
| 273 |
+
|
| 274 |
+
def _link_model_table(sections: dict[str, list[list[str]]], link_summary: pd.DataFrame) -> pd.DataFrame:
|
| 275 |
+
records: dict[str, dict[str, Any]] = {}
|
| 276 |
+
for section, link_type in (("CONDUITS", "conduit"), ("PUMPS", "pump"), ("ORIFICES", "orifice"), ("WEIRS", "weir"), ("OUTLETS", "outlet")):
|
| 277 |
+
for row in sections.get(section, []):
|
| 278 |
+
if len(row) < 3:
|
| 279 |
+
continue
|
| 280 |
+
rec = {"Link ID": row[0], "Model Type": link_type, "From Node": row[1], "To Node": row[2]}
|
| 281 |
+
if section == "CONDUITS" and len(row) >= 9:
|
| 282 |
+
try:
|
| 283 |
+
rec.update({
|
| 284 |
+
"Length (m)": float(row[3]), "Manning n": float(row[4]),
|
| 285 |
+
"Inlet Offset (m)": float(row[5]), "Outlet Offset (m)": float(row[6]),
|
| 286 |
+
"Initial Flow": float(row[7]), "Maximum Flow": float(row[8]),
|
| 287 |
+
})
|
| 288 |
+
except ValueError:
|
| 289 |
+
pass
|
| 290 |
+
records[row[0]] = rec
|
| 291 |
+
for row in sections.get("XSECTIONS", []):
|
| 292 |
+
if len(row) >= 3 and row[0] in records:
|
| 293 |
+
records[row[0]]["Shape"] = row[1]
|
| 294 |
+
try:
|
| 295 |
+
records[row[0]]["Geom1 (m)"] = float(row[2])
|
| 296 |
+
if len(row) > 3: records[row[0]]["Geom2"] = float(row[3])
|
| 297 |
+
if len(row) > 4: records[row[0]]["Geom3"] = float(row[4])
|
| 298 |
+
if len(row) > 5: records[row[0]]["Geom4"] = float(row[5])
|
| 299 |
+
if len(row) > 6: records[row[0]]["Barrels"] = int(float(row[6]))
|
| 300 |
+
except ValueError:
|
| 301 |
+
pass
|
| 302 |
+
model_df = pd.DataFrame(records.values())
|
| 303 |
+
if model_df.empty:
|
| 304 |
+
return link_summary.copy()
|
| 305 |
+
if link_summary is not None and not link_summary.empty:
|
| 306 |
+
drop_overlap = [c for c in ["From Node", "To Node", "Length (m)"] if c in link_summary.columns]
|
| 307 |
+
result_df = link_summary.drop(columns=drop_overlap, errors="ignore")
|
| 308 |
+
return model_df.merge(result_df, on="Link ID", how="left")
|
| 309 |
+
return model_df
|
| 310 |
+
|
| 311 |
+
|
| 312 |
+
def _node_hgl_table(node_summary: pd.DataFrame) -> pd.DataFrame:
|
| 313 |
+
if node_summary is None or node_summary.empty:
|
| 314 |
+
return pd.DataFrame()
|
| 315 |
+
df = node_summary.copy()
|
| 316 |
+
df["Maximum HGL (m)"] = df.get("Invert (m)", 0) + df.get("Peak Depth (m)", 0)
|
| 317 |
+
df["Ground/Rim (m)"] = df.get("Invert (m)", 0) + df.get("Full Depth (m)", 0)
|
| 318 |
+
df["Freeboard (m)"] = df["Ground/Rim (m)"] - df["Maximum HGL (m)"]
|
| 319 |
+
cols = ["Node ID", "Type", "Invert (m)", "Ground/Rim (m)", "Maximum HGL (m)", "Peak Depth (m)", "Freeboard (m)", "Peak Flooding (m³/s)", "Status"]
|
| 320 |
+
return df[[c for c in cols if c in df.columns]]
|
| 321 |
+
|
| 322 |
+
|
| 323 |
+
|
| 324 |
+
def _parse_time_seconds(value: str | None) -> float:
|
| 325 |
+
"""Parse SWMM HH:MM:SS or numeric-second time values."""
|
| 326 |
+
if value is None:
|
| 327 |
+
return 0.0
|
| 328 |
+
text = str(value).strip()
|
| 329 |
+
if not text:
|
| 330 |
+
return 0.0
|
| 331 |
+
try:
|
| 332 |
+
return float(text)
|
| 333 |
+
except ValueError:
|
| 334 |
+
pass
|
| 335 |
+
parts = text.split(":")
|
| 336 |
+
try:
|
| 337 |
+
nums = [float(x) for x in parts]
|
| 338 |
+
except ValueError:
|
| 339 |
+
return 0.0
|
| 340 |
+
if len(nums) == 3:
|
| 341 |
+
return nums[0] * 3600.0 + nums[1] * 60.0 + nums[2]
|
| 342 |
+
if len(nums) == 2:
|
| 343 |
+
return nums[0] * 60.0 + nums[1]
|
| 344 |
+
return 0.0
|
| 345 |
+
|
| 346 |
+
|
| 347 |
+
def _enhance_subcatchment_results(df: pd.DataFrame, units: UnitContext, report_step_s: float) -> pd.DataFrame:
|
| 348 |
+
"""Add defensible event coefficients and integrated runoff volumes in native units."""
|
| 349 |
+
if df is None or df.empty:
|
| 350 |
+
return pd.DataFrame()
|
| 351 |
+
out = df.copy()
|
| 352 |
+
out = out.loc[:, ~out.columns.duplicated()].copy()
|
| 353 |
+
area_col = next((c for c in out.columns if c.startswith("Area (")), None)
|
| 354 |
+
q_col = next((c for c in out.columns if c.startswith("Peak Runoff (")), None)
|
| 355 |
+
rain_col = next((c for c in out.columns if c.startswith("Peak Rainfall (")), None)
|
| 356 |
+
sum_col = next((c for c in out.columns if c.startswith("Runoff Sum (")), None)
|
| 357 |
+
|
| 358 |
+
if area_col and q_col and rain_col:
|
| 359 |
+
area = pd.to_numeric(out[area_col], errors="coerce")
|
| 360 |
+
q = pd.to_numeric(out[q_col], errors="coerce")
|
| 361 |
+
intensity = pd.to_numeric(out[rain_col], errors="coerce")
|
| 362 |
+
denom = pd.Series(float("nan"), index=out.index)
|
| 363 |
+
if units.flow_units == "CFS":
|
| 364 |
+
denom = 1.008 * intensity * area # Q(cfs)=1.008*C*i(in/hr)*A(ac)
|
| 365 |
+
elif units.flow_units == "CMS":
|
| 366 |
+
denom = 0.0027777778 * intensity * area # Q(m3/s)=0.0027778*C*i(mm/hr)*A(ha)
|
| 367 |
+
elif units.flow_units == "LPS":
|
| 368 |
+
denom = 2.7777778 * intensity * area # Q(L/s)=2.7778*C*i(mm/hr)*A(ha)
|
| 369 |
+
coeff = q / denom.where(denom > 0)
|
| 370 |
+
out["Peak-Flow Runoff Coefficient"] = coeff.where((coeff >= 0) & (coeff <= 1.5))
|
| 371 |
+
out.drop(columns=["Runoff Coefficient"], errors="ignore", inplace=True)
|
| 372 |
+
|
| 373 |
+
if sum_col and report_step_s > 0:
|
| 374 |
+
runoff_sum = pd.to_numeric(out[sum_col], errors="coerce").fillna(0.0)
|
| 375 |
+
if units.flow_units == "CFS":
|
| 376 |
+
volume_ft3 = runoff_sum * report_step_s
|
| 377 |
+
out["Runoff Volume (ft³)"] = volume_ft3
|
| 378 |
+
out["Runoff Volume (ac-ft)"] = volume_ft3 / 43560.0
|
| 379 |
+
out["Runoff Volume (MG)"] = volume_ft3 * 7.48051948 / 1_000_000.0
|
| 380 |
+
if area_col:
|
| 381 |
+
area = pd.to_numeric(out[area_col], errors="coerce")
|
| 382 |
+
out["Runoff Depth (in)"] = (volume_ft3 / (area * 43560.0)) * 12.0
|
| 383 |
+
elif units.flow_units == "CMS":
|
| 384 |
+
volume_m3 = runoff_sum * report_step_s
|
| 385 |
+
out["Runoff Volume (m³)"] = volume_m3
|
| 386 |
+
if area_col:
|
| 387 |
+
area = pd.to_numeric(out[area_col], errors="coerce")
|
| 388 |
+
out["Runoff Depth (mm)"] = volume_m3 / (area * 10.0)
|
| 389 |
+
elif units.flow_units == "LPS":
|
| 390 |
+
volume_m3 = runoff_sum * report_step_s / 1000.0
|
| 391 |
+
out["Runoff Volume (m³)"] = volume_m3
|
| 392 |
+
if area_col:
|
| 393 |
+
area = pd.to_numeric(out[area_col], errors="coerce")
|
| 394 |
+
out["Runoff Depth (mm)"] = volume_m3 / (area * 10.0)
|
| 395 |
+
elif units.flow_units in {"GPM", "MGD", "IMGD", "MLD", "AFD"}:
|
| 396 |
+
out[f"Integrated Runoff ({units.flow}-s)"] = runoff_sum * report_step_s
|
| 397 |
+
out.drop(columns=[sum_col], errors="ignore", inplace=True)
|
| 398 |
+
|
| 399 |
+
# Add convenient secondary volume units when the application already supplied an integrated native volume.
|
| 400 |
+
if units.flow_units == "CFS" and "Runoff Volume (ft³)" in out.columns:
|
| 401 |
+
volume_ft3 = pd.to_numeric(out["Runoff Volume (ft³)"], errors="coerce")
|
| 402 |
+
out["Runoff Volume (ac-ft)"] = volume_ft3 / 43560.0
|
| 403 |
+
out["Runoff Volume (MG)"] = volume_ft3 * 7.48051948 / 1_000_000.0
|
| 404 |
+
elif units.flow_units == "LPS" and "Runoff Volume (L)" in out.columns:
|
| 405 |
+
out["Runoff Volume (m³)"] = pd.to_numeric(out["Runoff Volume (L)"], errors="coerce") / 1000.0
|
| 406 |
+
return out
|
| 407 |
+
|
| 408 |
+
|
| 409 |
+
def _populate_outfall_flows(node_df: pd.DataFrame, link_df: pd.DataFrame, units: UnitContext) -> pd.DataFrame:
|
| 410 |
+
"""Populate outfall peak inflow from incoming links when node output is absent or zero."""
|
| 411 |
+
if node_df is None or node_df.empty:
|
| 412 |
+
return pd.DataFrame()
|
| 413 |
+
out = node_df.copy()
|
| 414 |
+
inflow_col = f"Peak Inflow ({units.flow})"
|
| 415 |
+
flow_col = f"Peak Flow ({units.flow})"
|
| 416 |
+
if inflow_col not in out.columns:
|
| 417 |
+
out[inflow_col] = 0.0
|
| 418 |
+
if link_df is None or link_df.empty or "To Node" not in link_df or flow_col not in link_df:
|
| 419 |
+
return out
|
| 420 |
+
incoming = link_df.groupby("To Node")[flow_col].max()
|
| 421 |
+
mask = out.get("Type", "").astype(str).str.lower().eq("outfall")
|
| 422 |
+
for idx in out.index[mask]:
|
| 423 |
+
node_id = out.at[idx, "Node ID"]
|
| 424 |
+
current = pd.to_numeric(pd.Series([out.at[idx, inflow_col]]), errors="coerce").iloc[0]
|
| 425 |
+
fallback = incoming.get(node_id)
|
| 426 |
+
if (pd.isna(current) or float(current) <= 0.0) and fallback is not None and not pd.isna(fallback):
|
| 427 |
+
out.at[idx, inflow_col] = float(fallback)
|
| 428 |
+
return out
|
| 429 |
+
|
| 430 |
+
|
| 431 |
+
def _split_link_appendix(link_table: pd.DataFrame, units: UnitContext) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
|
| 432 |
+
geom_cols = ["Link ID", "From Node", "To Node", "Model Type", "Shape", f"Length ({units.length})",
|
| 433 |
+
f"Geom1 ({units.length})", "Geom2", "Geom3", "Geom4", "Barrels"]
|
| 434 |
+
params_cols = ["Link ID", "Manning n", f"Inlet Offset ({units.length})", f"Outlet Offset ({units.length})",
|
| 435 |
+
"Initial Flow", "Maximum Flow"]
|
| 436 |
+
results_cols = ["Link ID", f"Peak Flow ({units.flow})", f"Peak Depth ({units.length})", "Depth Ratio",
|
| 437 |
+
f"Peak Velocity ({units.velocity})", "Status"]
|
| 438 |
+
return tuple(link_table[[c for c in cols if c in link_table.columns]].copy() for cols in (geom_cols, params_cols, results_cols))
|
| 439 |
+
|
| 440 |
+
|
| 441 |
+
def _split_subcatchment_appendix(sub_table: pd.DataFrame, units: UnitContext) -> tuple[pd.DataFrame, pd.DataFrame]:
|
| 442 |
+
model_cols = ["Sub ID", "Rain Gage", "Outlet", f"Area ({units.area})", "Impervious (%)", f"Width ({units.length})",
|
| 443 |
+
"Slope (%)", "n Imperv.", "n Perv.", "Dstore Imperv.", "Dstore Perv.", "Zero Imperv. (%)", "Routing", "Connected To"]
|
| 444 |
+
result_cols = ["Sub ID", f"Peak Runoff ({units.flow})", "Peak-Flow Runoff Coefficient",
|
| 445 |
+
f"Peak Rainfall ({units.rainfall})", "Runoff Volume (ft³)", "Runoff Volume (ac-ft)",
|
| 446 |
+
"Runoff Volume (MG)", "Runoff Volume (m³)", "Runoff Depth (in)", "Runoff Depth (mm)"]
|
| 447 |
+
return (sub_table[[c for c in model_cols if c in sub_table.columns]].copy(),
|
| 448 |
+
sub_table[[c for c in result_cols if c in sub_table.columns]].copy())
|
| 449 |
+
|
| 450 |
+
|
| 451 |
+
|
| 452 |
+
def _read_sqlite_tables(result_db_bytes: bytes | None, table_names: list[str]) -> dict[str, pd.DataFrame]:
|
| 453 |
+
"""Read selected result tables from the in-memory SQLite export package."""
|
| 454 |
+
if not result_db_bytes:
|
| 455 |
+
return {}
|
| 456 |
+
tables: dict[str, pd.DataFrame] = {}
|
| 457 |
+
tmp_name = None
|
| 458 |
+
try:
|
| 459 |
+
with tempfile.NamedTemporaryFile(suffix=".sqlite", delete=False) as tmp:
|
| 460 |
+
tmp.write(result_db_bytes)
|
| 461 |
+
tmp_name = tmp.name
|
| 462 |
+
with sqlite3.connect(tmp_name) as con:
|
| 463 |
+
available = {r[0] for r in con.execute("SELECT name FROM sqlite_master WHERE type='table'")}
|
| 464 |
+
for name in table_names:
|
| 465 |
+
if name in available:
|
| 466 |
+
tables[name] = pd.read_sql_query(f'SELECT * FROM "{name}"', con)
|
| 467 |
+
except Exception:
|
| 468 |
+
return tables
|
| 469 |
+
finally:
|
| 470 |
+
if tmp_name:
|
| 471 |
+
try:
|
| 472 |
+
Path(tmp_name).unlink(missing_ok=True)
|
| 473 |
+
except Exception:
|
| 474 |
+
pass
|
| 475 |
+
return tables
|
| 476 |
+
|
| 477 |
+
|
| 478 |
+
def _control_table(sections: dict[str, list[list[str]]], link_table: pd.DataFrame, units: UnitContext) -> pd.DataFrame:
|
| 479 |
+
"""Create a control-specific schedule for orifices, weirs and outlets."""
|
| 480 |
+
peak_col = f"Peak Flow ({units.flow})"
|
| 481 |
+
peak_lookup = {}
|
| 482 |
+
if link_table is not None and not link_table.empty and peak_col in link_table.columns:
|
| 483 |
+
peak_lookup = link_table.set_index("Link ID")[peak_col].to_dict()
|
| 484 |
+
rows: list[dict[str, Any]] = []
|
| 485 |
+
for row in sections.get("ORIFICES", []):
|
| 486 |
+
if len(row) >= 6:
|
| 487 |
+
rows.append({
|
| 488 |
+
"Control ID": row[0], "Control Type": "Orifice", "From Node": row[1], "To Node": row[2],
|
| 489 |
+
"Subtype": row[3], f"Offset ({units.length})": _num(row[4]), "Discharge Coefficient": _num(row[5]),
|
| 490 |
+
"Flap Gate": row[6] if len(row) > 6 else "", "Opening/Closing Time": _num(row[7]) if len(row) > 7 else None,
|
| 491 |
+
peak_col: peak_lookup.get(row[0]),
|
| 492 |
+
})
|
| 493 |
+
for row in sections.get("WEIRS", []):
|
| 494 |
+
if len(row) >= 6:
|
| 495 |
+
rows.append({
|
| 496 |
+
"Control ID": row[0], "Control Type": "Weir", "From Node": row[1], "To Node": row[2],
|
| 497 |
+
"Subtype": row[3], f"Crest Height ({units.length})": _num(row[4]), "Discharge Coefficient": _num(row[5]),
|
| 498 |
+
"Flap Gate": row[6] if len(row) > 6 else "", "End Contractions": _num(row[7]) if len(row) > 7 else None,
|
| 499 |
+
"End Coefficient": _num(row[8]) if len(row) > 8 else None, "Surcharge Allowed": row[9] if len(row) > 9 else "",
|
| 500 |
+
peak_col: peak_lookup.get(row[0]),
|
| 501 |
+
})
|
| 502 |
+
for row in sections.get("OUTLETS", []):
|
| 503 |
+
if len(row) >= 6:
|
| 504 |
+
rows.append({
|
| 505 |
+
"Control ID": row[0], "Control Type": "Outlet", "From Node": row[1], "To Node": row[2],
|
| 506 |
+
f"Offset ({units.length})": _num(row[3]), "Rating Type": row[4], "Rating Curve/Parameters": " ".join(row[5:]),
|
| 507 |
+
peak_col: peak_lookup.get(row[0]),
|
| 508 |
+
})
|
| 509 |
+
return pd.DataFrame(rows)
|
| 510 |
+
|
| 511 |
+
|
| 512 |
+
def _num(value: Any) -> float | None:
|
| 513 |
+
try:
|
| 514 |
+
return float(value)
|
| 515 |
+
except Exception:
|
| 516 |
+
return None
|
| 517 |
+
|
| 518 |
+
|
| 519 |
+
def _control_reconciliation(control_df: pd.DataFrame, link_table: pd.DataFrame, units: UnitContext) -> pd.DataFrame:
|
| 520 |
+
"""Compare parallel control peaks with the downstream conveyance peak as a screening check."""
|
| 521 |
+
peak_col = f"Peak Flow ({units.flow})"
|
| 522 |
+
if control_df is None or control_df.empty or link_table is None or link_table.empty or peak_col not in link_table.columns:
|
| 523 |
+
return pd.DataFrame()
|
| 524 |
+
rows = []
|
| 525 |
+
for to_node, grp in control_df.groupby("To Node", dropna=False):
|
| 526 |
+
control_sum = pd.to_numeric(grp[peak_col], errors="coerce").fillna(0).sum()
|
| 527 |
+
downstream = link_table[(link_table.get("From Node") == to_node) & (link_table.get("Model Type", link_table.get("Type", "")).astype(str).str.lower() == "conduit")]
|
| 528 |
+
downstream_peak = pd.to_numeric(downstream[peak_col], errors="coerce").max() if not downstream.empty else float("nan")
|
| 529 |
+
diff = downstream_peak - control_sum if pd.notna(downstream_peak) else float("nan")
|
| 530 |
+
pct = abs(diff) / max(abs(downstream_peak), 1e-12) * 100 if pd.notna(diff) else float("nan")
|
| 531 |
+
rows.append({
|
| 532 |
+
"Receiving Node": to_node, "Controls": ", ".join(grp["Control ID"].astype(str)),
|
| 533 |
+
f"Sum of Control Peaks ({units.flow})": control_sum,
|
| 534 |
+
"Downstream Link": ", ".join(downstream["Link ID"].astype(str)) if not downstream.empty else "Not identified",
|
| 535 |
+
f"Downstream Peak ({units.flow})": downstream_peak,
|
| 536 |
+
f"Difference ({units.flow})": diff, "Difference (%)": pct,
|
| 537 |
+
"Check": "Consistent" if pd.notna(pct) and pct <= 2.0 else "Review timing / routing",
|
| 538 |
+
})
|
| 539 |
+
return pd.DataFrame(rows)
|
| 540 |
+
|
| 541 |
+
|
| 542 |
+
def _storage_table(sections: dict[str, list[list[str]]], node_report: pd.DataFrame, db_tables: dict[str, pd.DataFrame], units: UnitContext) -> pd.DataFrame:
|
| 543 |
+
"""Create storage-specific input/result table using node time series when available."""
|
| 544 |
+
node_ts = db_tables.get("node_timeseries", pd.DataFrame())
|
| 545 |
+
node_lookup = node_report.set_index("Node ID").to_dict("index") if node_report is not None and not node_report.empty else {}
|
| 546 |
+
rows = []
|
| 547 |
+
for r in sections.get("STORAGE", []):
|
| 548 |
+
if len(r) < 5:
|
| 549 |
+
continue
|
| 550 |
+
sid = r[0]; invert = _num(r[1]); max_depth = _num(r[2]); init_depth = _num(r[3]); shape = r[4]
|
| 551 |
+
rec: dict[str, Any] = {"Storage ID": sid, f"Invert ({units.length})": invert, f"Maximum Depth ({units.length})": max_depth,
|
| 552 |
+
f"Initial Depth ({units.length})": init_depth, "Shape/Curve": shape,
|
| 553 |
+
"Shape Parameters": " ".join(r[5:])}
|
| 554 |
+
nrec = node_lookup.get(sid, {})
|
| 555 |
+
rec[f"Reported Peak Depth ({units.length})"] = nrec.get(f"Peak Depth ({units.length})")
|
| 556 |
+
rec[f"Maximum HGL ({units.length})"] = nrec.get(f"Invert ({units.length})", invert) + (nrec.get(f"Peak Depth ({units.length})") or 0) if invert is not None else None
|
| 557 |
+
if not node_ts.empty and "node_id" in node_ts.columns:
|
| 558 |
+
g = node_ts[node_ts["node_id"].astype(str) == sid].copy()
|
| 559 |
+
if not g.empty:
|
| 560 |
+
for src, label in (("depth", f"Time-Series Peak Depth ({units.length})"), ("volume", f"Maximum Stored Volume ({units.storage})"),
|
| 561 |
+
("inflow", f"Peak Inflow ({units.flow})"), ("outflow", f"Peak Outflow ({units.flow})")):
|
| 562 |
+
if src in g:
|
| 563 |
+
rec[label] = pd.to_numeric(g[src], errors="coerce").max()
|
| 564 |
+
if "volume" in g and "timestamp" in g:
|
| 565 |
+
vals = pd.to_numeric(g["volume"], errors="coerce")
|
| 566 |
+
if vals.notna().any(): rec["Time of Maximum Storage"] = g.loc[vals.idxmax(), "timestamp"]
|
| 567 |
+
peak_depth = rec.get(f"Time-Series Peak Depth ({units.length})", rec.get(f"Reported Peak Depth ({units.length})"))
|
| 568 |
+
rec["Depth Utilization (%)"] = 100 * float(peak_depth) / float(max_depth) if peak_depth is not None and max_depth and max_depth > 0 else None
|
| 569 |
+
rec["Status"] = "Review zero storage response" if (peak_depth is not None and float(peak_depth) == 0 and (rec.get(f"Peak Outflow ({units.flow})") or 0) > 0) else "OK"
|
| 570 |
+
rows.append(rec)
|
| 571 |
+
return pd.DataFrame(rows)
|
| 572 |
+
|
| 573 |
+
|
| 574 |
+
def _area_classification(sub_table: pd.DataFrame, units: UnitContext, overrides: dict[str, str] | None = None) -> pd.DataFrame:
|
| 575 |
+
"""Separate likely proposed, pre-development and external/comparison catchments by naming convention."""
|
| 576 |
+
if sub_table is None or sub_table.empty:
|
| 577 |
+
return pd.DataFrame()
|
| 578 |
+
area_col = f"Area ({units.area})"
|
| 579 |
+
if area_col not in sub_table:
|
| 580 |
+
return pd.DataFrame()
|
| 581 |
+
overrides = {str(k): str(v) for k, v in (overrides or {}).items()}
|
| 582 |
+
def classify(name: str) -> str:
|
| 583 |
+
if str(name) in overrides:
|
| 584 |
+
return overrides[str(name)]
|
| 585 |
+
n = str(name).lower().replace("_", " ")
|
| 586 |
+
if any(k in n for k in ("predevelop", "pre develop", "pre-development", "existing", "predev")):
|
| 587 |
+
return "Pre-development / comparison"
|
| 588 |
+
if any(k in n for k in ("external", "offsite", "off-site", "upstream")):
|
| 589 |
+
return "External area"
|
| 590 |
+
return "Proposed / modelled development"
|
| 591 |
+
d = sub_table[["Sub ID", area_col]].copy(); d["Area Category"] = d["Sub ID"].map(classify)
|
| 592 |
+
return d.groupby("Area Category", as_index=False)[area_col].sum()
|
| 593 |
+
|
| 594 |
+
|
| 595 |
+
def _critical_elements(node_df: pd.DataFrame, link_df: pd.DataFrame, units: UnitContext, criteria: ReportCriteria) -> tuple[pd.DataFrame, pd.DataFrame]:
|
| 596 |
+
node_rows = []
|
| 597 |
+
if node_df is not None and not node_df.empty:
|
| 598 |
+
for _, r in node_df.iterrows():
|
| 599 |
+
node_type = str(r.get("Type", "")).lower()
|
| 600 |
+
# Storage facilities use Calgary storage-classification criteria and are
|
| 601 |
+
# intentionally excluded from generic junction freeboard screening.
|
| 602 |
+
if node_type in {"outfall", "storage"}:
|
| 603 |
+
continue
|
| 604 |
+
ratio = pd.to_numeric(pd.Series([r.get("Depth Ratio")]), errors="coerce").iloc[0]
|
| 605 |
+
free = pd.to_numeric(pd.Series([r.get(f"Freeboard ({units.length})")]), errors="coerce").iloc[0]
|
| 606 |
+
flooding = pd.to_numeric(pd.Series([r.get(f"Peak Flooding ({units.flow})")]), errors="coerce").fillna(0).iloc[0]
|
| 607 |
+
issues = []
|
| 608 |
+
if pd.notna(ratio) and ratio >= criteria.node_depth_ratio:
|
| 609 |
+
issues.append(f"depth ratio ≥ {criteria.node_depth_ratio:.2f}")
|
| 610 |
+
if pd.notna(free) and free <= criteria.minimum_freeboard:
|
| 611 |
+
issues.append(f"freeboard ≤ {criteria.minimum_freeboard:g} {units.length}")
|
| 612 |
+
if flooding > 0:
|
| 613 |
+
issues.append("flooding reported")
|
| 614 |
+
if issues:
|
| 615 |
+
node_rows.append({"Node ID": r.get("Node ID"), "Type": r.get("Type"), "Depth Ratio": ratio,
|
| 616 |
+
f"Freeboard ({units.length})": free, f"Peak Flooding ({units.flow})": flooding,
|
| 617 |
+
"Review Issue": "; ".join(issues)})
|
| 618 |
+
link_rows = []
|
| 619 |
+
if link_df is not None and not link_df.empty:
|
| 620 |
+
vel_col = f"Peak Velocity ({units.velocity})"
|
| 621 |
+
for _, r in link_df.iterrows():
|
| 622 |
+
if str(r.get("Type", r.get("Model Type", ""))).lower() not in ("conduit", ""):
|
| 623 |
+
continue
|
| 624 |
+
ratio = pd.to_numeric(pd.Series([r.get("Depth Ratio")]), errors="coerce").iloc[0]
|
| 625 |
+
vel = pd.to_numeric(pd.Series([r.get(vel_col)]), errors="coerce").iloc[0]
|
| 626 |
+
issues = []
|
| 627 |
+
if pd.notna(vel) and vel >= criteria.velocity_threshold:
|
| 628 |
+
issues.append(f"velocity ≥ {criteria.velocity_threshold:g} {units.velocity}")
|
| 629 |
+
if pd.notna(ratio) and ratio >= criteria.conduit_depth_ratio:
|
| 630 |
+
issues.append(f"depth ratio ≥ {criteria.conduit_depth_ratio:.2f}")
|
| 631 |
+
if issues:
|
| 632 |
+
link_rows.append({"Link ID": r.get("Link ID"), "From Node": r.get("From Node"), "To Node": r.get("To Node"),
|
| 633 |
+
f"Peak Flow ({units.flow})": r.get(f"Peak Flow ({units.flow})"), vel_col: vel,
|
| 634 |
+
"Depth Ratio": ratio, "Review Issue": "; ".join(issues)})
|
| 635 |
+
return pd.DataFrame(node_rows), pd.DataFrame(link_rows)
|
| 636 |
+
|
| 637 |
+
def _summary_findings(node_df: pd.DataFrame, link_df: pd.DataFrame, sub_df: pd.DataFrame, metadata: dict[str, Any], units: UnitContext, options: dict[str, str], criteria: ReportCriteria) -> list[str]:
|
| 638 |
+
findings: list[str] = []
|
| 639 |
+
for label, key in (("Runoff quantity continuity error", "runoff_error"), ("Flow-routing continuity error", "flow_error"), ("Water-quality continuity error", "quality_error")):
|
| 640 |
+
val = metadata.get(key)
|
| 641 |
+
if val is not None:
|
| 642 |
+
try:
|
| 643 |
+
v = float(val); findings.append(f"{label}: {v:.3f}%.")
|
| 644 |
+
if key == "flow_error" and abs(v) > criteria.continuity_warning:
|
| 645 |
+
findings.append(f"⚠️ Flow-routing continuity error exceeds the {criteria.continuity_warning:g}% warning threshold and should be reviewed.")
|
| 646 |
+
elif key == "flow_error" and abs(v) > criteria.continuity_review:
|
| 647 |
+
findings.append(f"Flow-routing continuity error exceeds the {criteria.continuity_review:g}% review threshold.")
|
| 648 |
+
except Exception: findings.append(f"{label}: {val}.")
|
| 649 |
+
if node_df is not None and not node_df.empty:
|
| 650 |
+
flooded_col = next((c for c in node_df.columns if c.startswith('Peak Flooding (')), None)
|
| 651 |
+
flooded = int((pd.to_numeric(node_df[flooded_col], errors='coerce').fillna(0) > 0).sum()) if flooded_col else 0
|
| 652 |
+
findings.append(f"Flooded nodes identified: {flooded}.")
|
| 653 |
+
junction_nodes = node_df[node_df.get('Type', '').astype(str).str.lower().isin(['junction', 'divider'])].copy()
|
| 654 |
+
if 'Depth Ratio' in junction_nodes and not junction_nodes.empty:
|
| 655 |
+
ratios = pd.to_numeric(junction_nodes['Depth Ratio'], errors='coerce')
|
| 656 |
+
if ratios.notna().any():
|
| 657 |
+
i = ratios.idxmax(); findings.append(f"Maximum junction depth ratio: {ratios.loc[i]:.3f} at {junction_nodes.loc[i, 'Node ID']}.")
|
| 658 |
+
free_col = f"Freeboard ({units.length})"
|
| 659 |
+
if ratios.loc[i] >= criteria.node_depth_ratio and free_col in junction_nodes:
|
| 660 |
+
findings.append(f"{junction_nodes.loc[i, 'Node ID']} has {_format_value(junction_nodes.loc[i, free_col])} {units.length} of modelled rim clearance and should be reviewed.")
|
| 661 |
+
storage_nodes = node_df[node_df.get('Type', '').astype(str).str.lower().eq('storage')].copy()
|
| 662 |
+
if 'Depth Ratio' in storage_nodes and not storage_nodes.empty:
|
| 663 |
+
ratios = pd.to_numeric(storage_nodes['Depth Ratio'], errors='coerce')
|
| 664 |
+
if ratios.notna().any():
|
| 665 |
+
i = ratios.idxmax(); findings.append(f"Maximum storage depth utilization ratio: {ratios.loc[i]:.3f} at {storage_nodes.loc[i, 'Node ID']}; assess using the applicable storage classification and ponding-depth criterion.")
|
| 666 |
+
conduits = link_df.copy() if link_df is not None else pd.DataFrame()
|
| 667 |
+
if not conduits.empty and 'Type' in conduits:
|
| 668 |
+
conduits = conduits[conduits['Type'].astype(str).str.lower().eq('conduit')]
|
| 669 |
+
if not conduits.empty:
|
| 670 |
+
if "Depth Ratio" in conduits:
|
| 671 |
+
ratios = pd.to_numeric(conduits["Depth Ratio"], errors="coerce")
|
| 672 |
+
if ratios.notna().any():
|
| 673 |
+
i=ratios.idxmax(); findings.append(f"Maximum conduit depth ratio: {ratios.loc[i]:.3f} at {conduits.loc[i, 'Link ID']}.")
|
| 674 |
+
vcol=f"Peak Velocity ({units.velocity})"
|
| 675 |
+
if vcol in conduits:
|
| 676 |
+
vals=pd.to_numeric(conduits[vcol], errors='coerce')
|
| 677 |
+
if vals.notna().any():
|
| 678 |
+
i=vals.idxmax(); findings.append(f"Maximum conduit velocity: {vals.loc[i]:.3f} {units.velocity} at {conduits.loc[i, 'Link ID']}.")
|
| 679 |
+
if sub_df is not None and not sub_df.empty:
|
| 680 |
+
acol=next((c for c in sub_df.columns if c.startswith('Area (')), None)
|
| 681 |
+
if acol: findings.append(f"Total model-database subcatchment area: {pd.to_numeric(sub_df[acol], errors='coerce').sum():.3f} {units.area}.")
|
| 682 |
+
routing = options.get("FLOW_ROUTING", "").upper()
|
| 683 |
+
if routing == "KINWAVE":
|
| 684 |
+
findings.append("⚠️ Kinematic-wave routing does not fully represent backwater, pressurization, reverse flow, or complex surcharge interactions; HGL and surcharge conclusions should be interpreted accordingly.")
|
| 685 |
+
warnings = metadata.get("warnings") or []
|
| 686 |
+
if warnings: findings.append(f"Simulation warnings recorded: {len(warnings)}. Review the attached metadata and model report.")
|
| 687 |
+
return findings
|
| 688 |
+
|
| 689 |
+
|
| 690 |
+
def _event_summary(sub_df: pd.DataFrame, metadata: dict[str, Any], options: dict[str, str], units: UnitContext) -> pd.DataFrame:
|
| 691 |
+
rain_total_col = next((c for c in sub_df.columns if c.startswith("Total Rainfall (")), None) if sub_df is not None else None
|
| 692 |
+
rain_peak_col = next((c for c in sub_df.columns if c.startswith("Peak Rainfall (")), None) if sub_df is not None else None
|
| 693 |
+
total_rain = pd.to_numeric(sub_df[rain_total_col], errors="coerce").max() if rain_total_col and not sub_df.empty else None
|
| 694 |
+
peak_rain = pd.to_numeric(sub_df[rain_peak_col], errors="coerce").max() if rain_peak_col and not sub_df.empty else None
|
| 695 |
+
start = metadata.get("start_time")
|
| 696 |
+
end = metadata.get("end_time")
|
| 697 |
+
duration = ""
|
| 698 |
+
try:
|
| 699 |
+
duration = str(pd.Timestamp(end) - pd.Timestamp(start)) if start and end else ""
|
| 700 |
+
except Exception:
|
| 701 |
+
pass
|
| 702 |
+
rows = [
|
| 703 |
+
{"Parameter": "Design event", "Value": metadata.get("design_storm", "Model design event")},
|
| 704 |
+
{"Parameter": "Simulation start", "Value": start or "Not identified"},
|
| 705 |
+
{"Parameter": "Simulation end", "Value": end or "Not identified"},
|
| 706 |
+
{"Parameter": "Simulation duration", "Value": duration or "Not identified"},
|
| 707 |
+
{"Parameter": "Rainfall / wet-weather timestep", "Value": options.get("WET_STEP", options.get("REPORT_STEP", "Not identified"))},
|
| 708 |
+
{"Parameter": "Reporting timestep", "Value": options.get("REPORT_STEP", "Not identified")},
|
| 709 |
+
]
|
| 710 |
+
if total_rain is not None and pd.notna(total_rain): rows.append({"Parameter": "Total event precipitation", "Value": f"{_format_value(total_rain)} {'in' if units.system == 'US Customary' else 'mm'}"})
|
| 711 |
+
if peak_rain is not None and pd.notna(peak_rain):
|
| 712 |
+
wet_step = str(options.get("WET_STEP", options.get("REPORT_STEP", "")))
|
| 713 |
+
label = "Maximum rainfall-interval intensity"
|
| 714 |
+
if wet_step in {"0:05", "00:05", "0:05:00", "00:05:00", "5 min", "5 minutes"}:
|
| 715 |
+
label = "Maximum 5-minute rainfall intensity"
|
| 716 |
+
rows.append({"Parameter": label, "Value": f"{_format_value(peak_rain)} {units.rainfall}"})
|
| 717 |
+
return pd.DataFrame(rows)
|
| 718 |
+
|
| 719 |
+
|
| 720 |
+
def _executive_summary(findings: list[str], critical_nodes: pd.DataFrame, critical_links: pd.DataFrame, units: UnitContext, criteria: ReportCriteria) -> list[str]:
|
| 721 |
+
lines = ["The simulation completed and the principal model results were screened using the project criteria listed in this report."]
|
| 722 |
+
if critical_nodes.empty:
|
| 723 |
+
lines.append(f"No junctions or dividers met the generic node screening criteria (depth ratio ≥ {criteria.node_depth_ratio:.2f}, rim clearance ≤ {criteria.minimum_freeboard:g} {units.length}, or flooding greater than zero). Storage facilities are assessed separately using their Calgary storage classification.")
|
| 724 |
+
else:
|
| 725 |
+
lines.append(f"{len(critical_nodes)} junction/divider node(s) met the generic node screening criteria; storage facilities are assessed separately using their Calgary storage classification.")
|
| 726 |
+
if critical_links.empty:
|
| 727 |
+
lines.append(f"No conduits met the critical-link screening criteria (depth ratio ≥ {criteria.conduit_depth_ratio:.2f} or velocity ≥ {criteria.velocity_threshold:g} {units.velocity}).")
|
| 728 |
+
else:
|
| 729 |
+
lines.append(f"{len(critical_links)} conduit(s) met the critical-link screening criteria, primarily due to velocity or depth utilization.")
|
| 730 |
+
warning_lines = [x for x in findings if "⚠️" in x]
|
| 731 |
+
if warning_lines:
|
| 732 |
+
lines.append("At least one numerical or modelling limitation requires review before the report is relied upon for design conclusions.")
|
| 733 |
+
return lines
|
| 734 |
+
|
| 735 |
+
|
| 736 |
+
def _provided(value: Any) -> bool:
|
| 737 |
+
return bool(str(value or "").strip()) and str(value).strip().lower() not in {"not provided", "n/a", "none", "unknown"}
|
| 738 |
+
|
| 739 |
+
def _build_swmr_checklist(metadata: ReportMetadata, criteria: ReportCriteria, *, has_model: bool, has_results: bool, has_storage: bool, has_controls: bool, has_overland: bool, has_outfalls: bool, has_conduits: bool) -> pd.DataFrame:
|
| 740 |
+
drawings = {x.strip().lower() for x in criteria.drawing_inventory}
|
| 741 |
+
reports = list(criteria.applicable_reports)
|
| 742 |
+
rows = []
|
| 743 |
+
def add(i, req, status, evidence, action, category):
|
| 744 |
+
override=(criteria.checklist_overrides or {}).get(i)
|
| 745 |
+
rows.append({"Item":i,"Category":category,"Requirement":req,"Status":override or status,"Report Evidence":evidence,"Outstanding Action":action})
|
| 746 |
+
admin_ok=all(_provided(x) for x in [metadata.project_name, metadata.client, metadata.consultant, metadata.outline_plan_no, metadata.prepared_by])
|
| 747 |
+
add("SWMR-01","Project, developer, consultant, planning and professional information", "Complete" if admin_ok else "Partially complete", "Cover page", "Complete missing administrative and professional fields", "Administration")
|
| 748 |
+
add("SWMR-02","Cover letter, circulation status, unresolved matters and departures", "Partially complete", "Outstanding Information and Actions", "Prepare signed cover letter and identify unresolved matters", "Administration")
|
| 749 |
+
add("SWMR-03","Applicable MDP/SMDP, pond report, prior SWMR and downstream reports", "Complete" if reports else "Missing", "Applicable Reports Register" if reports else "—", "Upload or list applicable drainage documents", "Criteria")
|
| 750 |
+
add("SWMR-04","Study area, legal description, adjacent lands, external drainage and location figure", "Partially complete" if _provided(metadata.legal_description) else "Missing", "Section 2 / model area table", "Add legal description, site location, external drainage and study-area figure", "Site")
|
| 751 |
+
add("SWMR-05","Design objectives and verified project criteria", "Partially complete", "Project Criteria Register", "Confirm current City amendments and project-specific criteria", "Criteria")
|
| 752 |
+
add("SWMR-06","Model methodology, software, routing, infiltration, storm, timesteps and continuity", "Complete" if has_model and has_results else "Missing", "Section 3 and design-event table", "Provide model/results" if not has_results else "Professional confirmation", "Methodology")
|
| 753 |
+
add("SWMR-07","Subcatchment boundaries, areas, imperviousness, widths, slopes and outlets", "Complete" if has_model else "Missing", "Catchment tables", "Confirm against drainage drawings", "Hydrology")
|
| 754 |
+
if has_conduits:
|
| 755 |
+
add("SWMR-08","Minor-system routed flows, cumulative design flows, pipe capacities and spare capacity", "Partially complete" if has_results else "Missing", "Minor-system tables", "Verify release rate and full-flow capacities", "Minor System")
|
| 756 |
+
add("SWMR-09","HGL, surcharge, rim clearance, downstream HWL and backwater assessment", "Partially complete" if has_results else "Missing", "Node HGL table", "Confirm downstream HWL and prepare profiles where required", "Minor System")
|
| 757 |
+
else:
|
| 758 |
+
add("SWMR-08","Minor-system routed flows, cumulative design flows, pipe capacities and spare capacity", "Not applicable", "No conduits identified in uploaded model", "Confirm whether a separate minor-system model is within the report scope", "Minor System")
|
| 759 |
+
add("SWMR-09","HGL, surcharge, rim clearance, downstream HWL and backwater assessment", "Requires professional confirmation", "Storage/outfall HGL table; no conduit network identified", "Confirm whether a separate minor-system model and downstream HWL assessment are required", "Minor System")
|
| 760 |
+
add("SWMR-10","Catchbasin, inlet, ICD and outlet rating information", "Complete" if has_controls else "Not applicable / missing", "Hydraulic controls table" if has_controls else "—", "Confirm inlet types, rating curves and drawing locations", "Minor System")
|
| 761 |
+
add("SWMR-11","Critical overland flows, depths, velocities, spill routes and escape routes", "Partially complete" if has_overland else "Requires drawing review", "Major-system depth-velocity table" if has_overland else "No overland route identified in model", "Confirm overland route, grading, containment, escape routes and safety on drawings", "Major System")
|
| 762 |
+
add("SWMR-12","Trap-low and storage volume, depth, spill, entrance grade and restrictive covenant information", "Partially complete" if has_storage else "Not applicable", "Storage assessment" if has_storage else "—", "Add spill elevations, building grades and covenant requirements", "Storage")
|
| 763 |
+
add("SWMR-13","Minor and major boundary inflows/outflows and supporting source", "Partially complete" if has_outfalls else "Missing", "Boundary outflow tables" if has_outfalls else "—", "Confirm downstream capacity and external inflows", "Boundary Conditions")
|
| 764 |
+
add("SWMR-14","Private-site permissible discharge and on-site storage requirements", "Missing", "—", "Provide applicable release rates and private-site storage criteria", "Private Sites")
|
| 765 |
+
add("SWMR-15","Water-quality treatment, BMPs, downstream treatment and source controls", "Missing", "—", "Document water-quality strategy and downstream treatment", "Water Quality")
|
| 766 |
+
required_drawings={"site location","study area","catchment plan","model schematic","overland drainage","storm drainage"}
|
| 767 |
+
found=len(required_drawings & drawings)
|
| 768 |
+
add("SWMR-16","Required figures and drawings", "Complete" if found==len(required_drawings) else ("Partially complete" if found else "Missing"), f"{found}/{len(required_drawings)} core drawings recorded", "Add missing site, catchment, schematic, overland and storm-drainage drawings", "Drawings")
|
| 769 |
+
add("SWMR-17","Model input/output files, formatted listings, model schematic, drawing reconciliation and auditable digital package", "Partially complete" if has_model and has_results else "Missing", "Digital model package and structured appendices", "Add fixed-width input/output listings, model schematic, drawing-to-model cross-reference, revision metadata and final authenticated files", "Appendices")
|
| 770 |
+
return pd.DataFrame(rows)
|
| 771 |
+
|
| 772 |
+
def _readiness_scores(checklist: pd.DataFrame) -> pd.DataFrame:
|
| 773 |
+
weights={"Complete":1.0,"Partially complete":0.5,"Requires professional confirmation":0.5,"Requires drawing review":0.5,"Not applicable":1.0,"Not applicable / missing":0.5,"Missing":0.0}
|
| 774 |
+
cats={"Model data completeness":["Methodology","Hydrology","Appendices"],"Hydraulic-result completeness":["Minor System","Major System","Storage","Boundary Conditions"],"Project information":["Administration","Site"],"Drawing completeness":["Drawings"],"Criteria verification":["Criteria","Private Sites","Water Quality"]}
|
| 775 |
+
rows=[]
|
| 776 |
+
for name,groups in cats.items():
|
| 777 |
+
d=checklist[checklist["Category"].isin(groups)]
|
| 778 |
+
score=100*sum(weights.get(str(x),0.25) for x in d["Status"])/max(len(d),1)
|
| 779 |
+
rows.append({"Readiness Dimension":name,"Score (%)":round(score,1)})
|
| 780 |
+
rows.append({"Readiness Dimension":"SWMR draft readiness","Score (%)":round(sum(r["Score (%)"] for r in rows)/len(rows),1)})
|
| 781 |
+
return pd.DataFrame(rows)
|
| 782 |
+
|
| 783 |
+
def _add_narrative_block(doc: Document, narrative_sections: Mapping[str, str] | None, key: str) -> bool:
|
| 784 |
+
"""Insert an approved narrative block while preserving simple paragraph/list structure."""
|
| 785 |
+
if not narrative_sections:
|
| 786 |
+
return False
|
| 787 |
+
text = str(narrative_sections.get(key, "") or "").strip()
|
| 788 |
+
if not text:
|
| 789 |
+
return False
|
| 790 |
+
for raw in text.splitlines():
|
| 791 |
+
line = raw.strip()
|
| 792 |
+
if not line:
|
| 793 |
+
continue
|
| 794 |
+
# Section headings are controlled by the deterministic report template.
|
| 795 |
+
if line.startswith("#"):
|
| 796 |
+
continue
|
| 797 |
+
if line.startswith(("- ", "* ")):
|
| 798 |
+
doc.add_paragraph(line[2:].strip(), style="List Bullet")
|
| 799 |
+
elif re.match(r"^\d+[.)]\s+", line):
|
| 800 |
+
doc.add_paragraph(re.sub(r"^\d+[.)]\s+", "", line), style="List Number")
|
| 801 |
+
else:
|
| 802 |
+
doc.add_paragraph(line)
|
| 803 |
+
return True
|
| 804 |
+
|
| 805 |
+
|
| 806 |
+
|
| 807 |
+
def _scenario_inp_sections(record: Mapping[str, Any]) -> dict[str, list[list[str]]]:
|
| 808 |
+
raw = (record.get("files", {}) or {}).get("inp", b"")
|
| 809 |
+
if isinstance(raw, bytes):
|
| 810 |
+
text = raw.decode("utf-8", errors="ignore")
|
| 811 |
+
else:
|
| 812 |
+
text = str(raw or "")
|
| 813 |
+
sections: dict[str, list[list[str]]] = {}
|
| 814 |
+
current = None
|
| 815 |
+
for original in text.splitlines():
|
| 816 |
+
line = original.strip()
|
| 817 |
+
if line.startswith("[") and line.endswith("]"):
|
| 818 |
+
current = line[1:-1].strip().upper()
|
| 819 |
+
sections.setdefault(current, [])
|
| 820 |
+
continue
|
| 821 |
+
if not current or not line or line.startswith(";"):
|
| 822 |
+
continue
|
| 823 |
+
data = line.split(";", 1)[0].strip()
|
| 824 |
+
if data:
|
| 825 |
+
sections[current].append(data.split())
|
| 826 |
+
return sections
|
| 827 |
+
|
| 828 |
+
def _series_peak(values: Mapping[str, Any], key: str, absolute: bool = False) -> float:
|
| 829 |
+
seq = values.get(key, []) or []
|
| 830 |
+
nums=[]
|
| 831 |
+
for v in seq:
|
| 832 |
+
try:
|
| 833 |
+
x=float(v); nums.append(abs(x) if absolute else x)
|
| 834 |
+
except (TypeError, ValueError):
|
| 835 |
+
pass
|
| 836 |
+
return max(nums, default=0.0)
|
| 837 |
+
|
| 838 |
+
def _safe_float(value: Any) -> float | None:
|
| 839 |
+
try:
|
| 840 |
+
return float(value)
|
| 841 |
+
except (TypeError, ValueError):
|
| 842 |
+
return None
|
| 843 |
+
|
| 844 |
+
|
| 845 |
+
def _peak_time(values: Mapping[str, Any], key: str, times: list[Any]) -> str:
|
| 846 |
+
seq = values.get(key, []) or []
|
| 847 |
+
if not seq:
|
| 848 |
+
return "Not available"
|
| 849 |
+
numeric=[]
|
| 850 |
+
for i, value in enumerate(seq):
|
| 851 |
+
try:
|
| 852 |
+
numeric.append((float(value), i))
|
| 853 |
+
except (TypeError, ValueError):
|
| 854 |
+
continue
|
| 855 |
+
if not numeric:
|
| 856 |
+
return "Not available"
|
| 857 |
+
_, idx=max(numeric, key=lambda x: x[0])
|
| 858 |
+
if idx < len(times):
|
| 859 |
+
return str(times[idx])
|
| 860 |
+
return str(idx)
|
| 861 |
+
|
| 862 |
+
|
| 863 |
+
def _event_short_label(row: Mapping[str, Any]) -> str:
|
| 864 |
+
role=str(row.get("Model Role", "Scenario") or "Scenario")
|
| 865 |
+
storm=str(row.get("Storm", "Model event") or "Model event")
|
| 866 |
+
low=storm.lower().replace("_", " ")
|
| 867 |
+
match=re.search(r"(?:^|\D)(\d+)\s*(?:y|yr|year)", low)
|
| 868 |
+
if not match:
|
| 869 |
+
match=re.search(r"(\d+)y", low.replace(" ", ""))
|
| 870 |
+
event=f"{match.group(1)}-Year" if match else storm
|
| 871 |
+
prefix="Base" if role.lower()=="base" else "Scenario"
|
| 872 |
+
return f"{prefix} – {event}"
|
| 873 |
+
|
| 874 |
+
|
| 875 |
+
def _scenario_detail_tables(record: Mapping[str, Any], units: UnitContext) -> dict[str, pd.DataFrame]:
|
| 876 |
+
results = record.get("results", {}) or {}
|
| 877 |
+
definition = record.get("definition", {}) or {}
|
| 878 |
+
summary = record.get("summary", {}) or {}
|
| 879 |
+
sections = _scenario_inp_sections(record)
|
| 880 |
+
storage_rows_inp = {r[0]: r for r in sections.get("STORAGE", []) if r}
|
| 881 |
+
storage_ids = set(storage_rows_inp)
|
| 882 |
+
outfall_ids = {r[0] for r in sections.get("OUTFALLS", []) if r}
|
| 883 |
+
conduit_ids = {r[0] for r in sections.get("CONDUITS", []) if r}
|
| 884 |
+
outlet_ids = {r[0] for r in sections.get("OUTLETS", []) if r}
|
| 885 |
+
node_ts = results.get("node_ts", {}) or {}
|
| 886 |
+
link_ts = results.get("link_ts", {}) or {}
|
| 887 |
+
sub_ts = results.get("sub_ts", {}) or {}
|
| 888 |
+
times = list(results.get("times", []) or [])
|
| 889 |
+
|
| 890 |
+
overview = pd.DataFrame([{
|
| 891 |
+
"Scenario": definition.get("scenario_name", summary.get("Scenario Name", "Scenario")),
|
| 892 |
+
"Source Model": summary.get("Source Model", (record.get("manifest", {}) or {}).get("base_model_name", "Not identified")),
|
| 893 |
+
"Rainfall Event": (definition.get("storm", {}) or {}).get("name", summary.get("Storm", "Model rainfall")),
|
| 894 |
+
"Storm Status": summary.get("Storm Status", (definition.get("storm", {}) or {}).get("source_status", "Not verified")),
|
| 895 |
+
"Runoff Error (%)": summary.get("Runoff Error (%)"),
|
| 896 |
+
"Flow Error (%)": summary.get("Flow Error (%)"),
|
| 897 |
+
f"Peak Subcatchment Runoff ({units.flow})": summary.get("Peak Subcatchment Runoff"),
|
| 898 |
+
f"Peak Link/Control Flow ({units.flow})": summary.get("Peak Link Flow"),
|
| 899 |
+
f"Maximum Node Inflow ({units.flow})": summary.get("Maximum Node Inflow"),
|
| 900 |
+
f"Maximum Node Flooding ({units.flow})": summary.get("Maximum Node Flooding"),
|
| 901 |
+
"Maximum Conduit Velocity": (summary.get("Maximum Link Velocity") if conduit_ids else "Not applicable — no conduits in model"),
|
| 902 |
+
"Maximum Conduit Depth Ratio": (summary.get("Maximum Modelled Depth Ratio") if conduit_ids else "Not applicable — no conduits in model"),
|
| 903 |
+
}])
|
| 904 |
+
|
| 905 |
+
sub_rows=[]
|
| 906 |
+
for sid, vals in sub_ts.items():
|
| 907 |
+
sub_rows.append({"Subcatchment": sid, f"Peak Runoff ({units.flow})": _series_peak(vals,"runoff"), f"Peak Rainfall ({units.rainfall})": _series_peak(vals,"rainfall")})
|
| 908 |
+
|
| 909 |
+
storage_rows=[]
|
| 910 |
+
for nid in sorted(storage_ids):
|
| 911 |
+
vals=node_ts.get(nid,{})
|
| 912 |
+
inp=storage_rows_inp.get(nid, [])
|
| 913 |
+
invert=_safe_float(inp[1]) if len(inp)>1 else None
|
| 914 |
+
max_depth=_safe_float(inp[2]) if len(inp)>2 else None
|
| 915 |
+
peak_depth=_series_peak(vals,"depth")
|
| 916 |
+
peak_head=_series_peak(vals,"head")
|
| 917 |
+
if not peak_head and invert is not None:
|
| 918 |
+
peak_head=invert+peak_depth
|
| 919 |
+
peak_volume=_series_peak(vals,"volume")
|
| 920 |
+
remaining=(max_depth-peak_depth) if max_depth is not None else None
|
| 921 |
+
utilization=(100.0*peak_depth/max_depth) if max_depth and max_depth>0 else None
|
| 922 |
+
storage_rows.append({
|
| 923 |
+
"Storage ID":nid,
|
| 924 |
+
f"Peak Depth ({units.length})":peak_depth,
|
| 925 |
+
f"Maximum HGL ({units.length})":peak_head,
|
| 926 |
+
f"Maximum Stored Volume ({units.storage})":peak_volume,
|
| 927 |
+
f"Peak Inflow ({units.flow})":_series_peak(vals,"inflow"),
|
| 928 |
+
f"Peak Outflow ({units.flow})":_series_peak(vals,"outflow"),
|
| 929 |
+
f"Peak Flooding ({units.flow})":_series_peak(vals,"flooding"),
|
| 930 |
+
"Time of Peak Storage":_peak_time(vals,"volume",times),
|
| 931 |
+
"Depth Utilization (%)":utilization,
|
| 932 |
+
f"Remaining Depth Margin ({units.length})":remaining,
|
| 933 |
+
})
|
| 934 |
+
|
| 935 |
+
outfall_rows=[]
|
| 936 |
+
for nid in sorted(outfall_ids):
|
| 937 |
+
vals=node_ts.get(nid,{})
|
| 938 |
+
outfall_rows.append({"Outfall ID":nid, f"Peak Depth ({units.length})":_series_peak(vals,"depth"), f"Peak Inflow ({units.flow})":_series_peak(vals,"inflow"), f"Peak Flooding ({units.flow})":_series_peak(vals,"flooding")})
|
| 939 |
+
control_rows=[]
|
| 940 |
+
for lid in sorted(outlet_ids):
|
| 941 |
+
vals=link_ts.get(lid,{})
|
| 942 |
+
control_rows.append({"Control ID":lid, f"Peak Flow ({units.flow})":_series_peak(vals,"flow",True), f"Peak Depth ({units.length})":_series_peak(vals,"depth"), "Time of Peak Flow":_peak_time(vals,"flow",times)})
|
| 943 |
+
conduit_rows=[]
|
| 944 |
+
for lid in sorted(conduit_ids):
|
| 945 |
+
vals=link_ts.get(lid,{})
|
| 946 |
+
diameter=float(vals.get("diameter",0) or 0)
|
| 947 |
+
depth=_series_peak(vals,"depth")
|
| 948 |
+
conduit_rows.append({"Conduit ID":lid, f"Peak Flow ({units.flow})":_series_peak(vals,"flow",True), f"Peak Velocity ({units.velocity})":_series_peak(vals,"velocity",True), "Modelled Depth Ratio": depth/diameter if diameter>0 else None})
|
| 949 |
+
return {"overview":overview,"subcatchments":pd.DataFrame(sub_rows),"storage":pd.DataFrame(storage_rows),"outfalls":pd.DataFrame(outfall_rows),"controls":pd.DataFrame(control_rows),"conduits":pd.DataFrame(conduit_rows)}
|
| 950 |
+
|
| 951 |
+
def _compact_scenario_comparison(df: pd.DataFrame) -> pd.DataFrame:
|
| 952 |
+
if df is None or df.empty:
|
| 953 |
+
return pd.DataFrame()
|
| 954 |
+
out=df.copy()
|
| 955 |
+
out.insert(0, "Display Scenario", [_event_short_label(row) for _, row in out.iterrows()])
|
| 956 |
+
wanted=["Display Scenario","Source Model","Storm","Storm Status","Simulation Status","Runoff Error (%)","Flow Error (%)","Peak Subcatchment Runoff","Maximum Storage Depth","Maximum Storage Volume","Peak Link Flow","Maximum Node Inflow","Maximum Node Flooding","Hydraulic Difference"]
|
| 957 |
+
return out[[c for c in wanted if c in out.columns]].copy()
|
| 958 |
+
|
| 959 |
+
def generate_report_package(
|
| 960 |
+
*,
|
| 961 |
+
metadata: ReportMetadata,
|
| 962 |
+
inp_sections: dict[str, list[list[str]]],
|
| 963 |
+
node_summary: pd.DataFrame,
|
| 964 |
+
link_summary: pd.DataFrame,
|
| 965 |
+
sub_summary: pd.DataFrame,
|
| 966 |
+
simulation_metadata: dict[str, Any],
|
| 967 |
+
result_db_bytes: bytes | None = None,
|
| 968 |
+
criteria: ReportCriteria | None = None,
|
| 969 |
+
narrative_sections: Mapping[str, str] | None = None,
|
| 970 |
+
scenario_comparison: pd.DataFrame | None = None,
|
| 971 |
+
scenario_analysis: str | None = None,
|
| 972 |
+
scenario_records: list[Mapping[str, Any]] | None = None,
|
| 973 |
+
scenario_reporting_mode: str = "Base report with scenario comparison",
|
| 974 |
+
preliminary_review_artifacts: Mapping[str, Any] | None = None,
|
| 975 |
+
) -> dict[str, bytes | str]:
|
| 976 |
+
"""Generate editable Word report and ZIP package entirely in memory."""
|
| 977 |
+
criteria = criteria or ReportCriteria()
|
| 978 |
+
options = _inp_options(inp_sections)
|
| 979 |
+
units = _unit_context(options.get("FLOW_UNITS", ""))
|
| 980 |
+
sub_table = _subcatchment_model_table(inp_sections, sub_summary)
|
| 981 |
+
link_table = _link_model_table(inp_sections, link_summary)
|
| 982 |
+
hgl_table = _node_hgl_table(node_summary)
|
| 983 |
+
|
| 984 |
+
# The simulation arrays are retained in the model's native unit system.
|
| 985 |
+
# Rename report headers to match that system; do not convert values.
|
| 986 |
+
sub_table = _rename_native_columns(sub_table, units)
|
| 987 |
+
link_table = _rename_native_columns(link_table, units)
|
| 988 |
+
hgl_table = _rename_native_columns(hgl_table, units)
|
| 989 |
+
sub_table = sub_table.loc[:, ~sub_table.columns.duplicated()].copy()
|
| 990 |
+
link_table = link_table.loc[:, ~link_table.columns.duplicated()].copy()
|
| 991 |
+
hgl_table = hgl_table.loc[:, ~hgl_table.columns.duplicated()].copy()
|
| 992 |
+
node_report = _rename_native_columns(node_summary, units)
|
| 993 |
+
link_report = _rename_native_columns(link_summary, units)
|
| 994 |
+
sub_report = _rename_native_columns(sub_summary, units)
|
| 995 |
+
|
| 996 |
+
report_step_s = _parse_time_seconds(options.get("REPORT_STEP"))
|
| 997 |
+
sub_report = _enhance_subcatchment_results(sub_report, units, report_step_s)
|
| 998 |
+
sub_table = _enhance_subcatchment_results(sub_table, units, report_step_s)
|
| 999 |
+
node_report = _populate_outfall_flows(node_report, link_report, units)
|
| 1000 |
+
db_tables = _read_sqlite_tables(result_db_bytes, ["node_timeseries", "link_timeseries"])
|
| 1001 |
+
control_table = _control_table(inp_sections, link_table, units)
|
| 1002 |
+
control_recon = _control_reconciliation(control_table, link_table, units)
|
| 1003 |
+
storage_table = _storage_table(inp_sections, node_report, db_tables, units)
|
| 1004 |
+
if not storage_table.empty and not control_table.empty:
|
| 1005 |
+
peak_col = f"Peak Flow ({units.flow})"
|
| 1006 |
+
for idx in storage_table.index:
|
| 1007 |
+
sid = storage_table.at[idx, "Storage ID"]
|
| 1008 |
+
outgoing_peak = pd.to_numeric(control_table.loc[control_table["From Node"].astype(str) == str(sid), peak_col], errors="coerce").fillna(0).sum() if peak_col in control_table else 0.0
|
| 1009 |
+
depth_col = next((c for c in storage_table.columns if c.startswith("Time-Series Peak Depth (")), None)
|
| 1010 |
+
peak_depth = storage_table.at[idx, depth_col] if depth_col else storage_table.at[idx, f"Reported Peak Depth ({units.length})"]
|
| 1011 |
+
if outgoing_peak > 0 and (pd.isna(peak_depth) or float(peak_depth) <= 0):
|
| 1012 |
+
storage_table.at[idx, "Status"] = "Review: control flow occurs but storage depth is zero"
|
| 1013 |
+
area_table = _area_classification(sub_table, units, criteria.area_classification)
|
| 1014 |
+
critical_nodes, critical_links = _critical_elements(_rename_native_columns(_node_hgl_table(node_report), units), link_report, units, criteria)
|
| 1015 |
+
|
| 1016 |
+
calgary = CalgaryCriteria(
|
| 1017 |
+
minor_release_rate_lps_ha=criteria.minor_release_rate_lps_ha,
|
| 1018 |
+
trap_low_max_depth_m=criteria.trap_low_max_depth_m,
|
| 1019 |
+
entrance_grade_margin_m=criteria.entrance_grade_margin_m,
|
| 1020 |
+
pipe_critical_velocity_mps=criteria.velocity_threshold if units.system == "SI" else 4.0,
|
| 1021 |
+
conduit_capacity_review_ratio=criteria.conduit_depth_ratio,
|
| 1022 |
+
conduit_capacity_warning_ratio=criteria.conduit_capacity_warning_ratio,
|
| 1023 |
+
continuity_review_pct=criteria.continuity_review,
|
| 1024 |
+
continuity_warning_pct=criteria.continuity_warning,
|
| 1025 |
+
special_link_limits=criteria.special_link_limits or {},
|
| 1026 |
+
storage_classification=criteria.storage_classification or {},
|
| 1027 |
+
outfall_classification=criteria.outfall_classification or {},
|
| 1028 |
+
)
|
| 1029 |
+
rain_gages = [r[1] for r in inp_sections.get("SUBCATCHMENTS", []) if len(r) > 1]
|
| 1030 |
+
inferred_event, inferred_event_note = infer_design_event(rain_gages, metadata.design_storm)
|
| 1031 |
+
if metadata.design_storm.strip().lower() in {"model design event", "not provided", ""}:
|
| 1032 |
+
metadata.design_storm = inferred_event
|
| 1033 |
+
criteria_table = criteria_register(calgary)
|
| 1034 |
+
# Calgary-specific calculations use SI design rules. In US models, tables remain available but are flagged for review.
|
| 1035 |
+
minor_capacity = build_minor_system_capacity_table(link_table, units.system, units.flow, units.length, calgary)
|
| 1036 |
+
# Checklist applicability is based on actual model element types, not on
|
| 1037 |
+
# whether the generic link table happens to contain outlet controls.
|
| 1038 |
+
_conduit_rows = inp_sections.get("CONDUITS", []) or []
|
| 1039 |
+
_xsection_rows = {str(r[0]): str(r[1]).upper() for r in (inp_sections.get("XSECTIONS", []) or []) if len(r) > 1}
|
| 1040 |
+
_open_shapes = {"TRAPEZOIDAL", "RECT_OPEN", "TRIANGULAR", "IRREGULAR", "STREET"}
|
| 1041 |
+
_has_overland = any(str(r[0]) in _xsection_rows and _xsection_rows[str(r[0])] in _open_shapes for r in _conduit_rows if r)
|
| 1042 |
+
checklist_table = _build_swmr_checklist(
|
| 1043 |
+
metadata,
|
| 1044 |
+
criteria,
|
| 1045 |
+
has_model=bool(inp_sections),
|
| 1046 |
+
has_results=not node_report.empty,
|
| 1047 |
+
has_storage=not storage_table.empty,
|
| 1048 |
+
has_controls=not control_table.empty,
|
| 1049 |
+
has_overland=_has_overland,
|
| 1050 |
+
has_outfalls=bool((node_report.get("Type", pd.Series(dtype=str)).astype(str).str.lower()=="outfall").any()),
|
| 1051 |
+
has_conduits=bool(_conduit_rows),
|
| 1052 |
+
)
|
| 1053 |
+
readiness_table = _readiness_scores(checklist_table)
|
| 1054 |
+
|
| 1055 |
+
doc = Document()
|
| 1056 |
+
sec = doc.sections[0]
|
| 1057 |
+
sec.top_margin = Inches(0.65)
|
| 1058 |
+
sec.bottom_margin = Inches(0.65)
|
| 1059 |
+
sec.left_margin = Inches(0.65)
|
| 1060 |
+
sec.right_margin = Inches(0.65)
|
| 1061 |
+
|
| 1062 |
+
styles = doc.styles
|
| 1063 |
+
styles["Normal"].font.name = "Arial"
|
| 1064 |
+
styles["Normal"].font.size = Pt(9)
|
| 1065 |
+
for sname in ("Title", "Heading 1", "Heading 2", "Heading 3"):
|
| 1066 |
+
styles[sname].font.name = "Arial"
|
| 1067 |
+
styles[sname].font.color.rgb = RGBColor(31, 78, 121)
|
| 1068 |
+
|
| 1069 |
+
title = doc.add_paragraph()
|
| 1070 |
+
title.alignment = WD_ALIGN_PARAGRAPH.CENTER
|
| 1071 |
+
r = title.add_run("STORMWATER MANAGEMENT REPORT")
|
| 1072 |
+
r.bold = True; r.font.size = Pt(20); r.font.color.rgb = RGBColor(31, 78, 121)
|
| 1073 |
+
p = doc.add_paragraph(metadata.project_name)
|
| 1074 |
+
p.alignment = WD_ALIGN_PARAGRAPH.CENTER
|
| 1075 |
+
p.runs[0].bold = True; p.runs[0].font.size = Pt(16)
|
| 1076 |
+
|
| 1077 |
+
cover = doc.add_table(rows=0, cols=2); cover.style = "Table Grid"
|
| 1078 |
+
for label, value in [
|
| 1079 |
+
("Subdivision (SB) #", metadata.subdivision_no), ("Outline Plan #", metadata.outline_plan_no),
|
| 1080 |
+
("Development Permit #", metadata.development_permit_no), ("Prepared for", metadata.client),
|
| 1081 |
+
("Prepared by", metadata.consultant), ("Consultant file number", metadata.consultant_file_no),
|
| 1082 |
+
("Responsible engineer", metadata.prepared_by), ("Checked by", metadata.checked_by),
|
| 1083 |
+
("Contact name", metadata.contact_name), ("Contact email", metadata.contact_email),
|
| 1084 |
+
("Legal description", metadata.legal_description), ("Circulation status", metadata.submission_status),
|
| 1085 |
+
("Construction drawing no.", metadata.construction_drawing_no), ("Development Agreement no.", metadata.development_agreement_no),
|
| 1086 |
+
("Report date", metadata.report_date), ("Reporting profile", metadata.municipality),
|
| 1087 |
+
]:
|
| 1088 |
+
cells = cover.add_row().cells
|
| 1089 |
+
_set_cell_text(cells[0], label, bold=True, size=9)
|
| 1090 |
+
_set_cell_text(cells[1], value, size=9)
|
| 1091 |
+
|
| 1092 |
+
doc.add_page_break()
|
| 1093 |
+
doc.add_heading("1.0 INTRODUCTION", level=1)
|
| 1094 |
+
if not _add_narrative_block(doc, narrative_sections, "introduction"):
|
| 1095 |
+
doc.add_paragraph(
|
| 1096 |
+
f"This report summarizes the hydrologic and hydraulic analysis for {metadata.project_name}. "
|
| 1097 |
+
"The document was generated from the deterministic EPA SWMM/OpenSWMM simulation results in the SWMM6 GIS Tool. "
|
| 1098 |
+
"Project-specific planning, survey, geotechnical, environmental, drawing, and approval information must be verified by the responsible professional engineer."
|
| 1099 |
+
)
|
| 1100 |
+
|
| 1101 |
+
doc.add_paragraph("Draft status: suitable for consultant/client discussion and structured engineering review. Human-in-the-loop completion, technical verification, drawing coordination, and professional authentication are required before municipal submission.")
|
| 1102 |
+
doc.add_heading("1.1 Calgary SWMR Checklist Summary", level=2)
|
| 1103 |
+
doc.add_paragraph("The completeness register tracks the principal Calgary SWMR checklist subjects. It is a draft-readiness tool, not a municipal compliance score.")
|
| 1104 |
+
_add_df_table(doc, "Table 1A - Calgary SWMR Completeness Register", checklist_table, landscape=True, font_size=7.0)
|
| 1105 |
+
_add_df_table(doc, "Table 1B - Draft Readiness Summary", readiness_table, font_size=8.0)
|
| 1106 |
+
if criteria.applicable_reports:
|
| 1107 |
+
doc.add_heading("1.2 Applicable Reports Register", level=2)
|
| 1108 |
+
for item in criteria.applicable_reports: doc.add_paragraph(item, style="List Bullet")
|
| 1109 |
+
_add_narrative_block(doc, narrative_sections, "applicable_criteria")
|
| 1110 |
+
|
| 1111 |
+
doc.add_heading("2.0 SITE DESCRIPTION AND DESIGN CRITERIA", level=1)
|
| 1112 |
+
_add_narrative_block(doc, narrative_sections, "site_description")
|
| 1113 |
+
native_area_col = f"Area ({units.area})"
|
| 1114 |
+
total_area = float(sub_report[native_area_col].sum()) if not sub_report.empty and native_area_col in sub_report else 0.0
|
| 1115 |
+
weighted_imp = 0.0
|
| 1116 |
+
if not sub_report.empty and total_area > 0 and "% Impervious" in sub_report:
|
| 1117 |
+
weighted_imp = float((sub_report[native_area_col] * sub_report["% Impervious"]).sum() / total_area)
|
| 1118 |
+
doc.add_paragraph(
|
| 1119 |
+
f"The model database contains {len(sub_summary) if sub_summary is not None else 0} subcatchments, "
|
| 1120 |
+
f"{len(node_summary) if node_summary is not None else 0} nodes, and {len(link_summary) if link_summary is not None else 0} links. "
|
| 1121 |
+
f"The combined model-database subcatchment area is {total_area:.3f} {units.area} and the area-weighted imperviousness is approximately {weighted_imp:.1f}%. "
|
| 1122 |
+
"Pre-development/comparison and external catchments are listed separately below where identifiable from their names."
|
| 1123 |
+
)
|
| 1124 |
+
_add_df_table(doc, "Table 2A - Model Area Classification", area_table, font_size=8.2)
|
| 1125 |
+
doc.add_heading("2.1 Design Objectives", level=2)
|
| 1126 |
+
for text in [
|
| 1127 |
+
"Confirm minor-system flows remain within selected hydraulic criteria.",
|
| 1128 |
+
"Assess overland conveyance depth and velocity against the applicable municipal criteria.",
|
| 1129 |
+
"Identify surcharge, flooding, instability, and continuity concerns.",
|
| 1130 |
+
"Confirm boundary outflows and runoff volumes for the selected design event.",
|
| 1131 |
+
]:
|
| 1132 |
+
doc.add_paragraph(text, style="List Bullet")
|
| 1133 |
+
|
| 1134 |
+
doc.add_heading("3.0 ANALYSIS METHODOLOGY AND DATA", level=1)
|
| 1135 |
+
_add_narrative_block(doc, narrative_sections, "methodology")
|
| 1136 |
+
doc.add_heading("3.1 Design Storm", level=2)
|
| 1137 |
+
simulation_metadata = dict(simulation_metadata or {})
|
| 1138 |
+
simulation_metadata["design_storm"] = metadata.design_storm
|
| 1139 |
+
event_table = _event_summary(sub_report, simulation_metadata, options, units)
|
| 1140 |
+
_add_df_table(doc, "Table 3A - Design Event Summary", event_table, font_size=8.2)
|
| 1141 |
+
doc.add_paragraph(inferred_event_note)
|
| 1142 |
+
if criteria.calgary_enabled:
|
| 1143 |
+
_add_df_table(doc, "Table 3B - Calgary Project Criteria Register", criteria_table, landscape=True, font_size=7.5)
|
| 1144 |
+
doc.add_heading("3.2 Computer Model", level=2)
|
| 1145 |
+
doc.add_paragraph(
|
| 1146 |
+
f"The analysis was completed using EPA SWMM/OpenSWMM. The report retains the model unit system ({units.system}; flow units {units.flow}). "
|
| 1147 |
+
f"routing model: {options.get('FLOW_ROUTING', 'not identified')}; infiltration model: {options.get('INFILTRATION', 'not identified')}; "
|
| 1148 |
+
f"reporting step: {options.get('REPORT_STEP', 'not identified')}; routing step: {options.get('ROUTING_STEP', 'not identified')}."
|
| 1149 |
+
)
|
| 1150 |
+
if options.get("FLOW_ROUTING", "").upper() == "KINWAVE":
|
| 1151 |
+
doc.add_paragraph("Modelling limitation: kinematic-wave routing does not fully represent backwater, pressurization, reverse flow, or complex surcharge interactions. Dynamic-wave routing should be considered where these effects are material.")
|
| 1152 |
+
doc.add_heading("3.3 Major-Minor System", level=2)
|
| 1153 |
+
doc.add_paragraph("Open channels, swales, gutters, culverts, and closed conduits represented in the model were reviewed. Final major/minor classification must be confirmed against the approved drainage concept and drawings.")
|
| 1154 |
+
if scenario_comparison is not None and not scenario_comparison.empty:
|
| 1155 |
+
doc.add_heading("3.4 Preliminary Design Scenarios", level=2)
|
| 1156 |
+
doc.add_paragraph(
|
| 1157 |
+
"The scenario register summarizes preliminary model alternatives generated and simulated through the Rev22 workflow. "
|
| 1158 |
+
"Scenario storm sources, parameter changes, and review status must be confirmed by the responsible engineer before design use."
|
| 1159 |
+
)
|
| 1160 |
+
doc.add_paragraph(f"Reporting mode: {scenario_reporting_mode}.")
|
| 1161 |
+
compact_scenario = _compact_scenario_comparison(scenario_comparison)
|
| 1162 |
+
_add_df_table(doc, "Table 3C - Preliminary Scenario Comparison", compact_scenario, landscape=True, font_size=7.2)
|
| 1163 |
+
if scenario_analysis:
|
| 1164 |
+
doc.add_heading("3.4.1 Scenario Comparison Analysis", level=3)
|
| 1165 |
+
for paragraph in str(scenario_analysis).split("\n\n"):
|
| 1166 |
+
if paragraph.strip():
|
| 1167 |
+
doc.add_paragraph(paragraph.strip())
|
| 1168 |
+
if scenario_records and scenario_reporting_mode != "Base report with scenario comparison":
|
| 1169 |
+
doc.add_heading("3.4.2 Event-Specific Deterministic Results", level=3)
|
| 1170 |
+
doc.add_paragraph("Each selected model-event combination is presented as a separate deterministic dataset. Values shown as not applicable indicate that the corresponding model element type is absent.")
|
| 1171 |
+
table_no = 1
|
| 1172 |
+
for record in scenario_records:
|
| 1173 |
+
definition = record.get("definition", {}) or {}
|
| 1174 |
+
summary = record.get("summary", {}) or {}
|
| 1175 |
+
label = _event_short_label(summary)
|
| 1176 |
+
full_label = definition.get("scenario_name", f"Scenario {table_no}")
|
| 1177 |
+
doc.add_heading(str(label), level=4)
|
| 1178 |
+
doc.add_paragraph(f"Model-event dataset: {full_label}. Source model: {summary.get('Source Model', 'Not identified')}. Rainfall event: {summary.get('Storm', 'Not identified')}.")
|
| 1179 |
+
detail = _scenario_detail_tables(record, units)
|
| 1180 |
+
for key, title in [("overview","Event Summary"),("storage","Storage Results"),("controls","Outlet and Control Results"),("outfalls","Outfall Results"),("conduits","Conduit Results"),("subcatchments","Subcatchment Results")]:
|
| 1181 |
+
frame=detail[key]
|
| 1182 |
+
if frame is not None and not frame.empty:
|
| 1183 |
+
_add_df_table(doc, f"Table 3D-{table_no} - {label}: {title}", frame, landscape=len(frame.columns)>6, font_size=7.4)
|
| 1184 |
+
table_no += 1
|
| 1185 |
+
doc.add_heading("3.5 Catchment Areas", level=2)
|
| 1186 |
+
else:
|
| 1187 |
+
doc.add_heading("3.4 Catchment Areas", level=2)
|
| 1188 |
+
_add_narrative_block(doc, narrative_sections, "hydrology")
|
| 1189 |
+
catchment_main_cols = [
|
| 1190 |
+
"Sub ID", "Rain Gage", "Outlet", f"Area ({units.area})", "Impervious (%)",
|
| 1191 |
+
f"Width ({units.length})", "Slope (%)", "n Imperv.", "n Perv.",
|
| 1192 |
+
f"Peak Runoff ({units.flow})", f"Peak Rainfall ({units.rainfall})", "Peak-Flow Runoff Coefficient",
|
| 1193 |
+
]
|
| 1194 |
+
_add_df_table(doc, "Table 4 - Catchment Parameters and Runoff Results", sub_table[[c for c in catchment_main_cols if c in sub_table.columns]], landscape=True)
|
| 1195 |
+
|
| 1196 |
+
doc.add_heading("4.0 RESULTS", level=1)
|
| 1197 |
+
doc.add_heading("4.1 Major-System and Spill-Route Assessment", level=2)
|
| 1198 |
+
_add_narrative_block(doc, narrative_sections, "major_system")
|
| 1199 |
+
overland = link_table.copy()
|
| 1200 |
+
if criteria.major_link_ids:
|
| 1201 |
+
selected = {str(x).strip() for x in criteria.major_link_ids if str(x).strip()}
|
| 1202 |
+
overland = overland[overland["Link ID"].astype(str).isin(selected)].copy()
|
| 1203 |
+
else:
|
| 1204 |
+
# Outlet/orifice/weir controls are not automatically classified as
|
| 1205 |
+
# major-system overland routes. Only open conduit/street shapes are
|
| 1206 |
+
# included unless the engineer explicitly supplies major_link_ids.
|
| 1207 |
+
model_type = overland.get("Model Type", overland.get("Type", pd.Series("", index=overland.index))).astype(str).str.lower()
|
| 1208 |
+
shape = overland.get("Shape", pd.Series("", index=overland.index)).fillna("").astype(str).str.upper()
|
| 1209 |
+
overland = overland[model_type.eq("conduit") & shape.isin(["TRAPEZOIDAL", "RECT_OPEN", "TRIANGULAR", "IRREGULAR", "STREET"])].copy()
|
| 1210 |
+
overland_cols = ["Link ID", "From Node", "To Node", "Shape", f"Peak Flow ({units.flow})", f"Peak Depth ({units.length})", f"Peak Velocity ({units.velocity})", "Depth Ratio", "Status"]
|
| 1211 |
+
_add_df_table(doc, "Table 9 - Overland Flow Assessment", overland[[c for c in overland_cols if c in overland.columns]], landscape=True)
|
| 1212 |
+
overland_compliance = build_overland_compliance_table(overland, calgary, units.flow, units.length, units.velocity) if units.system == "SI" else pd.DataFrame()
|
| 1213 |
+
if not overland_compliance.empty:
|
| 1214 |
+
_add_df_table(doc, "Table 9A - Calgary Major-System Depth-Velocity Screening", overland_compliance, landscape=True, font_size=7.2)
|
| 1215 |
+
|
| 1216 |
+
outfalls = node_report[node_report["Type"].astype(str).str.lower() == "outfall"].copy() if not node_report.empty and "Type" in node_report else pd.DataFrame()
|
| 1217 |
+
if not outfalls.empty:
|
| 1218 |
+
outfalls["Status"] = "Boundary condition"
|
| 1219 |
+
outfalls["Depth Ratio"] = "N/A"
|
| 1220 |
+
outfall_cols = ["Node ID", "Type", f"Invert ({units.length})", f"Peak Depth ({units.length})", f"Peak Inflow ({units.flow})", f"Peak Flooding ({units.flow})", "Status"]
|
| 1221 |
+
_add_df_table(doc, "Table 10 - Major System Boundary Conditions - Outflows", outfalls[[c for c in outfall_cols if c in outfalls.columns]])
|
| 1222 |
+
|
| 1223 |
+
doc.add_heading("4.2 Minor-System Assessment", level=2)
|
| 1224 |
+
_add_narrative_block(doc, narrative_sections, "minor_system")
|
| 1225 |
+
if f"Diameter ({units.length})" in link_table.columns:
|
| 1226 |
+
link_table = link_table.rename(columns={f"Diameter ({units.length})": f"Diameter / Geom1 ({units.length})"})
|
| 1227 |
+
minor_cols = ["Link ID", "From Node", "To Node", "Shape", f"Length ({units.length})", f"Diameter / Geom1 ({units.length})", "Manning n", f"Peak Flow ({units.flow})", f"Peak Depth ({units.length})", "Depth Ratio", f"Peak Velocity ({units.velocity})", "Status"]
|
| 1228 |
+
conduit_table = link_table[link_table.get("Model Type", link_table.get("Type", "")).astype(str).str.lower().eq("conduit")].copy()
|
| 1229 |
+
_add_df_table(doc, "Table 12A - Conduit, Culvert, Swale and Gutter Analysis", conduit_table[[c for c in minor_cols if c in conduit_table.columns]], landscape=True)
|
| 1230 |
+
if not minor_capacity.empty:
|
| 1231 |
+
_add_df_table(doc, "Table 12A-1 - Calgary Minor-System Capacity Screening", minor_capacity, landscape=True, font_size=7.2)
|
| 1232 |
+
doc.add_heading("4.3 Hydraulic Controls", level=2)
|
| 1233 |
+
_add_narrative_block(doc, narrative_sections, "hydraulic_controls")
|
| 1234 |
+
if not criteria.suppress_empty_sections or not control_table.empty:
|
| 1235 |
+
_add_df_table(doc, "Table 12B - Hydraulic Controls", control_table, landscape=True, font_size=7.2)
|
| 1236 |
+
if not criteria.suppress_empty_sections or not control_recon.empty:
|
| 1237 |
+
_add_df_table(doc, "Table 12C - Hydraulic Control Flow Reconciliation", control_recon, landscape=True, font_size=7.2)
|
| 1238 |
+
doc.add_heading("4.4 Storage and Trap-Low Assessment", level=2)
|
| 1239 |
+
_add_narrative_block(doc, narrative_sections, "storage")
|
| 1240 |
+
storage_calgary = apply_storage_classification(storage_table, calgary, units.length) if not storage_table.empty else pd.DataFrame()
|
| 1241 |
+
if not criteria.suppress_empty_sections or not storage_calgary.empty:
|
| 1242 |
+
_add_df_table(doc, "Table 13 - Storage Unit Performance", storage_calgary, landscape=True, font_size=7.0)
|
| 1243 |
+
if criteria.suppress_empty_sections and control_table.empty and storage_table.empty:
|
| 1244 |
+
doc.add_paragraph("The model does not contain hydraulic controls or storage units; related result tables are not applicable.")
|
| 1245 |
+
|
| 1246 |
+
doc.add_heading("4.5 HGL, Surcharge, and Flooding Assessment", level=2)
|
| 1247 |
+
_add_narrative_block(doc, narrative_sections, "hgl_surcharge")
|
| 1248 |
+
hgl_display = hgl_table.copy()
|
| 1249 |
+
if not hgl_display.empty and "Type" in hgl_display:
|
| 1250 |
+
mask = hgl_display["Type"].astype(str).str.lower().eq("outfall")
|
| 1251 |
+
for c in [f"Ground/Rim ({units.length})", f"Freeboard ({units.length})"]:
|
| 1252 |
+
if c in hgl_display:
|
| 1253 |
+
hgl_display[c] = hgl_display[c].astype(object)
|
| 1254 |
+
hgl_display.loc[mask, c] = "N/A"
|
| 1255 |
+
if "Status" in hgl_display: hgl_display.loc[mask, "Status"] = "Boundary condition"
|
| 1256 |
+
_add_df_table(doc, "Table 15 - Summary of Node HGL / Surcharge Conditions", hgl_display, landscape=True)
|
| 1257 |
+
doc.add_heading("4.6 Outfalls and Boundary Conditions", level=2)
|
| 1258 |
+
_add_narrative_block(doc, narrative_sections, "outfalls")
|
| 1259 |
+
_add_df_table(doc, "Table 16 - Minor System Boundary Conditions - Outflows", outfalls[[c for c in outfall_cols if c in outfalls.columns]])
|
| 1260 |
+
if narrative_sections and str(narrative_sections.get("private_sites", "")).strip():
|
| 1261 |
+
doc.add_heading("4.7 Private-Site Discharge and Storage", level=2)
|
| 1262 |
+
_add_narrative_block(doc, narrative_sections, "private_sites")
|
| 1263 |
+
if narrative_sections and str(narrative_sections.get("water_quality", "")).strip():
|
| 1264 |
+
doc.add_heading("4.8 Water Quality and BMPs", level=2)
|
| 1265 |
+
_add_narrative_block(doc, narrative_sections, "water_quality")
|
| 1266 |
+
if narrative_sections and str(narrative_sections.get("model_documentation", "")).strip():
|
| 1267 |
+
doc.add_heading("4.9 Model Input/Output Documentation", level=2)
|
| 1268 |
+
_add_narrative_block(doc, narrative_sections, "model_documentation")
|
| 1269 |
+
|
| 1270 |
+
doc.add_heading("5.0 SUMMARY OF FINDINGS, CONCLUSIONS, AND RECOMMENDATIONS", level=1)
|
| 1271 |
+
findings = _summary_findings(node_report, link_report, sub_report, simulation_metadata, units, options, criteria)
|
| 1272 |
+
doc.add_heading("5.1 Executive Summary", level=2)
|
| 1273 |
+
if not _add_narrative_block(doc, narrative_sections, "executive_summary"):
|
| 1274 |
+
for line in _executive_summary(findings, critical_nodes, critical_links, units, criteria):
|
| 1275 |
+
doc.add_paragraph(line)
|
| 1276 |
+
doc.add_heading("5.2 Detailed Findings", level=2)
|
| 1277 |
+
for finding in findings:
|
| 1278 |
+
doc.add_paragraph(finding, style="List Bullet")
|
| 1279 |
+
doc.add_paragraph(
|
| 1280 |
+
f"QA/QC screening criteria for junctions/dividers: depth ratio ≥ {criteria.node_depth_ratio:.2f}; rim clearance ≤ {criteria.minimum_freeboard:g} {units.length}; storage facilities are assessed separately by classification; "
|
| 1281 |
+
f"conduit depth ratio ≥ {criteria.conduit_depth_ratio:.2f}; velocity ≥ {criteria.velocity_threshold:g} {units.velocity}; "
|
| 1282 |
+
f"continuity review/warning thresholds = {criteria.continuity_review:g}%/{criteria.continuity_warning:g}%.",
|
| 1283 |
+
style=None,
|
| 1284 |
+
)
|
| 1285 |
+
doc.add_paragraph(
|
| 1286 |
+
"Peak-flow runoff coefficient is calculated from the Rational Method relationship and is a screening statistic; it is not necessarily equivalent to the volumetric event runoff coefficient.",
|
| 1287 |
+
style=None,
|
| 1288 |
+
)
|
| 1289 |
+
if not criteria.suppress_empty_sections or not critical_nodes.empty:
|
| 1290 |
+
_add_df_table(doc, "Table 17A - Critical Nodes Requiring Review", critical_nodes, font_size=8.0)
|
| 1291 |
+
if not criteria.suppress_empty_sections or not critical_links.empty:
|
| 1292 |
+
_add_df_table(doc, "Table 17B - Critical Conduits Requiring Review", critical_links, landscape=True, font_size=7.8)
|
| 1293 |
+
doc.add_heading("5.3 Outstanding Information and Actions", level=2)
|
| 1294 |
+
outstanding = checklist_table[~checklist_table["Status"].isin(["Complete", "Not applicable"])][["Item","Requirement","Status","Outstanding Action"]]
|
| 1295 |
+
_add_df_table(doc, "Table 18 - Outstanding SWMR Information and Actions", outstanding, landscape=True, font_size=7.2)
|
| 1296 |
+
_add_narrative_block(doc, narrative_sections, "outstanding_actions")
|
| 1297 |
+
doc.add_heading("5.4 Conclusions and Recommendations", level=2)
|
| 1298 |
+
_add_narrative_block(doc, narrative_sections, "conclusions")
|
| 1299 |
+
doc.add_paragraph(
|
| 1300 |
+
"This automatically generated report is a model-data population aid. It does not replace engineering judgment, "
|
| 1301 |
+
"municipal criteria verification, design-drawing review, boundary-condition confirmation, or professional authentication."
|
| 1302 |
+
)
|
| 1303 |
+
|
| 1304 |
+
doc.add_page_break()
|
| 1305 |
+
doc.add_heading("APPENDIX A - MODEL DATA", level=1)
|
| 1306 |
+
link_geom, link_params, link_results = _split_link_appendix(conduit_table, units)
|
| 1307 |
+
_add_df_table(doc, "Table A-1 - Link Connectivity and Geometry", link_geom, landscape=True, font_size=7.0)
|
| 1308 |
+
_add_df_table(doc, "Table A-2 - Link Hydraulic Parameters", link_params, landscape=True, font_size=7.2)
|
| 1309 |
+
_add_df_table(doc, "Table A-3 - Link Simulation Results", link_results, landscape=True, font_size=7.2)
|
| 1310 |
+
if not criteria.suppress_empty_sections or not control_table.empty:
|
| 1311 |
+
_add_df_table(doc, "Table A-4 - Hydraulic Control Parameters and Results", control_table, landscape=True, font_size=7.0)
|
| 1312 |
+
if not criteria.suppress_empty_sections or not storage_calgary.empty:
|
| 1313 |
+
_add_df_table(doc, "Table A-5 - Storage Unit Parameters and Results", storage_calgary, landscape=True, font_size=6.8)
|
| 1314 |
+
_add_df_table(doc, "Table A-6 - Node Model and Result Summary", hgl_display, landscape=True, font_size=7.0)
|
| 1315 |
+
|
| 1316 |
+
if criteria.calgary_enabled:
|
| 1317 |
+
doc.add_heading("APPENDIX B - CALGARY QA/QC", level=1)
|
| 1318 |
+
_add_df_table(doc, "Table B-1 - Project Criteria Register", criteria_table, landscape=True, font_size=7.2)
|
| 1319 |
+
if not overland_compliance.empty:
|
| 1320 |
+
_add_df_table(doc, "Table B-2 - Major-System Depth-Velocity Screening", overland_compliance, landscape=True, font_size=7.0)
|
| 1321 |
+
if not minor_capacity.empty:
|
| 1322 |
+
_add_df_table(doc, "Table B-3 - Minor-System Capacity Screening", minor_capacity, landscape=True, font_size=7.0)
|
| 1323 |
+
_add_df_table(doc, "Table B-4 - Calgary SWMR Completeness Register", checklist_table, landscape=True, font_size=6.8)
|
| 1324 |
+
_add_df_table(doc, "Table B-5 - Draft Readiness Summary", readiness_table, font_size=8.0)
|
| 1325 |
+
|
| 1326 |
+
doc.add_heading("APPENDIX C - SUBCATCHMENT DATA", level=1)
|
| 1327 |
+
sub_model, sub_results = _split_subcatchment_appendix(sub_table, units)
|
| 1328 |
+
_add_df_table(doc, "Table B-1 - Subcatchment Model Parameters", sub_model, landscape=True, font_size=7.0)
|
| 1329 |
+
_add_df_table(doc, "Table B-2 - Subcatchment Simulation Results", sub_results, landscape=True, font_size=7.2)
|
| 1330 |
+
|
| 1331 |
+
llm_context = build_llm_report_context(
|
| 1332 |
+
metadata=asdict(metadata), criteria=calgary, findings=findings,
|
| 1333 |
+
tables={
|
| 1334 |
+
"design_event": event_table, "criteria_register": criteria_table,
|
| 1335 |
+
"minor_system": minor_capacity, "major_system": overland_compliance,
|
| 1336 |
+
"storage": storage_calgary, "critical_nodes": critical_nodes,
|
| 1337 |
+
"critical_conduits": critical_links, "outfalls": outfalls,
|
| 1338 |
+
"swmr_checklist": checklist_table, "draft_readiness": readiness_table,
|
| 1339 |
+
"scenario_comparison": scenario_comparison if scenario_comparison is not None else pd.DataFrame(),
|
| 1340 |
+
},
|
| 1341 |
+
)
|
| 1342 |
+
docx_buffer = io.BytesIO(); doc.save(docx_buffer); docx_bytes = docx_buffer.getvalue()
|
| 1343 |
+
base = _safe_name(metadata.project_name)
|
| 1344 |
+
|
| 1345 |
+
zip_buffer = io.BytesIO()
|
| 1346 |
+
with zipfile.ZipFile(zip_buffer, "w", compression=zipfile.ZIP_DEFLATED) as zf:
|
| 1347 |
+
zf.writestr(f"{base}_SWM_Report.docx", docx_bytes)
|
| 1348 |
+
zf.writestr("tables/node_summary.csv", node_report.to_csv(index=False) if node_summary is not None else "")
|
| 1349 |
+
zf.writestr("tables/link_summary.csv", link_report.to_csv(index=False) if link_summary is not None else "")
|
| 1350 |
+
zf.writestr("tables/subcatchment_summary.csv", sub_report.to_csv(index=False) if sub_summary is not None else "")
|
| 1351 |
+
zf.writestr("tables/catchment_model_results.csv", sub_table.to_csv(index=False))
|
| 1352 |
+
zf.writestr("tables/link_model_results.csv", link_table.to_csv(index=False))
|
| 1353 |
+
zf.writestr("tables/node_hgl_summary.csv", hgl_table.to_csv(index=False))
|
| 1354 |
+
zf.writestr("tables/hydraulic_controls.csv", control_table.to_csv(index=False))
|
| 1355 |
+
zf.writestr("tables/control_flow_reconciliation.csv", control_recon.to_csv(index=False))
|
| 1356 |
+
zf.writestr("tables/storage_unit_performance.csv", storage_table.to_csv(index=False))
|
| 1357 |
+
zf.writestr("tables/model_area_classification.csv", area_table.to_csv(index=False))
|
| 1358 |
+
zf.writestr("tables/critical_nodes.csv", critical_nodes.to_csv(index=False))
|
| 1359 |
+
zf.writestr("tables/critical_conduits.csv", critical_links.to_csv(index=False))
|
| 1360 |
+
zf.writestr("tables/design_event_summary.csv", event_table.to_csv(index=False))
|
| 1361 |
+
zf.writestr("tables/calgary_criteria_register.csv", criteria_table.to_csv(index=False))
|
| 1362 |
+
zf.writestr("tables/calgary_minor_system_capacity.csv", minor_capacity.to_csv(index=False))
|
| 1363 |
+
zf.writestr("tables/calgary_major_system_compliance.csv", overland_compliance.to_csv(index=False))
|
| 1364 |
+
zf.writestr("tables/calgary_storage_assessment.csv", storage_calgary.to_csv(index=False))
|
| 1365 |
+
zf.writestr("tables/calgary_swmr_completeness_register.csv", checklist_table.to_csv(index=False))
|
| 1366 |
+
zf.writestr("tables/swmr_draft_readiness.csv", readiness_table.to_csv(index=False))
|
| 1367 |
+
if scenario_comparison is not None and not scenario_comparison.empty:
|
| 1368 |
+
zf.writestr("tables/preliminary_scenario_comparison.csv", scenario_comparison.to_csv(index=False))
|
| 1369 |
+
if scenario_analysis:
|
| 1370 |
+
zf.writestr("narratives/preliminary_scenario_comparison_analysis.txt", str(scenario_analysis))
|
| 1371 |
+
if scenario_records:
|
| 1372 |
+
for record in scenario_records:
|
| 1373 |
+
sid = str((record.get("definition", {}) or {}).get("scenario_id", "scenario"))
|
| 1374 |
+
if sid == "BASE_MODEL":
|
| 1375 |
+
continue
|
| 1376 |
+
for key, frame in _scenario_detail_tables(record, units).items():
|
| 1377 |
+
if frame is not None and not frame.empty:
|
| 1378 |
+
zf.writestr(f"tables/scenarios/{sid}_{key}.csv", frame.to_csv(index=False))
|
| 1379 |
+
zf.writestr("metadata/llm_report_context.json", json.dumps(llm_context, indent=2, default=str))
|
| 1380 |
+
zf.writestr("metadata/approved_narrative_sections.json", json.dumps(dict(narrative_sections or {}), indent=2, ensure_ascii=False))
|
| 1381 |
+
zf.writestr("metadata/project_metadata.json", json.dumps(asdict(metadata), indent=2))
|
| 1382 |
+
zf.writestr("metadata/report_criteria.json", json.dumps(asdict(criteria), indent=2, default=str))
|
| 1383 |
+
zf.writestr("metadata/simulation_metadata.json", json.dumps(simulation_metadata, indent=2, default=str))
|
| 1384 |
+
zf.writestr("metadata/inp_sections.json", json.dumps(inp_sections, indent=2, default=str))
|
| 1385 |
+
if preliminary_review_artifacts:
|
| 1386 |
+
zf.writestr("preliminary_design/review_manifest.json", json.dumps(preliminary_review_artifacts.get("manifest", {}), indent=2, default=str))
|
| 1387 |
+
findings_df = preliminary_review_artifacts.get("findings")
|
| 1388 |
+
if isinstance(findings_df, pd.DataFrame):
|
| 1389 |
+
zf.writestr("preliminary_design/findings_register.csv", findings_df.to_csv(index=False))
|
| 1390 |
+
elif findings_df:
|
| 1391 |
+
zf.writestr("preliminary_design/findings_register.json", json.dumps(findings_df, indent=2, default=str))
|
| 1392 |
+
if preliminary_review_artifacts.get("ai_review"):
|
| 1393 |
+
zf.writestr("preliminary_design/ai_review.md", str(preliminary_review_artifacts.get("ai_review")))
|
| 1394 |
+
if preliminary_review_artifacts.get("reviewed_model"):
|
| 1395 |
+
zf.writestr("preliminary_design/reviewed_scenario_base.inp", str(preliminary_review_artifacts.get("reviewed_model")))
|
| 1396 |
+
if result_db_bytes:
|
| 1397 |
+
zf.writestr("database/swmm_complete_results.sqlite", result_db_bytes)
|
| 1398 |
+
|
| 1399 |
+
return {
|
| 1400 |
+
"docx": docx_bytes,
|
| 1401 |
+
"zip": zip_buffer.getvalue(),
|
| 1402 |
+
"docx_name": f"{base}_SWM_Report.docx",
|
| 1403 |
+
"zip_name": f"{base}_SWM_Report_Package.zip",
|
| 1404 |
+
"catchment_table": sub_table,
|
| 1405 |
+
"link_table": link_table,
|
| 1406 |
+
"hgl_table": hgl_table,
|
| 1407 |
+
"control_table": control_table,
|
| 1408 |
+
"storage_table": storage_calgary,
|
| 1409 |
+
"area_table": area_table,
|
| 1410 |
+
"calgary_criteria_table": criteria_table,
|
| 1411 |
+
"calgary_minor_capacity": minor_capacity,
|
| 1412 |
+
"calgary_overland_compliance": overland_compliance,
|
| 1413 |
+
"swmr_checklist": checklist_table, "draft_readiness": readiness_table,
|
| 1414 |
+
"llm_context": llm_context,
|
| 1415 |
+
"narrative_sections": dict(narrative_sections or {}),
|
| 1416 |
+
"scenario_comparison": scenario_comparison if scenario_comparison is not None else pd.DataFrame(),
|
| 1417 |
+
}
|
requirements.txt
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Server environment (OpenSWMM intentionally absent — worker venv only)
|
| 2 |
+
mcp>=1.10,<2
|
| 3 |
+
fastapi>=0.110
|
| 4 |
+
uvicorn>=0.29
|
| 5 |
+
httpx>=0.27
|
| 6 |
+
pandas>=2.0
|
| 7 |
+
numpy>=1.26
|
| 8 |
+
python-docx>=1.1
|
| 9 |
+
PyYAML>=6.0
|
results_db.py
ADDED
|
@@ -0,0 +1,333 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Complete SQLite store for SWMM model inputs, outputs, summaries and AI retrieval."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import json
|
| 5 |
+
import re
|
| 6 |
+
import sqlite3
|
| 7 |
+
import tempfile
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
from typing import Any
|
| 10 |
+
|
| 11 |
+
import pandas as pd
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class ResultDatabase:
|
| 15 |
+
"""File-backed SQLite database containing the complete model and simulation dataset.
|
| 16 |
+
|
| 17 |
+
The database is intentionally file-backed rather than ``:memory:`` so the user can
|
| 18 |
+
download it, inspect it in any SQLite client, and retain an auditable simulation
|
| 19 |
+
artefact. The LLM still receives only bounded query results.
|
| 20 |
+
"""
|
| 21 |
+
|
| 22 |
+
def __init__(self, db_path: str | Path | None = None) -> None:
|
| 23 |
+
if db_path is None:
|
| 24 |
+
fd, name = tempfile.mkstemp(prefix="swmm_model_", suffix=".sqlite")
|
| 25 |
+
Path(name).unlink(missing_ok=True)
|
| 26 |
+
try:
|
| 27 |
+
import os
|
| 28 |
+
os.close(fd)
|
| 29 |
+
except OSError:
|
| 30 |
+
pass
|
| 31 |
+
self.path = Path(name)
|
| 32 |
+
else:
|
| 33 |
+
self.path = Path(db_path)
|
| 34 |
+
self.connection = sqlite3.connect(str(self.path), check_same_thread=False)
|
| 35 |
+
self.connection.execute("PRAGMA journal_mode=WAL")
|
| 36 |
+
self.connection.execute("PRAGMA synchronous=NORMAL")
|
| 37 |
+
self.connection.execute("PRAGMA foreign_keys=ON")
|
| 38 |
+
|
| 39 |
+
@staticmethod
|
| 40 |
+
def _safe_table_name(section: str) -> str:
|
| 41 |
+
clean = re.sub(r"[^a-zA-Z0-9_]+", "_", section.strip().lower()).strip("_")
|
| 42 |
+
return f"inp_{clean or 'unknown'}"
|
| 43 |
+
|
| 44 |
+
def _load_complete_input(self, inp_path: str | Path) -> None:
|
| 45 |
+
path = Path(inp_path)
|
| 46 |
+
text = path.read_text(encoding="utf-8", errors="replace")
|
| 47 |
+
line_rows: list[dict[str, Any]] = []
|
| 48 |
+
section_rows: dict[str, list[dict[str, Any]]] = {}
|
| 49 |
+
catalog: list[dict[str, Any]] = []
|
| 50 |
+
current_section = "PREAMBLE"
|
| 51 |
+
section_row_no = 0
|
| 52 |
+
|
| 53 |
+
for line_no, raw in enumerate(text.splitlines(), start=1):
|
| 54 |
+
stripped = raw.strip()
|
| 55 |
+
is_section = stripped.startswith("[") and "]" in stripped
|
| 56 |
+
if is_section:
|
| 57 |
+
current_section = stripped[1:stripped.index("]")].strip().upper()
|
| 58 |
+
section_row_no = 0
|
| 59 |
+
elif stripped and not stripped.startswith(";"):
|
| 60 |
+
section_row_no += 1
|
| 61 |
+
|
| 62 |
+
line_rows.append({
|
| 63 |
+
"line_no": line_no,
|
| 64 |
+
"section_name": current_section,
|
| 65 |
+
"section_row_no": section_row_no if not is_section else 0,
|
| 66 |
+
"raw_text": raw,
|
| 67 |
+
"stripped_text": stripped,
|
| 68 |
+
"is_blank": int(not stripped),
|
| 69 |
+
"is_comment": int(stripped.startswith(";")),
|
| 70 |
+
"is_section_header": int(is_section),
|
| 71 |
+
})
|
| 72 |
+
|
| 73 |
+
if stripped and not stripped.startswith(";") and not is_section:
|
| 74 |
+
# Keep both the exact raw line and all whitespace-delimited values.
|
| 75 |
+
tokens = stripped.split()
|
| 76 |
+
row = {
|
| 77 |
+
"row_no": section_row_no,
|
| 78 |
+
"source_line_no": line_no,
|
| 79 |
+
"raw_text": raw,
|
| 80 |
+
}
|
| 81 |
+
for i, token in enumerate(tokens, start=1):
|
| 82 |
+
row[f"value_{i}"] = token
|
| 83 |
+
section_rows.setdefault(current_section, []).append(row)
|
| 84 |
+
|
| 85 |
+
pd.DataFrame(line_rows).to_sql(
|
| 86 |
+
"model_input_lines", self.connection, if_exists="replace", index=False
|
| 87 |
+
)
|
| 88 |
+
pd.DataFrame([{
|
| 89 |
+
"file_name": path.name,
|
| 90 |
+
"absolute_path_at_run": str(path),
|
| 91 |
+
"byte_size": path.stat().st_size,
|
| 92 |
+
"line_count": len(line_rows),
|
| 93 |
+
"full_text": text,
|
| 94 |
+
}]).to_sql("model_input_file", self.connection, if_exists="replace", index=False)
|
| 95 |
+
|
| 96 |
+
for section, rows in section_rows.items():
|
| 97 |
+
table = self._safe_table_name(section)
|
| 98 |
+
pd.DataFrame(rows).to_sql(table, self.connection, if_exists="replace", index=False)
|
| 99 |
+
catalog.append({
|
| 100 |
+
"section_name": section,
|
| 101 |
+
"table_name": table,
|
| 102 |
+
"row_count": len(rows),
|
| 103 |
+
"max_values_per_row": max(
|
| 104 |
+
(sum(1 for k in r if k.startswith("value_")) for r in rows), default=0
|
| 105 |
+
),
|
| 106 |
+
})
|
| 107 |
+
|
| 108 |
+
pd.DataFrame(catalog).to_sql(
|
| 109 |
+
"model_input_section_catalog", self.connection, if_exists="replace", index=False
|
| 110 |
+
)
|
| 111 |
+
|
| 112 |
+
@staticmethod
|
| 113 |
+
def _time_strings(times: list[Any]) -> list[str]:
|
| 114 |
+
return [t.isoformat(sep=" ") if hasattr(t, "isoformat") else str(t) for t in times]
|
| 115 |
+
|
| 116 |
+
def _load_complete_outputs(self, results: dict[str, Any]) -> None:
|
| 117 |
+
times = results.get("times", [])
|
| 118 |
+
time_strings = self._time_strings(times)
|
| 119 |
+
|
| 120 |
+
node_rows: list[dict[str, Any]] = []
|
| 121 |
+
for node_id, data in results.get("node_ts", {}).items():
|
| 122 |
+
n = max((len(data.get(k, [])) for k in (
|
| 123 |
+
"depth", "flooding", "inflow", "head", "outflow", "volume"
|
| 124 |
+
)), default=0)
|
| 125 |
+
for i in range(n):
|
| 126 |
+
node_rows.append({
|
| 127 |
+
"time_index": i,
|
| 128 |
+
"timestamp": time_strings[i] if i < len(time_strings) else str(i),
|
| 129 |
+
"node_id": node_id,
|
| 130 |
+
"depth": data.get("depth", [None] * n)[i],
|
| 131 |
+
"flooding": data.get("flooding", [None] * n)[i],
|
| 132 |
+
"inflow": data.get("inflow", [None] * n)[i],
|
| 133 |
+
"head": data.get("head", [None] * n)[i],
|
| 134 |
+
"outflow": data.get("outflow", [None] * n)[i],
|
| 135 |
+
"volume": data.get("volume", [None] * n)[i],
|
| 136 |
+
})
|
| 137 |
+
pd.DataFrame(node_rows, columns=[
|
| 138 |
+
"time_index", "timestamp", "node_id", "depth", "flooding",
|
| 139 |
+
"inflow", "head", "outflow", "volume"
|
| 140 |
+
]).to_sql("node_timeseries", self.connection, if_exists="replace", index=False, chunksize=5000)
|
| 141 |
+
|
| 142 |
+
node_static = [{
|
| 143 |
+
"node_id": node_id,
|
| 144 |
+
"invert_elevation": data.get("invert_elevation"),
|
| 145 |
+
"full_depth": data.get("full_depth"),
|
| 146 |
+
} for node_id, data in results.get("node_ts", {}).items()]
|
| 147 |
+
pd.DataFrame(node_static, columns=["node_id", "invert_elevation", "full_depth"]).to_sql(
|
| 148 |
+
"node_output_metadata", self.connection, if_exists="replace", index=False
|
| 149 |
+
)
|
| 150 |
+
|
| 151 |
+
link_rows: list[dict[str, Any]] = []
|
| 152 |
+
for link_id, data in results.get("link_ts", {}).items():
|
| 153 |
+
n = max((len(data.get(k, [])) for k in (
|
| 154 |
+
"flow", "depth", "velocity", "volume", "capacity"
|
| 155 |
+
)), default=0)
|
| 156 |
+
for i in range(n):
|
| 157 |
+
link_rows.append({
|
| 158 |
+
"time_index": i,
|
| 159 |
+
"timestamp": time_strings[i] if i < len(time_strings) else str(i),
|
| 160 |
+
"link_id": link_id,
|
| 161 |
+
"flow": data.get("flow", [None] * n)[i],
|
| 162 |
+
"depth": data.get("depth", [None] * n)[i],
|
| 163 |
+
"velocity": data.get("velocity", [None] * n)[i],
|
| 164 |
+
"volume": data.get("volume", [None] * n)[i],
|
| 165 |
+
"capacity": data.get("capacity", [None] * n)[i],
|
| 166 |
+
})
|
| 167 |
+
pd.DataFrame(link_rows, columns=[
|
| 168 |
+
"time_index", "timestamp", "link_id", "flow", "depth",
|
| 169 |
+
"velocity", "volume", "capacity"
|
| 170 |
+
]).to_sql("link_timeseries", self.connection, if_exists="replace", index=False, chunksize=5000)
|
| 171 |
+
|
| 172 |
+
link_static = [{
|
| 173 |
+
"link_id": link_id,
|
| 174 |
+
"length": data.get("length"),
|
| 175 |
+
"roughness": data.get("roughness"),
|
| 176 |
+
"diameter": data.get("diameter"),
|
| 177 |
+
} for link_id, data in results.get("link_ts", {}).items()]
|
| 178 |
+
pd.DataFrame(link_static, columns=["link_id", "length", "roughness", "diameter"]).to_sql(
|
| 179 |
+
"link_output_metadata", self.connection, if_exists="replace", index=False
|
| 180 |
+
)
|
| 181 |
+
|
| 182 |
+
sub_rows: list[dict[str, Any]] = []
|
| 183 |
+
for sub_id, data in results.get("sub_ts", {}).items():
|
| 184 |
+
n = max((len(data.get(k, [])) for k in ("runoff", "rainfall", "infil")), default=0)
|
| 185 |
+
for i in range(n):
|
| 186 |
+
sub_rows.append({
|
| 187 |
+
"time_index": i,
|
| 188 |
+
"timestamp": time_strings[i] if i < len(time_strings) else str(i),
|
| 189 |
+
"subcatchment_id": sub_id,
|
| 190 |
+
"runoff": data.get("runoff", [None] * n)[i],
|
| 191 |
+
"rainfall": data.get("rainfall", [None] * n)[i],
|
| 192 |
+
"infiltration": data.get("infil", [None] * n)[i],
|
| 193 |
+
})
|
| 194 |
+
pd.DataFrame(sub_rows, columns=[
|
| 195 |
+
"time_index", "timestamp", "subcatchment_id", "runoff", "rainfall", "infiltration"
|
| 196 |
+
]).to_sql("subcatchment_timeseries", self.connection, if_exists="replace", index=False, chunksize=5000)
|
| 197 |
+
|
| 198 |
+
metadata = results.get("metadata", {})
|
| 199 |
+
meta_rows = []
|
| 200 |
+
warnings = metadata.get("warnings", []) or []
|
| 201 |
+
for key, value in metadata.items():
|
| 202 |
+
if key == "warnings":
|
| 203 |
+
continue
|
| 204 |
+
if isinstance(value, (dict, list, tuple)):
|
| 205 |
+
value = json.dumps(value, default=str)
|
| 206 |
+
elif hasattr(value, "isoformat"):
|
| 207 |
+
value = value.isoformat()
|
| 208 |
+
meta_rows.append({"key": key, "value": value})
|
| 209 |
+
pd.DataFrame(meta_rows, columns=["key", "value"]).to_sql(
|
| 210 |
+
"simulation_metadata", self.connection, if_exists="replace", index=False
|
| 211 |
+
)
|
| 212 |
+
pd.DataFrame(warnings, columns=["code", "message"]).to_sql(
|
| 213 |
+
"simulation_warnings", self.connection, if_exists="replace", index=False
|
| 214 |
+
)
|
| 215 |
+
|
| 216 |
+
# Indexes materially reduce retrieval cost for large models.
|
| 217 |
+
self.connection.executescript("""
|
| 218 |
+
CREATE INDEX IF NOT EXISTS idx_node_ts_id_time ON node_timeseries(node_id, time_index);
|
| 219 |
+
CREATE INDEX IF NOT EXISTS idx_node_ts_flood ON node_timeseries(flooding DESC);
|
| 220 |
+
CREATE INDEX IF NOT EXISTS idx_link_ts_id_time ON link_timeseries(link_id, time_index);
|
| 221 |
+
CREATE INDEX IF NOT EXISTS idx_link_ts_capacity ON link_timeseries(capacity DESC);
|
| 222 |
+
CREATE INDEX IF NOT EXISTS idx_sub_ts_id_time ON subcatchment_timeseries(subcatchment_id, time_index);
|
| 223 |
+
CREATE INDEX IF NOT EXISTS idx_input_section ON model_input_lines(section_name, section_row_no);
|
| 224 |
+
""")
|
| 225 |
+
|
| 226 |
+
def load(
|
| 227 |
+
self,
|
| 228 |
+
node_summary: pd.DataFrame,
|
| 229 |
+
link_summary: pd.DataFrame,
|
| 230 |
+
sub_summary: pd.DataFrame,
|
| 231 |
+
*,
|
| 232 |
+
inp_path: str | Path,
|
| 233 |
+
results: dict[str, Any],
|
| 234 |
+
) -> None:
|
| 235 |
+
node_summary.to_sql("node_summary", self.connection, if_exists="replace", index=False)
|
| 236 |
+
link_summary.to_sql("link_summary", self.connection, if_exists="replace", index=False)
|
| 237 |
+
sub_summary.to_sql("subcatchment_summary", self.connection, if_exists="replace", index=False)
|
| 238 |
+
self._load_complete_input(inp_path)
|
| 239 |
+
self._load_complete_outputs(results)
|
| 240 |
+
self.connection.commit()
|
| 241 |
+
|
| 242 |
+
def table_catalog(self) -> pd.DataFrame:
|
| 243 |
+
return pd.read_sql_query("""
|
| 244 |
+
SELECT name AS table_name
|
| 245 |
+
FROM sqlite_master
|
| 246 |
+
WHERE type='table' AND name NOT LIKE 'sqlite_%'
|
| 247 |
+
ORDER BY name
|
| 248 |
+
""", self.connection)
|
| 249 |
+
|
| 250 |
+
def export_bytes(self) -> bytes:
|
| 251 |
+
self.connection.commit()
|
| 252 |
+
# Checkpoint WAL so the downloaded main file is self-contained.
|
| 253 |
+
try:
|
| 254 |
+
self.connection.execute("PRAGMA wal_checkpoint(FULL)")
|
| 255 |
+
except sqlite3.DatabaseError:
|
| 256 |
+
pass
|
| 257 |
+
return self.path.read_bytes()
|
| 258 |
+
|
| 259 |
+
@staticmethod
|
| 260 |
+
def _quoted(value: str) -> str:
|
| 261 |
+
return value.replace("'", "''")
|
| 262 |
+
|
| 263 |
+
def engineering_context(self, question: str, limit: int = 20) -> str:
|
| 264 |
+
"""Return compact SQL-derived context while retaining the full database locally."""
|
| 265 |
+
q = question.lower()
|
| 266 |
+
parts: list[str] = []
|
| 267 |
+
|
| 268 |
+
# Asset IDs in the question are used to retrieve exact time series.
|
| 269 |
+
ids = re.findall(r"\b[A-Za-z][A-Za-z0-9_.:-]*\b", question)
|
| 270 |
+
known_noise = {"which", "what", "when", "where", "show", "compare", "node", "nodes",
|
| 271 |
+
"link", "links", "pipe", "pipes", "conduit", "flow", "depth", "runoff",
|
| 272 |
+
"the", "and", "for", "from", "with", "during", "model"}
|
| 273 |
+
ids = [x for x in ids if x.lower() not in known_noise][:8]
|
| 274 |
+
|
| 275 |
+
if any(k in q for k in ("input", "roughness", "diameter", "length", "invert", "elevation",
|
| 276 |
+
"option", "rain gage", "timeseries", "control", "weir", "orifice",
|
| 277 |
+
"pump", "storage", "infiltration", "subarea")):
|
| 278 |
+
section_terms = {
|
| 279 |
+
"roughness": "CONDUITS", "diameter": "XSECTIONS", "length": "CONDUITS",
|
| 280 |
+
"invert": "JUNCTIONS", "elevation": "JUNCTIONS", "control": "CONTROLS",
|
| 281 |
+
"pump": "PUMPS", "weir": "WEIRS", "orifice": "ORIFICES",
|
| 282 |
+
"storage": "STORAGE", "infiltration": "INFILTRATION", "subarea": "SUBAREAS",
|
| 283 |
+
"rain": "RAINGAGES", "option": "OPTIONS",
|
| 284 |
+
}
|
| 285 |
+
selected = {v for k, v in section_terms.items() if k in q}
|
| 286 |
+
if not selected:
|
| 287 |
+
selected = {"OPTIONS", "JUNCTIONS", "CONDUITS", "XSECTIONS", "SUBCATCHMENTS"}
|
| 288 |
+
names = ",".join(f"'{self._quoted(s)}'" for s in selected)
|
| 289 |
+
sql = f"""SELECT section_name, section_row_no, raw_text
|
| 290 |
+
FROM model_input_lines
|
| 291 |
+
WHERE section_name IN ({names}) AND is_comment=0 AND is_blank=0
|
| 292 |
+
LIMIT {int(limit * 2)}"""
|
| 293 |
+
parts.append("MODEL INPUT\n" + pd.read_sql_query(sql, self.connection).to_csv(index=False))
|
| 294 |
+
|
| 295 |
+
if any(k in q for k in ("flood", "node", "manhole", "head", "inflow", "outflow")):
|
| 296 |
+
parts.append("NODE SUMMARY\n" + pd.read_sql_query(
|
| 297 |
+
f'''SELECT * FROM node_summary ORDER BY "Peak Flooding (m³/s)" DESC, "Depth Ratio" DESC LIMIT {int(limit)}''',
|
| 298 |
+
self.connection).to_csv(index=False))
|
| 299 |
+
for asset_id in ids:
|
| 300 |
+
df = pd.read_sql_query(
|
| 301 |
+
"""SELECT * FROM node_timeseries WHERE lower(node_id)=lower(?)
|
| 302 |
+
ORDER BY time_index LIMIT ?""", self.connection, params=(asset_id, int(limit * 3)))
|
| 303 |
+
if not df.empty:
|
| 304 |
+
parts.append(f"NODE TIMESERIES {asset_id}\n" + df.to_csv(index=False))
|
| 305 |
+
|
| 306 |
+
if any(k in q for k in ("conduit", "pipe", "link", "surcharge", "velocity", "capacity", "flow")):
|
| 307 |
+
parts.append("LINK SUMMARY\n" + pd.read_sql_query(
|
| 308 |
+
f'''SELECT * FROM link_summary ORDER BY "Depth Ratio" DESC, "Peak Flow (m³/s)" DESC LIMIT {int(limit)}''',
|
| 309 |
+
self.connection).to_csv(index=False))
|
| 310 |
+
for asset_id in ids:
|
| 311 |
+
df = pd.read_sql_query(
|
| 312 |
+
"""SELECT * FROM link_timeseries WHERE lower(link_id)=lower(?)
|
| 313 |
+
ORDER BY time_index LIMIT ?""", self.connection, params=(asset_id, int(limit * 3)))
|
| 314 |
+
if not df.empty:
|
| 315 |
+
parts.append(f"LINK TIMESERIES {asset_id}\n" + df.to_csv(index=False))
|
| 316 |
+
|
| 317 |
+
if any(k in q for k in ("subcatch", "runoff", "rain", "catchment", "hydrology", "infiltration")):
|
| 318 |
+
parts.append("SUBCATCHMENT SUMMARY\n" + pd.read_sql_query(
|
| 319 |
+
f'''SELECT * FROM subcatchment_summary ORDER BY "Peak Runoff (m³/s)" DESC LIMIT {int(limit)}''',
|
| 320 |
+
self.connection).to_csv(index=False))
|
| 321 |
+
for asset_id in ids:
|
| 322 |
+
df = pd.read_sql_query(
|
| 323 |
+
"""SELECT * FROM subcatchment_timeseries WHERE lower(subcatchment_id)=lower(?)
|
| 324 |
+
ORDER BY time_index LIMIT ?""", self.connection, params=(asset_id, int(limit * 3)))
|
| 325 |
+
if not df.empty:
|
| 326 |
+
parts.append(f"SUBCATCHMENT TIMESERIES {asset_id}\n" + df.to_csv(index=False))
|
| 327 |
+
|
| 328 |
+
if not parts:
|
| 329 |
+
for table in ("simulation_metadata", "node_summary", "link_summary", "subcatchment_summary"):
|
| 330 |
+
parts.append(table.upper() + "\n" + pd.read_sql_query(
|
| 331 |
+
f"SELECT * FROM {table} LIMIT 10", self.connection).to_csv(index=False))
|
| 332 |
+
|
| 333 |
+
return "\n".join(parts)
|
rpt_reconciliation.py
ADDED
|
@@ -0,0 +1,464 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Deterministic reconciliation of app-extracted results against the SWMM .rpt file.
|
| 2 |
+
|
| 3 |
+
Rationale
|
| 4 |
+
---------
|
| 5 |
+
The application builds its result tables (and therefore the Calgary velocity
|
| 6 |
+
screening and the Table 9A depth-velocity assessment) from time series read
|
| 7 |
+
through the OpenSWMM Python API inside the isolated worker. The engine's own
|
| 8 |
+
``.rpt`` file independently reports per-timestep maxima in its Link Flow
|
| 9 |
+
Summary and Node Depth Summary. On the Kincora Phase 2 reference model these
|
| 10 |
+
two sources were found to disagree: pipe peak velocities differed by 5-12%,
|
| 11 |
+
and peak velocities in irregular-transect (street) channels were understated
|
| 12 |
+
by up to a factor of six in the API-derived tables, while the ``.rpt`` values
|
| 13 |
+
matched the original consultant's SWMM 5.0.022 run almost exactly.
|
| 14 |
+
|
| 15 |
+
This module parses the ``.rpt`` summaries and produces an auditable
|
| 16 |
+
reconciliation table plus findings-register entries compatible with the
|
| 17 |
+
Preliminary Design Assistant. It performs no simulation and calls no LLM.
|
| 18 |
+
|
| 19 |
+
Notes and limitations
|
| 20 |
+
---------------------
|
| 21 |
+
* RPT object names are truncated to 20 characters by the engine. Where a
|
| 22 |
+
worker ID is longer than 20 characters, matching falls back to the
|
| 23 |
+
truncated prefix; ambiguous prefixes are reported as ``Unmatched`` rather
|
| 24 |
+
than guessed.
|
| 25 |
+
* Only CONDUIT and CHANNEL rows carry velocity; OUTLET/DUMMY/PUMP/ORIFICE/
|
| 26 |
+
WEIR rows are reconciled on peak flow only.
|
| 27 |
+
* The ``.rpt`` is treated as the authoritative cross-check because it is the
|
| 28 |
+
engine's own per-timestep statistic. A discrepancy does not by itself say
|
| 29 |
+
which value is "true"; it says the two extraction paths disagree and the
|
| 30 |
+
responsible engineer must not rely on the affected screening rows until
|
| 31 |
+
the cause is resolved.
|
| 32 |
+
"""
|
| 33 |
+
from __future__ import annotations
|
| 34 |
+
|
| 35 |
+
import math
|
| 36 |
+
import re
|
| 37 |
+
from dataclasses import dataclass
|
| 38 |
+
from pathlib import Path
|
| 39 |
+
from typing import Any, Mapping, Sequence
|
| 40 |
+
|
| 41 |
+
import pandas as pd
|
| 42 |
+
|
| 43 |
+
_RPT_NAME_WIDTH = 20
|
| 44 |
+
_VELOCITY_TYPES = {"CONDUIT", "CHANNEL"}
|
| 45 |
+
_FLOW_ONLY_TYPES = {"OUTLET", "DUMMY", "PUMP", "ORIFICE", "WEIR"}
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
@dataclass
|
| 49 |
+
class ReconciliationTolerances:
|
| 50 |
+
"""Screening tolerances for worker-vs-RPT deltas.
|
| 51 |
+
|
| 52 |
+
Values are relative unless stated. Absolute floors avoid flagging noise
|
| 53 |
+
on near-zero quantities (e.g. a 0.002 vs 0.004 m/s trickle).
|
| 54 |
+
"""
|
| 55 |
+
|
| 56 |
+
velocity_review_pct: float = 5.0
|
| 57 |
+
velocity_discrepancy_pct: float = 15.0
|
| 58 |
+
velocity_abs_floor: float = 0.05 # m/s or ft/s
|
| 59 |
+
flow_review_pct: float = 2.0
|
| 60 |
+
flow_discrepancy_pct: float = 10.0
|
| 61 |
+
flow_abs_floor: float = 0.005 # model flow units
|
| 62 |
+
depth_review_pct: float = 5.0
|
| 63 |
+
depth_discrepancy_pct: float = 15.0
|
| 64 |
+
depth_abs_floor: float = 0.01 # m or ft
|
| 65 |
+
continuity_abs_review: float = 0.05 # absolute percentage points
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
# ---------------------------------------------------------------------------
|
| 69 |
+
# RPT parsing
|
| 70 |
+
# ---------------------------------------------------------------------------
|
| 71 |
+
|
| 72 |
+
def _section_lines(text: str, header: str) -> list[str]:
|
| 73 |
+
"""Return the body lines of a starred RPT section, or an empty list."""
|
| 74 |
+
pattern = re.compile(
|
| 75 |
+
r"^\s*" + re.escape(header) + r"\s*$", re.MULTILINE)
|
| 76 |
+
match = pattern.search(text)
|
| 77 |
+
if not match:
|
| 78 |
+
return []
|
| 79 |
+
lines = text[match.end():].splitlines()
|
| 80 |
+
# The data table starts after the LAST dashed rule that precedes the
|
| 81 |
+
# first data row: title banner (****), blank, dashed rule, column
|
| 82 |
+
# headers, dashed rule, data rows, blank line.
|
| 83 |
+
last_rule = -1
|
| 84 |
+
for i, line in enumerate(lines):
|
| 85 |
+
stripped = line.strip()
|
| 86 |
+
if stripped and set(stripped) <= {"-"}:
|
| 87 |
+
last_rule = i
|
| 88 |
+
continue
|
| 89 |
+
if last_rule >= 0 and stripped and not stripped.startswith("*"):
|
| 90 |
+
# Column-header lines sit between the two rules; a line after a
|
| 91 |
+
# rule that is followed by another rule is a header, so only
|
| 92 |
+
# accept this as the data start if no further rule intervenes
|
| 93 |
+
# before the next blank line.
|
| 94 |
+
remainder = lines[i:]
|
| 95 |
+
if any(set(l.strip()) <= {"-"} and l.strip() for l in remainder[:4]):
|
| 96 |
+
continue # still inside the header block
|
| 97 |
+
body: list[str] = []
|
| 98 |
+
for data_line in remainder:
|
| 99 |
+
if not data_line.strip():
|
| 100 |
+
break
|
| 101 |
+
if data_line.strip().startswith("*"):
|
| 102 |
+
break
|
| 103 |
+
body.append(data_line.rstrip("\n"))
|
| 104 |
+
return body
|
| 105 |
+
return []
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def _row_tokens(line: str) -> tuple[str, list[str]]:
|
| 109 |
+
"""Split an RPT summary row into (object name, remaining tokens).
|
| 110 |
+
|
| 111 |
+
The name field is fixed-width (20 chars) and may itself contain no
|
| 112 |
+
spaces in SWMM inputs, but is sliced positionally to be safe.
|
| 113 |
+
"""
|
| 114 |
+
name = line[:2 + _RPT_NAME_WIDTH].strip()
|
| 115 |
+
rest = line[2 + _RPT_NAME_WIDTH:].split()
|
| 116 |
+
return name, rest
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def _to_float(token: str) -> float | None:
|
| 120 |
+
try:
|
| 121 |
+
value = float(token)
|
| 122 |
+
except (TypeError, ValueError):
|
| 123 |
+
return None
|
| 124 |
+
return value if math.isfinite(value) else None
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def parse_link_flow_summary(rpt_path: str | Path) -> pd.DataFrame:
|
| 128 |
+
"""Parse the Link Flow Summary into a DataFrame.
|
| 129 |
+
|
| 130 |
+
Columns: ``Link ID``, ``RPT Type``, ``RPT Peak |Flow|``,
|
| 131 |
+
``RPT Time of Max``, ``RPT Peak |Velocity|``, ``RPT Max/Full Flow``,
|
| 132 |
+
``RPT Max/Full Depth``. Velocity columns are NaN for flow-only types.
|
| 133 |
+
"""
|
| 134 |
+
text = Path(rpt_path).read_text(encoding="utf-8", errors="replace")
|
| 135 |
+
rows: list[dict[str, Any]] = []
|
| 136 |
+
for line in _section_lines(text, "Link Flow Summary"):
|
| 137 |
+
name, tokens = _row_tokens(line)
|
| 138 |
+
if not name or len(tokens) < 2:
|
| 139 |
+
continue
|
| 140 |
+
rtype = tokens[0].upper()
|
| 141 |
+
record: dict[str, Any] = {
|
| 142 |
+
"Link ID": name, "RPT Type": rtype,
|
| 143 |
+
"RPT Peak |Flow|": _to_float(tokens[1]),
|
| 144 |
+
"RPT Time of Max": None, "RPT Peak |Velocity|": None,
|
| 145 |
+
"RPT Max/Full Flow": None, "RPT Max/Full Depth": None,
|
| 146 |
+
}
|
| 147 |
+
if len(tokens) >= 4:
|
| 148 |
+
record["RPT Time of Max"] = f"{tokens[2]} {tokens[3]}"
|
| 149 |
+
if rtype in _VELOCITY_TYPES and len(tokens) >= 7:
|
| 150 |
+
record["RPT Peak |Velocity|"] = _to_float(tokens[4])
|
| 151 |
+
record["RPT Max/Full Flow"] = _to_float(tokens[5])
|
| 152 |
+
record["RPT Max/Full Depth"] = _to_float(tokens[6])
|
| 153 |
+
rows.append(record)
|
| 154 |
+
return pd.DataFrame(rows)
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
def parse_node_depth_summary(rpt_path: str | Path) -> pd.DataFrame:
|
| 158 |
+
"""Parse the Node Depth Summary into a DataFrame."""
|
| 159 |
+
text = Path(rpt_path).read_text(encoding="utf-8", errors="replace")
|
| 160 |
+
rows: list[dict[str, Any]] = []
|
| 161 |
+
for line in _section_lines(text, "Node Depth Summary"):
|
| 162 |
+
name, tokens = _row_tokens(line)
|
| 163 |
+
if not name or len(tokens) < 4:
|
| 164 |
+
continue
|
| 165 |
+
rows.append({
|
| 166 |
+
"Node ID": name, "RPT Type": tokens[0].upper(),
|
| 167 |
+
"RPT Avg Depth": _to_float(tokens[1]),
|
| 168 |
+
"RPT Max Depth": _to_float(tokens[2]),
|
| 169 |
+
"RPT Max HGL": _to_float(tokens[3]),
|
| 170 |
+
})
|
| 171 |
+
return pd.DataFrame(rows)
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
def parse_continuity_errors(rpt_path: str | Path) -> dict[str, float | None]:
|
| 175 |
+
"""Return runoff and flow-routing continuity errors in PERCENT.
|
| 176 |
+
|
| 177 |
+
The first ``Continuity Error (%)`` line in the RPT belongs to the runoff
|
| 178 |
+
quantity block and the second to flow routing, matching the engine's
|
| 179 |
+
output order.
|
| 180 |
+
"""
|
| 181 |
+
text = Path(rpt_path).read_text(encoding="utf-8", errors="replace")
|
| 182 |
+
values = [
|
| 183 |
+
_to_float(m.group(1))
|
| 184 |
+
for m in re.finditer(r"Continuity Error \(%\) \.+\s+(-?[\d.]+)", text)
|
| 185 |
+
]
|
| 186 |
+
return {
|
| 187 |
+
"runoff_error_pct": values[0] if len(values) > 0 else None,
|
| 188 |
+
"flow_error_pct": values[1] if len(values) > 1 else None,
|
| 189 |
+
}
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
# ---------------------------------------------------------------------------
|
| 193 |
+
# Reconciliation
|
| 194 |
+
# ---------------------------------------------------------------------------
|
| 195 |
+
|
| 196 |
+
def _match_rpt_row(object_id: str, rpt: pd.DataFrame, id_col: str) -> pd.Series | None:
|
| 197 |
+
exact = rpt[rpt[id_col].astype(str) == str(object_id)]
|
| 198 |
+
if len(exact) == 1:
|
| 199 |
+
return exact.iloc[0]
|
| 200 |
+
prefix = str(object_id)[:_RPT_NAME_WIDTH]
|
| 201 |
+
by_prefix = rpt[rpt[id_col].astype(str) == prefix]
|
| 202 |
+
if len(by_prefix) == 1:
|
| 203 |
+
return by_prefix.iloc[0]
|
| 204 |
+
return None
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
def _classify(worker: float | None, reference: float | None,
|
| 208 |
+
review_pct: float, discrepancy_pct: float,
|
| 209 |
+
abs_floor: float) -> tuple[str, float | None]:
|
| 210 |
+
"""Return (status, delta_pct). Deltas below the absolute floor pass."""
|
| 211 |
+
if worker is None or reference is None:
|
| 212 |
+
return "Unavailable", None
|
| 213 |
+
if abs(worker - reference) <= abs_floor:
|
| 214 |
+
return "OK", 0.0 if reference == 0 else round(
|
| 215 |
+
100.0 * (worker - reference) / abs(reference), 2)
|
| 216 |
+
if reference == 0:
|
| 217 |
+
return "Discrepancy", None
|
| 218 |
+
delta = 100.0 * (worker - reference) / abs(reference)
|
| 219 |
+
if abs(delta) <= review_pct:
|
| 220 |
+
return "OK", round(delta, 2)
|
| 221 |
+
if abs(delta) <= discrepancy_pct:
|
| 222 |
+
return "Review", round(delta, 2)
|
| 223 |
+
return "Discrepancy", round(delta, 2)
|
| 224 |
+
|
| 225 |
+
|
| 226 |
+
def reconcile_links(
|
| 227 |
+
link_summary: pd.DataFrame,
|
| 228 |
+
rpt_path: str | Path,
|
| 229 |
+
tolerances: ReconciliationTolerances | None = None,
|
| 230 |
+
) -> pd.DataFrame:
|
| 231 |
+
"""Reconcile the app's link table against the RPT Link Flow Summary.
|
| 232 |
+
|
| 233 |
+
``link_summary`` must contain ``Link ID`` plus any of the recognised
|
| 234 |
+
worker columns (``Peak Flow (...)``, ``Peak Velocity (...)``,
|
| 235 |
+
``Depth Ratio``). Unrecognised columns are ignored so unit-suffix
|
| 236 |
+
variations (m³/s vs cfs, m/s vs ft/s) are handled transparently.
|
| 237 |
+
"""
|
| 238 |
+
tol = tolerances or ReconciliationTolerances()
|
| 239 |
+
if link_summary is None or link_summary.empty:
|
| 240 |
+
return pd.DataFrame()
|
| 241 |
+
rpt = parse_link_flow_summary(rpt_path)
|
| 242 |
+
if rpt.empty:
|
| 243 |
+
return pd.DataFrame()
|
| 244 |
+
|
| 245 |
+
flow_col = next((c for c in link_summary.columns if c.startswith("Peak Flow (")), None)
|
| 246 |
+
vel_col = next((c for c in link_summary.columns if c.startswith("Peak Velocity (")), None)
|
| 247 |
+
depth_ratio_col = "Depth Ratio" if "Depth Ratio" in link_summary.columns else None
|
| 248 |
+
|
| 249 |
+
rows: list[dict[str, Any]] = []
|
| 250 |
+
for _, record in link_summary.iterrows():
|
| 251 |
+
link_id = str(record["Link ID"])
|
| 252 |
+
matched = _match_rpt_row(link_id, rpt, "Link ID")
|
| 253 |
+
if matched is None:
|
| 254 |
+
rows.append({"Link ID": link_id, "RPT Type": None,
|
| 255 |
+
"Overall Status": "Unmatched"})
|
| 256 |
+
continue
|
| 257 |
+
entry: dict[str, Any] = {"Link ID": link_id, "RPT Type": matched["RPT Type"]}
|
| 258 |
+
statuses: list[str] = []
|
| 259 |
+
|
| 260 |
+
worker_flow = _to_float(record.get(flow_col)) if flow_col else None
|
| 261 |
+
entry["Worker Peak Flow"] = worker_flow
|
| 262 |
+
entry["RPT Peak Flow"] = matched["RPT Peak |Flow|"]
|
| 263 |
+
status, delta = _classify(worker_flow, matched["RPT Peak |Flow|"],
|
| 264 |
+
tol.flow_review_pct, tol.flow_discrepancy_pct,
|
| 265 |
+
tol.flow_abs_floor)
|
| 266 |
+
entry["Flow Delta (%)"], entry["Flow Status"] = delta, status
|
| 267 |
+
statuses.append(status)
|
| 268 |
+
|
| 269 |
+
if matched["RPT Type"] in _VELOCITY_TYPES:
|
| 270 |
+
worker_vel = _to_float(record.get(vel_col)) if vel_col else None
|
| 271 |
+
entry["Worker Peak Velocity"] = worker_vel
|
| 272 |
+
entry["RPT Peak Velocity"] = matched["RPT Peak |Velocity|"]
|
| 273 |
+
status, delta = _classify(worker_vel, matched["RPT Peak |Velocity|"],
|
| 274 |
+
tol.velocity_review_pct,
|
| 275 |
+
tol.velocity_discrepancy_pct,
|
| 276 |
+
tol.velocity_abs_floor)
|
| 277 |
+
entry["Velocity Delta (%)"], entry["Velocity Status"] = delta, status
|
| 278 |
+
statuses.append(status)
|
| 279 |
+
|
| 280 |
+
if depth_ratio_col is not None:
|
| 281 |
+
worker_ratio = _to_float(record.get(depth_ratio_col))
|
| 282 |
+
entry["Worker Depth Ratio"] = worker_ratio
|
| 283 |
+
entry["RPT Max/Full Depth"] = matched["RPT Max/Full Depth"]
|
| 284 |
+
status, delta = _classify(worker_ratio, matched["RPT Max/Full Depth"],
|
| 285 |
+
tol.depth_review_pct,
|
| 286 |
+
tol.depth_discrepancy_pct,
|
| 287 |
+
tol.depth_abs_floor)
|
| 288 |
+
entry["Depth Ratio Delta (%)"], entry["Depth Ratio Status"] = delta, status
|
| 289 |
+
statuses.append(status)
|
| 290 |
+
|
| 291 |
+
order = {"Discrepancy": 3, "Unmatched": 3, "Review": 2, "Unavailable": 1, "OK": 0}
|
| 292 |
+
entry["Overall Status"] = max(statuses, key=lambda s: order.get(s, 0)) if statuses else "Unavailable"
|
| 293 |
+
rows.append(entry)
|
| 294 |
+
return pd.DataFrame(rows)
|
| 295 |
+
|
| 296 |
+
|
| 297 |
+
def reconcile_nodes(
|
| 298 |
+
node_summary: pd.DataFrame,
|
| 299 |
+
rpt_path: str | Path,
|
| 300 |
+
tolerances: ReconciliationTolerances | None = None,
|
| 301 |
+
) -> pd.DataFrame:
|
| 302 |
+
"""Reconcile app node peak depths against the RPT Node Depth Summary."""
|
| 303 |
+
tol = tolerances or ReconciliationTolerances()
|
| 304 |
+
if node_summary is None or node_summary.empty:
|
| 305 |
+
return pd.DataFrame()
|
| 306 |
+
rpt = parse_node_depth_summary(rpt_path)
|
| 307 |
+
if rpt.empty:
|
| 308 |
+
return pd.DataFrame()
|
| 309 |
+
depth_col = next((c for c in node_summary.columns if c.startswith("Peak Depth (")), None)
|
| 310 |
+
if depth_col is None:
|
| 311 |
+
return pd.DataFrame()
|
| 312 |
+
rows: list[dict[str, Any]] = []
|
| 313 |
+
for _, record in node_summary.iterrows():
|
| 314 |
+
node_id = str(record["Node ID"])
|
| 315 |
+
matched = _match_rpt_row(node_id, rpt, "Node ID")
|
| 316 |
+
if matched is None:
|
| 317 |
+
rows.append({"Node ID": node_id, "Overall Status": "Unmatched"})
|
| 318 |
+
continue
|
| 319 |
+
worker_depth = _to_float(record.get(depth_col))
|
| 320 |
+
status, delta = _classify(worker_depth, matched["RPT Max Depth"],
|
| 321 |
+
tol.depth_review_pct, tol.depth_discrepancy_pct,
|
| 322 |
+
tol.depth_abs_floor)
|
| 323 |
+
rows.append({
|
| 324 |
+
"Node ID": node_id, "RPT Type": matched["RPT Type"],
|
| 325 |
+
"Worker Peak Depth": worker_depth,
|
| 326 |
+
"RPT Max Depth": matched["RPT Max Depth"],
|
| 327 |
+
"Depth Delta (%)": delta, "Overall Status": status,
|
| 328 |
+
})
|
| 329 |
+
return pd.DataFrame(rows)
|
| 330 |
+
|
| 331 |
+
|
| 332 |
+
def reconcile_continuity(
|
| 333 |
+
simulation_metadata: Mapping[str, Any],
|
| 334 |
+
rpt_path: str | Path,
|
| 335 |
+
tolerances: ReconciliationTolerances | None = None,
|
| 336 |
+
) -> pd.DataFrame:
|
| 337 |
+
"""Reconcile worker continuity errors against the RPT, in percent.
|
| 338 |
+
|
| 339 |
+
Detects the fraction-vs-percent inconsistency: if the worker value is
|
| 340 |
+
approximately the RPT value divided by 100, the row is marked
|
| 341 |
+
``Unit inconsistency (fraction vs percent)`` rather than a numeric
|
| 342 |
+
discrepancy, since the underlying simulation agrees.
|
| 343 |
+
"""
|
| 344 |
+
tol = tolerances or ReconciliationTolerances()
|
| 345 |
+
rpt_values = parse_continuity_errors(rpt_path)
|
| 346 |
+
pairs = [
|
| 347 |
+
("Runoff continuity", simulation_metadata.get("runoff_error"),
|
| 348 |
+
rpt_values["runoff_error_pct"]),
|
| 349 |
+
("Flow routing continuity", simulation_metadata.get("flow_error"),
|
| 350 |
+
rpt_values["flow_error_pct"]),
|
| 351 |
+
]
|
| 352 |
+
rows: list[dict[str, Any]] = []
|
| 353 |
+
for label, worker, reference in pairs:
|
| 354 |
+
worker_f, ref_f = _to_float(worker), _to_float(reference)
|
| 355 |
+
if worker_f is None or ref_f is None:
|
| 356 |
+
status = "Unavailable"
|
| 357 |
+
elif abs(worker_f - ref_f) <= tol.continuity_abs_review:
|
| 358 |
+
status = "OK"
|
| 359 |
+
elif abs(worker_f * 100.0 - ref_f) <= tol.continuity_abs_review:
|
| 360 |
+
status = "Unit inconsistency (fraction vs percent)"
|
| 361 |
+
else:
|
| 362 |
+
status = "Discrepancy"
|
| 363 |
+
rows.append({"Quantity": label, "Worker Value": worker_f,
|
| 364 |
+
"RPT Value (%)": ref_f, "Status": status})
|
| 365 |
+
return pd.DataFrame(rows)
|
| 366 |
+
|
| 367 |
+
|
| 368 |
+
# ---------------------------------------------------------------------------
|
| 369 |
+
# Findings-register integration
|
| 370 |
+
# ---------------------------------------------------------------------------
|
| 371 |
+
|
| 372 |
+
def reconciliation_findings(
|
| 373 |
+
link_recon: pd.DataFrame,
|
| 374 |
+
node_recon: pd.DataFrame | None = None,
|
| 375 |
+
continuity_recon: pd.DataFrame | None = None,
|
| 376 |
+
start_index: int = 1,
|
| 377 |
+
) -> list[dict[str, Any]]:
|
| 378 |
+
"""Convert reconciliation discrepancies into PDA-style finding dicts.
|
| 379 |
+
|
| 380 |
+
Only ``Review``, ``Discrepancy``, ``Unmatched`` and unit-inconsistency
|
| 381 |
+
rows generate findings. Severity: Discrepancy/Unmatched -> High,
|
| 382 |
+
Review -> Medium. These findings carry no proposed model edit; the
|
| 383 |
+
recommended action is to resolve the extraction path before relying on
|
| 384 |
+
the affected screening tables.
|
| 385 |
+
"""
|
| 386 |
+
findings: list[dict[str, Any]] = []
|
| 387 |
+
counter = start_index
|
| 388 |
+
|
| 389 |
+
def add(severity: str, object_type: str, object_id: str, basis: str) -> None:
|
| 390 |
+
nonlocal counter
|
| 391 |
+
findings.append({
|
| 392 |
+
"finding_id": f"RPT-{counter:03d}",
|
| 393 |
+
"category": "Result reconciliation",
|
| 394 |
+
"severity": severity,
|
| 395 |
+
"finding_type": "Worker/RPT disagreement",
|
| 396 |
+
"object_type": object_type,
|
| 397 |
+
"object_id": object_id,
|
| 398 |
+
"rule_id": "QA-RECON-001",
|
| 399 |
+
"criterion_status": "Deterministic cross-check",
|
| 400 |
+
"deterministic_basis": basis,
|
| 401 |
+
"recommended_action": (
|
| 402 |
+
"Do not rely on the affected screening rows until the API "
|
| 403 |
+
"extraction and the engine report file agree. Verify units, "
|
| 404 |
+
"extraction property, and sampling of the worker time series."
|
| 405 |
+
),
|
| 406 |
+
"engineer_decision": "Defer",
|
| 407 |
+
"resolution_status": "Open",
|
| 408 |
+
})
|
| 409 |
+
counter += 1
|
| 410 |
+
|
| 411 |
+
if link_recon is not None and not link_recon.empty:
|
| 412 |
+
for _, row in link_recon.iterrows():
|
| 413 |
+
status = row.get("Overall Status")
|
| 414 |
+
if status in {"OK", "Unavailable", None}:
|
| 415 |
+
continue
|
| 416 |
+
severity = "High" if status in {"Discrepancy", "Unmatched"} else "Medium"
|
| 417 |
+
parts = []
|
| 418 |
+
for label, w, r, d in [
|
| 419 |
+
("velocity", row.get("Worker Peak Velocity"), row.get("RPT Peak Velocity"), row.get("Velocity Delta (%)")),
|
| 420 |
+
("flow", row.get("Worker Peak Flow"), row.get("RPT Peak Flow"), row.get("Flow Delta (%)")),
|
| 421 |
+
("depth ratio", row.get("Worker Depth Ratio"), row.get("RPT Max/Full Depth"), row.get("Depth Ratio Delta (%)")),
|
| 422 |
+
]:
|
| 423 |
+
if d is not None and not (isinstance(d, float) and math.isnan(d)) and abs(d) > 5.0:
|
| 424 |
+
parts.append(f"peak {label} worker={w} vs rpt={r} ({d:+.1f}%)")
|
| 425 |
+
basis = (f"Link '{row['Link ID']}': " + "; ".join(parts)) if parts else (
|
| 426 |
+
f"Link '{row['Link ID']}': status {status}.")
|
| 427 |
+
add(severity, "LINK", str(row["Link ID"]), basis)
|
| 428 |
+
|
| 429 |
+
if node_recon is not None and not node_recon.empty:
|
| 430 |
+
for _, row in node_recon.iterrows():
|
| 431 |
+
if row.get("Overall Status") in {"Review", "Discrepancy", "Unmatched"}:
|
| 432 |
+
severity = "Medium" if row["Overall Status"] == "Review" else "High"
|
| 433 |
+
add(severity, "NODE", str(row["Node ID"]),
|
| 434 |
+
f"Node '{row['Node ID']}': peak depth worker="
|
| 435 |
+
f"{row.get('Worker Peak Depth')} vs rpt={row.get('RPT Max Depth')}"
|
| 436 |
+
f" ({row.get('Depth Delta (%)')}%).")
|
| 437 |
+
|
| 438 |
+
if continuity_recon is not None and not continuity_recon.empty:
|
| 439 |
+
for _, row in continuity_recon.iterrows():
|
| 440 |
+
if row["Status"] not in {"OK", "Unavailable"}:
|
| 441 |
+
add("Medium", "MODEL", "MODEL",
|
| 442 |
+
f"{row['Quantity']}: worker={row['Worker Value']} vs "
|
| 443 |
+
f"rpt={row['RPT Value (%)']}%. {row['Status']}.")
|
| 444 |
+
return findings
|
| 445 |
+
|
| 446 |
+
|
| 447 |
+
def reconciliation_summary(link_recon: pd.DataFrame) -> dict[str, Any]:
|
| 448 |
+
"""Compact machine-readable summary for the report package metadata."""
|
| 449 |
+
if link_recon is None or link_recon.empty:
|
| 450 |
+
return {"links_checked": 0, "ok": 0, "review": 0,
|
| 451 |
+
"discrepancy": 0, "unmatched": 0, "verdict": "Not performed"}
|
| 452 |
+
counts = link_recon["Overall Status"].value_counts().to_dict()
|
| 453 |
+
discrepancies = counts.get("Discrepancy", 0) + counts.get("Unmatched", 0)
|
| 454 |
+
verdict = ("Pass - worker tables agree with engine report" if discrepancies == 0
|
| 455 |
+
and counts.get("Review", 0) == 0 else
|
| 456 |
+
"Review required - worker tables disagree with engine report")
|
| 457 |
+
return {
|
| 458 |
+
"links_checked": int(len(link_recon)),
|
| 459 |
+
"ok": int(counts.get("OK", 0)),
|
| 460 |
+
"review": int(counts.get("Review", 0)),
|
| 461 |
+
"discrepancy": int(counts.get("Discrepancy", 0)),
|
| 462 |
+
"unmatched": int(counts.get("Unmatched", 0)),
|
| 463 |
+
"verdict": verdict,
|
| 464 |
+
}
|
scenario_manager.py
ADDED
|
@@ -0,0 +1,600 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Preliminary SWMM scenario generation, execution, and comparison.
|
| 2 |
+
|
| 3 |
+
Rev22.1 intentionally treats generated storms and parameter changes as preliminary
|
| 4 |
+
engineering scenarios. It does not represent generated rainfall as an approved
|
| 5 |
+
City of Calgary design storm unless the user supplies and verifies the source.
|
| 6 |
+
"""
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
from dataclasses import asdict, dataclass, field
|
| 10 |
+
from datetime import datetime, timedelta
|
| 11 |
+
from io import BytesIO
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
from typing import Any, Iterable
|
| 14 |
+
import csv
|
| 15 |
+
import hashlib
|
| 16 |
+
import json
|
| 17 |
+
import math
|
| 18 |
+
import re
|
| 19 |
+
import tempfile
|
| 20 |
+
import zipfile
|
| 21 |
+
|
| 22 |
+
import pandas as pd
|
| 23 |
+
|
| 24 |
+
from swmm_core import run_swmm
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
@dataclass
|
| 28 |
+
class StormDefinition:
|
| 29 |
+
mode: str = "existing_model"
|
| 30 |
+
name: str = "Existing model rainfall"
|
| 31 |
+
return_period: str = "Model-defined"
|
| 32 |
+
duration_minutes: int = 60
|
| 33 |
+
interval_minutes: int = 5
|
| 34 |
+
total_depth_mm: float | None = None
|
| 35 |
+
peak_position: float = 0.40
|
| 36 |
+
source_status: str = "Existing model input"
|
| 37 |
+
source_reference: str = ""
|
| 38 |
+
selected_timeseries: str = ""
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
@dataclass
|
| 42 |
+
class ScenarioChange:
|
| 43 |
+
object_type: str
|
| 44 |
+
object_id: str
|
| 45 |
+
parameter: str
|
| 46 |
+
old_value: Any = None
|
| 47 |
+
new_value: Any = None
|
| 48 |
+
status: str = "requested"
|
| 49 |
+
note: str = ""
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
@dataclass
|
| 53 |
+
class ScenarioDefinition:
|
| 54 |
+
scenario_id: str
|
| 55 |
+
scenario_name: str
|
| 56 |
+
description: str = ""
|
| 57 |
+
storm: StormDefinition = field(default_factory=StormDefinition)
|
| 58 |
+
imperviousness_overrides: dict[str, float] = field(default_factory=dict)
|
| 59 |
+
conduit_diameter_overrides: dict[str, float] = field(default_factory=dict)
|
| 60 |
+
conduit_roughness_overrides: dict[str, float] = field(default_factory=dict)
|
| 61 |
+
storage_depth_overrides: dict[str, float] = field(default_factory=dict)
|
| 62 |
+
simulation_hours: float | None = None
|
| 63 |
+
review_status: str = "Preliminary"
|
| 64 |
+
created_at: str = field(default_factory=lambda: datetime.now().isoformat(timespec="seconds"))
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def safe_scenario_id(value: str) -> str:
|
| 68 |
+
cleaned = re.sub(r"[^A-Za-z0-9_-]+", "_", value.strip()).strip("_")
|
| 69 |
+
return cleaned[:64] or "scenario"
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def _split_sections(text: str) -> tuple[list[str], dict[str, list[str]]]:
|
| 73 |
+
preamble: list[str] = []
|
| 74 |
+
sections: dict[str, list[str]] = {}
|
| 75 |
+
current: str | None = None
|
| 76 |
+
for raw in text.splitlines():
|
| 77 |
+
m = re.match(r"^\s*\[([^]]+)\]\s*$", raw)
|
| 78 |
+
if m:
|
| 79 |
+
current = m.group(1).upper()
|
| 80 |
+
sections.setdefault(current, [])
|
| 81 |
+
elif current is None:
|
| 82 |
+
preamble.append(raw)
|
| 83 |
+
else:
|
| 84 |
+
sections[current].append(raw)
|
| 85 |
+
return preamble, sections
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def _join_sections(preamble: list[str], sections: dict[str, list[str]]) -> str:
|
| 89 |
+
out = list(preamble)
|
| 90 |
+
if out and out[-1].strip():
|
| 91 |
+
out.append("")
|
| 92 |
+
for name, lines in sections.items():
|
| 93 |
+
out.append(f"[{name}]")
|
| 94 |
+
out.extend(lines)
|
| 95 |
+
out.append("")
|
| 96 |
+
return "\n".join(out).rstrip() + "\n"
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def _data_tokens(line: str) -> tuple[list[str], str] | None:
|
| 100 |
+
body, sep, comment = line.partition(";")
|
| 101 |
+
if not body.strip() or body.lstrip().startswith(";"):
|
| 102 |
+
return None
|
| 103 |
+
return body.split(), (sep + comment if sep else "")
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def _replace_tokens(line: str, tokens: list[str], comment: str) -> str:
|
| 107 |
+
return " ".join(str(x) for x in tokens) + ((" " + comment) if comment else "")
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def _update_named_row(lines: list[str], object_id: str, token_index: int, new_value: Any) -> tuple[list[str], Any, bool]:
|
| 111 |
+
updated: list[str] = []
|
| 112 |
+
old = None
|
| 113 |
+
changed = False
|
| 114 |
+
for line in lines:
|
| 115 |
+
parsed = _data_tokens(line)
|
| 116 |
+
if parsed and parsed[0] and parsed[0][0].casefold() == object_id.casefold():
|
| 117 |
+
tokens, comment = parsed
|
| 118 |
+
if len(tokens) > token_index:
|
| 119 |
+
old = tokens[token_index]
|
| 120 |
+
tokens[token_index] = f"{new_value:g}" if isinstance(new_value, float) else str(new_value)
|
| 121 |
+
line = _replace_tokens(line, tokens, comment)
|
| 122 |
+
changed = True
|
| 123 |
+
updated.append(line)
|
| 124 |
+
return updated, old, changed
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def _set_option(lines: list[str], key: str, value: str) -> list[str]:
|
| 128 |
+
result: list[str] = []
|
| 129 |
+
found = False
|
| 130 |
+
for line in lines:
|
| 131 |
+
parsed = _data_tokens(line)
|
| 132 |
+
if parsed and parsed[0][0].upper() == key.upper():
|
| 133 |
+
tokens, comment = parsed
|
| 134 |
+
tokens = [key, value]
|
| 135 |
+
line = _replace_tokens(line, tokens, comment)
|
| 136 |
+
found = True
|
| 137 |
+
result.append(line)
|
| 138 |
+
if not found:
|
| 139 |
+
result.append(f"{key:<20} {value}")
|
| 140 |
+
return result
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def chicago_style_incremental_depths(total_depth_mm: float, duration_minutes: int, interval_minutes: int, peak_position: float = 0.4) -> list[float]:
|
| 144 |
+
"""Create a preliminary, mass-conserving Chicago-style alternating-block storm.
|
| 145 |
+
|
| 146 |
+
This is deliberately labelled preliminary. It is useful for scenario workflow
|
| 147 |
+
testing, but it is not a replacement for a verified municipal IDF-based storm.
|
| 148 |
+
"""
|
| 149 |
+
n = max(1, int(math.ceil(duration_minutes / interval_minutes)))
|
| 150 |
+
# Smooth synthetic intensity pattern with a sharp peak and positive tails.
|
| 151 |
+
ranks = list(range(1, n + 1))
|
| 152 |
+
weights = [1.0 / (r ** 0.72) for r in ranks]
|
| 153 |
+
scale = total_depth_mm / sum(weights)
|
| 154 |
+
blocks = [w * scale for w in weights]
|
| 155 |
+
peak_idx = min(n - 1, max(0, round((n - 1) * peak_position)))
|
| 156 |
+
order = [peak_idx]
|
| 157 |
+
offset = 1
|
| 158 |
+
while len(order) < n:
|
| 159 |
+
right = peak_idx + offset
|
| 160 |
+
left = peak_idx - offset
|
| 161 |
+
if right < n:
|
| 162 |
+
order.append(right)
|
| 163 |
+
if left >= 0 and len(order) < n:
|
| 164 |
+
order.append(left)
|
| 165 |
+
offset += 1
|
| 166 |
+
result = [0.0] * n
|
| 167 |
+
for block, idx in zip(sorted(blocks, reverse=True), order):
|
| 168 |
+
result[idx] = block
|
| 169 |
+
# exact conservation after floating-point arithmetic
|
| 170 |
+
result[-1] += total_depth_mm - sum(result)
|
| 171 |
+
return result
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
def _inject_preliminary_storm(sections: dict[str, list[str]], storm: StormDefinition, scenario_id: str) -> None:
|
| 175 |
+
if storm.total_depth_mm is None:
|
| 176 |
+
raise ValueError("Total storm depth is required for a generated preliminary storm.")
|
| 177 |
+
interval = max(1, int(storm.interval_minutes))
|
| 178 |
+
duration = max(interval, int(storm.duration_minutes))
|
| 179 |
+
series_id = f"SCN_{safe_scenario_id(scenario_id)}_RAIN"
|
| 180 |
+
depths = chicago_style_incremental_depths(float(storm.total_depth_mm), duration, interval, float(storm.peak_position))
|
| 181 |
+
|
| 182 |
+
# Replace/add the time series. Values are interval rainfall depths; the gage is VOLUME.
|
| 183 |
+
ts_lines = [ln for ln in sections.get("TIMESERIES", []) if not (_data_tokens(ln) and _data_tokens(ln)[0][0].casefold() == series_id.casefold())]
|
| 184 |
+
start = datetime(2000, 1, 1, 0, 0)
|
| 185 |
+
ts_lines.append(f"; Preliminary scenario storm: {storm.name}; source status: {storm.source_status}")
|
| 186 |
+
for i, depth in enumerate(depths):
|
| 187 |
+
t = start + timedelta(minutes=i * interval)
|
| 188 |
+
ts_lines.append(f"{series_id:<24} {t.strftime('%m/%d/%Y')} {t.strftime('%H:%M')} {depth:.6f}")
|
| 189 |
+
sections["TIMESERIES"] = ts_lines
|
| 190 |
+
|
| 191 |
+
# Point all model rain gages to the generated series while retaining their IDs.
|
| 192 |
+
rg_lines: list[str] = []
|
| 193 |
+
for line in sections.get("RAINGAGES", []):
|
| 194 |
+
parsed = _data_tokens(line)
|
| 195 |
+
if parsed:
|
| 196 |
+
tokens, comment = parsed
|
| 197 |
+
if len(tokens) >= 1:
|
| 198 |
+
gage_id = tokens[0]
|
| 199 |
+
tokens = [gage_id, "VOLUME", f"0:{interval:02d}", "1.0", "TIMESERIES", series_id]
|
| 200 |
+
line = _replace_tokens(line, tokens, comment)
|
| 201 |
+
rg_lines.append(line)
|
| 202 |
+
sections["RAINGAGES"] = rg_lines
|
| 203 |
+
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
def extract_rainfall_event_catalog(text: str) -> list[dict[str, str]]:
|
| 207 |
+
"""Return rainfall time-series available inside an INP model.
|
| 208 |
+
|
| 209 |
+
SWMM models often retain several design-event series (for example 1:5 and
|
| 210 |
+
1:100) while only one series is referenced by the active rain gage.
|
| 211 |
+
"""
|
| 212 |
+
_, sections = _split_sections(text)
|
| 213 |
+
active = set()
|
| 214 |
+
gages = {}
|
| 215 |
+
for line in sections.get("RAINGAGES", []):
|
| 216 |
+
parsed = _data_tokens(line)
|
| 217 |
+
if not parsed:
|
| 218 |
+
continue
|
| 219 |
+
tokens, _ = parsed
|
| 220 |
+
if len(tokens) >= 6 and tokens[4].upper() == "TIMESERIES":
|
| 221 |
+
gages[tokens[0]] = tokens[5]
|
| 222 |
+
active.add(tokens[5].casefold())
|
| 223 |
+
ids=[]
|
| 224 |
+
seen=set()
|
| 225 |
+
for line in sections.get("TIMESERIES", []):
|
| 226 |
+
parsed=_data_tokens(line)
|
| 227 |
+
if not parsed:
|
| 228 |
+
continue
|
| 229 |
+
tokens,_=parsed
|
| 230 |
+
if not tokens:
|
| 231 |
+
continue
|
| 232 |
+
sid=tokens[0]
|
| 233 |
+
if sid.casefold() in seen:
|
| 234 |
+
continue
|
| 235 |
+
seen.add(sid.casefold())
|
| 236 |
+
ids.append({"event_id":sid,"active":"Yes" if sid.casefold() in active else "No","used_by":", ".join(k for k,v in gages.items() if v.casefold()==sid.casefold())})
|
| 237 |
+
return ids
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
def _select_existing_timeseries(sections: dict[str, list[str]], series_id: str) -> None:
|
| 241 |
+
"""Point every TIMESERIES-based rain gage to an existing model series."""
|
| 242 |
+
available={x["event_id"].casefold():x["event_id"] for x in extract_rainfall_event_catalog(_join_sections([], sections))}
|
| 243 |
+
if series_id.casefold() not in available:
|
| 244 |
+
raise ValueError(f"Rainfall time series {series_id!r} was not found in the selected model.")
|
| 245 |
+
canonical=available[series_id.casefold()]
|
| 246 |
+
new=[]
|
| 247 |
+
changed=0
|
| 248 |
+
for line in sections.get("RAINGAGES", []):
|
| 249 |
+
parsed=_data_tokens(line)
|
| 250 |
+
if parsed:
|
| 251 |
+
tokens,comment=parsed
|
| 252 |
+
if len(tokens)>=6 and tokens[4].upper()=="TIMESERIES":
|
| 253 |
+
tokens[5]=canonical
|
| 254 |
+
line=_replace_tokens(line,tokens,comment)
|
| 255 |
+
changed+=1
|
| 256 |
+
new.append(line)
|
| 257 |
+
if changed==0:
|
| 258 |
+
raise ValueError("The selected model has no TIMESERIES-based rain gage to assign.")
|
| 259 |
+
sections["RAINGAGES"]=new
|
| 260 |
+
|
| 261 |
+
def build_scenario_inp(base_inp: str | Path, scenario: ScenarioDefinition, output_path: str | Path) -> dict[str, Any]:
|
| 262 |
+
base = Path(base_inp)
|
| 263 |
+
text = base.read_text(encoding="utf-8", errors="ignore")
|
| 264 |
+
preamble, sections = _split_sections(text)
|
| 265 |
+
change_log: list[ScenarioChange] = []
|
| 266 |
+
|
| 267 |
+
if scenario.storm.mode == "generated_preliminary":
|
| 268 |
+
_inject_preliminary_storm(sections, scenario.storm, scenario.scenario_id)
|
| 269 |
+
change_log.append(ScenarioChange("storm", scenario.storm.name, "rainfall_series", None, scenario.storm.total_depth_mm, "applied", scenario.storm.source_status))
|
| 270 |
+
elif scenario.storm.mode == "existing_timeseries" and scenario.storm.selected_timeseries:
|
| 271 |
+
_select_existing_timeseries(sections, scenario.storm.selected_timeseries)
|
| 272 |
+
change_log.append(ScenarioChange("storm", scenario.storm.selected_timeseries, "active_timeseries", None, scenario.storm.selected_timeseries, "applied", "Existing series selected from model"))
|
| 273 |
+
|
| 274 |
+
if scenario.simulation_hours and scenario.simulation_hours > 0:
|
| 275 |
+
start = datetime(2000, 1, 1, 0, 0)
|
| 276 |
+
end = start + timedelta(hours=float(scenario.simulation_hours))
|
| 277 |
+
opts = sections.setdefault("OPTIONS", [])
|
| 278 |
+
for key, val in [
|
| 279 |
+
("START_DATE", start.strftime("%m/%d/%Y")),
|
| 280 |
+
("START_TIME", start.strftime("%H:%M:%S")),
|
| 281 |
+
("REPORT_START_DATE", start.strftime("%m/%d/%Y")),
|
| 282 |
+
("REPORT_START_TIME", start.strftime("%H:%M:%S")),
|
| 283 |
+
("END_DATE", end.strftime("%m/%d/%Y")),
|
| 284 |
+
("END_TIME", end.strftime("%H:%M:%S")),
|
| 285 |
+
]:
|
| 286 |
+
opts = _set_option(opts, key, val)
|
| 287 |
+
sections["OPTIONS"] = opts
|
| 288 |
+
|
| 289 |
+
# SWMM [SUBCATCHMENTS]: Name RainGage Outlet Area %Imperv Width %Slope ...
|
| 290 |
+
for oid, value in scenario.imperviousness_overrides.items():
|
| 291 |
+
lines, old, ok = _update_named_row(sections.get("SUBCATCHMENTS", []), oid, 4, float(value))
|
| 292 |
+
sections["SUBCATCHMENTS"] = lines
|
| 293 |
+
change_log.append(ScenarioChange("subcatchment", oid, "imperviousness_percent", old, value, "applied" if ok else "not_found"))
|
| 294 |
+
|
| 295 |
+
# SWMM [XSECTIONS]: Link Shape Geom1 ... ; circular Geom1 is diameter.
|
| 296 |
+
for oid, value in scenario.conduit_diameter_overrides.items():
|
| 297 |
+
lines, old, ok = _update_named_row(sections.get("XSECTIONS", []), oid, 2, float(value))
|
| 298 |
+
sections["XSECTIONS"] = lines
|
| 299 |
+
change_log.append(ScenarioChange("link", oid, "diameter_or_geom1", old, value, "applied" if ok else "not_found"))
|
| 300 |
+
|
| 301 |
+
# SWMM [CONDUITS]: Name From To Length Roughness ...
|
| 302 |
+
for oid, value in scenario.conduit_roughness_overrides.items():
|
| 303 |
+
lines, old, ok = _update_named_row(sections.get("CONDUITS", []), oid, 4, float(value))
|
| 304 |
+
sections["CONDUITS"] = lines
|
| 305 |
+
change_log.append(ScenarioChange("conduit", oid, "manning_n", old, value, "applied" if ok else "not_found"))
|
| 306 |
+
|
| 307 |
+
# SWMM [STORAGE]: Name Elev MaxDepth InitDepth Shape ...
|
| 308 |
+
for oid, value in scenario.storage_depth_overrides.items():
|
| 309 |
+
lines, old, ok = _update_named_row(sections.get("STORAGE", []), oid, 2, float(value))
|
| 310 |
+
sections["STORAGE"] = lines
|
| 311 |
+
change_log.append(ScenarioChange("storage", oid, "maximum_depth", old, value, "applied" if ok else "not_found"))
|
| 312 |
+
|
| 313 |
+
output = Path(output_path)
|
| 314 |
+
output.parent.mkdir(parents=True, exist_ok=True)
|
| 315 |
+
output.write_text(_join_sections(preamble, sections), encoding="utf-8")
|
| 316 |
+
return {
|
| 317 |
+
"scenario": asdict(scenario),
|
| 318 |
+
"changes": [asdict(x) for x in change_log],
|
| 319 |
+
"input_sha256": hashlib.sha256(output.read_bytes()).hexdigest(),
|
| 320 |
+
"output_path": str(output),
|
| 321 |
+
}
|
| 322 |
+
|
| 323 |
+
|
| 324 |
+
def summarize_scenario_results(results: dict[str, Any], scenario: ScenarioDefinition) -> dict[str, Any]:
|
| 325 |
+
node_ts = results.get("node_ts", {}) or {}
|
| 326 |
+
link_ts = results.get("link_ts", {}) or {}
|
| 327 |
+
sub_ts = results.get("sub_ts", {}) or {}
|
| 328 |
+
meta = results.get("metadata", {}) or {}
|
| 329 |
+
|
| 330 |
+
max_flood = max((max(v.get("flooding", []) or [0.0]) for v in node_ts.values()), default=0.0)
|
| 331 |
+
total_peak_inflow = max((max(v.get("inflow", []) or [0.0]) for v in node_ts.values()), default=0.0)
|
| 332 |
+
max_velocity = 0.0
|
| 333 |
+
max_velocity_link = ""
|
| 334 |
+
max_depth_ratio = 0.0
|
| 335 |
+
max_depth_ratio_link = ""
|
| 336 |
+
peak_link_flow = 0.0
|
| 337 |
+
for lid, values in link_ts.items():
|
| 338 |
+
vel = max((abs(float(x)) for x in values.get("velocity", []) or [0.0]), default=0.0)
|
| 339 |
+
if vel > max_velocity:
|
| 340 |
+
max_velocity, max_velocity_link = vel, lid
|
| 341 |
+
depth = max((float(x) for x in values.get("depth", []) or [0.0]), default=0.0)
|
| 342 |
+
diameter = float(values.get("diameter", 0.0) or 0.0)
|
| 343 |
+
ratio = depth / diameter if diameter > 0 else 0.0
|
| 344 |
+
if ratio > max_depth_ratio:
|
| 345 |
+
max_depth_ratio, max_depth_ratio_link = ratio, lid
|
| 346 |
+
peak_link_flow = max(peak_link_flow, max((abs(float(x)) for x in values.get("flow", []) or [0.0]), default=0.0))
|
| 347 |
+
peak_runoff = max((max(v.get("runoff", []) or [0.0]) for v in sub_ts.values()), default=0.0)
|
| 348 |
+
return {
|
| 349 |
+
"Scenario ID": scenario.scenario_id,
|
| 350 |
+
"Scenario Name": scenario.scenario_name,
|
| 351 |
+
"Storm": scenario.storm.name,
|
| 352 |
+
"Storm Status": scenario.storm.source_status,
|
| 353 |
+
"Simulation Status": "Completed",
|
| 354 |
+
"Runoff Error (%)": float(meta.get("runoff_error", 0.0) or 0.0),
|
| 355 |
+
"Flow Error (%)": float(meta.get("flow_error", 0.0) or 0.0),
|
| 356 |
+
"Peak Subcatchment Runoff": peak_runoff,
|
| 357 |
+
"Peak Link Flow": peak_link_flow,
|
| 358 |
+
"Maximum Link Velocity": max_velocity,
|
| 359 |
+
"Velocity Link": max_velocity_link,
|
| 360 |
+
"Maximum Modelled Depth Ratio": max_depth_ratio,
|
| 361 |
+
"Depth-Ratio Link": max_depth_ratio_link,
|
| 362 |
+
"Maximum Node Flooding": max_flood,
|
| 363 |
+
"Maximum Node Inflow": total_peak_inflow,
|
| 364 |
+
"Review Status": scenario.review_status,
|
| 365 |
+
"Model Role": "Scenario",
|
| 366 |
+
}
|
| 367 |
+
|
| 368 |
+
|
| 369 |
+
def run_scenario(base_inp: str | Path, scenario: ScenarioDefinition, work_dir: str | Path | None = None) -> dict[str, Any]:
|
| 370 |
+
root = Path(work_dir) if work_dir else Path(tempfile.mkdtemp(prefix="swmm_scenario_"))
|
| 371 |
+
root.mkdir(parents=True, exist_ok=True)
|
| 372 |
+
sid = safe_scenario_id(scenario.scenario_id)
|
| 373 |
+
inp = root / f"{sid}.inp"
|
| 374 |
+
manifest = build_scenario_inp(base_inp, scenario, inp)
|
| 375 |
+
rpt = root / f"{sid}.rpt"
|
| 376 |
+
out = root / f"{sid}.out"
|
| 377 |
+
results = run_swmm(inp, rpt, out)
|
| 378 |
+
# Attach final-input geometry to result records so scenario depth-ratio
|
| 379 |
+
# summaries are computed from the scenario model rather than shown as zero.
|
| 380 |
+
final_sections = {}
|
| 381 |
+
try:
|
| 382 |
+
_, final_sections = _split_sections(inp.read_text(encoding="utf-8", errors="ignore"))
|
| 383 |
+
diameters = {}
|
| 384 |
+
for line in final_sections.get("XSECTIONS", []):
|
| 385 |
+
parsed = _data_tokens(line)
|
| 386 |
+
if parsed:
|
| 387 |
+
tokens, _ = parsed
|
| 388 |
+
if len(tokens) >= 3:
|
| 389 |
+
try:
|
| 390 |
+
diameters[tokens[0]] = float(tokens[2])
|
| 391 |
+
except ValueError:
|
| 392 |
+
pass
|
| 393 |
+
for lid, values in (results.get("link_ts", {}) or {}).items():
|
| 394 |
+
if lid in diameters:
|
| 395 |
+
values["diameter"] = diameters[lid]
|
| 396 |
+
except Exception:
|
| 397 |
+
pass
|
| 398 |
+
summary = summarize_scenario_results(results, scenario)
|
| 399 |
+
summary["Conduit Count"] = len(final_sections.get("CONDUITS", []) or [])
|
| 400 |
+
storage_ids = [(_data_tokens(line)[0][0] if _data_tokens(line) else "") for line in (final_sections.get("STORAGE", []) or [])]
|
| 401 |
+
storage_ids = [x for x in storage_ids if x]
|
| 402 |
+
storage_depth = 0.0
|
| 403 |
+
storage_volume = 0.0
|
| 404 |
+
storage_node = ""
|
| 405 |
+
for node_id in storage_ids:
|
| 406 |
+
values = (results.get("node_ts", {}) or {}).get(node_id, {})
|
| 407 |
+
depth = max((float(x) for x in values.get("depth", []) or [0.0]), default=0.0)
|
| 408 |
+
volume = max((float(x) for x in values.get("volume", []) or [0.0]), default=0.0)
|
| 409 |
+
if depth > storage_depth:
|
| 410 |
+
storage_depth, storage_node = depth, node_id
|
| 411 |
+
storage_volume = max(storage_volume, volume)
|
| 412 |
+
summary["Maximum Storage Depth"] = storage_depth if storage_ids else None
|
| 413 |
+
summary["Maximum Storage Volume"] = storage_volume if storage_ids else None
|
| 414 |
+
summary["Controlling Storage Node"] = storage_node if storage_ids else ""
|
| 415 |
+
manifest.update({
|
| 416 |
+
"report_path": str(rpt),
|
| 417 |
+
"output_path": str(out),
|
| 418 |
+
"report_sha256": hashlib.sha256(rpt.read_bytes()).hexdigest() if rpt.exists() else None,
|
| 419 |
+
"summary": summary,
|
| 420 |
+
})
|
| 421 |
+
return {"definition": asdict(scenario), "manifest": manifest, "results": results, "summary": summary, "files": {"inp": inp.read_bytes(), "rpt": rpt.read_bytes() if rpt.exists() else b"", "out": out.read_bytes() if out.exists() else b""}}
|
| 422 |
+
|
| 423 |
+
|
| 424 |
+
|
| 425 |
+
def base_model_record(results: dict[str, Any], *, scenario_name: str = "Base Model", review_status: str = "Reference model", source_name: str = "Base Model") -> dict[str, Any]:
|
| 426 |
+
"""Create a comparison record for the currently simulated base model."""
|
| 427 |
+
definition = ScenarioDefinition(
|
| 428 |
+
scenario_id="BASE_MODEL",
|
| 429 |
+
scenario_name=scenario_name,
|
| 430 |
+
description="Current uploaded and simulated reference model.",
|
| 431 |
+
storm=StormDefinition(
|
| 432 |
+
mode="existing_model",
|
| 433 |
+
name="Existing model rainfall",
|
| 434 |
+
return_period="Model-defined",
|
| 435 |
+
source_status="Base model input",
|
| 436 |
+
),
|
| 437 |
+
review_status=review_status,
|
| 438 |
+
)
|
| 439 |
+
summary = summarize_scenario_results(results or {}, definition)
|
| 440 |
+
summary["Model Role"] = "Base"
|
| 441 |
+
summary["Source Model"] = source_name
|
| 442 |
+
summary["Input Changed"] = "Reference"
|
| 443 |
+
summary["Input SHA256 (short)"] = ""
|
| 444 |
+
summary["Conduit Count"] = sum(1 for values in ((results or {}).get("link_ts", {}) or {}).values() if float(values.get("diameter", 0) or 0) > 0)
|
| 445 |
+
return {
|
| 446 |
+
"definition": asdict(definition),
|
| 447 |
+
"manifest": {"summary": summary, "model_role": "Base", "source": "Current uploaded model"},
|
| 448 |
+
"results": results or {},
|
| 449 |
+
"summary": summary,
|
| 450 |
+
"files": {},
|
| 451 |
+
}
|
| 452 |
+
|
| 453 |
+
|
| 454 |
+
def comparison_with_base(base_results: dict[str, Any] | None, records: Iterable[dict[str, Any]], *, base_name: str = "Base Model") -> pd.DataFrame:
|
| 455 |
+
"""Return one comparison table containing the base model and scenario records."""
|
| 456 |
+
combined: list[dict[str, Any]] = []
|
| 457 |
+
if base_results:
|
| 458 |
+
combined.append(base_model_record(base_results, scenario_name=base_name, source_name=base_name))
|
| 459 |
+
combined.extend(list(records))
|
| 460 |
+
df = comparison_dataframe(combined)
|
| 461 |
+
if not df.empty and "Model Role" not in df.columns:
|
| 462 |
+
df.insert(0, "Model Role", ["Base" if str(v) == "BASE_MODEL" else "Scenario" for v in df.get("Scenario ID", [])])
|
| 463 |
+
elif not df.empty:
|
| 464 |
+
df["Model Role"] = df["Model Role"].fillna("Scenario")
|
| 465 |
+
if not df.empty:
|
| 466 |
+
base_rows = df[df["Scenario ID"].astype(str) == "BASE_MODEL"]
|
| 467 |
+
if not base_rows.empty:
|
| 468 |
+
b = base_rows.iloc[0]
|
| 469 |
+
metric_cols = ["Peak Subcatchment Runoff", "Peak Link Flow", "Maximum Link Velocity", "Maximum Modelled Depth Ratio", "Maximum Node Flooding", "Maximum Node Inflow"]
|
| 470 |
+
flags = []
|
| 471 |
+
for _, row in df.iterrows():
|
| 472 |
+
if str(row.get("Scenario ID")) == "BASE_MODEL":
|
| 473 |
+
flags.append("Reference")
|
| 474 |
+
continue
|
| 475 |
+
changed = False
|
| 476 |
+
for col in metric_cols:
|
| 477 |
+
try:
|
| 478 |
+
if abs(float(row.get(col, 0) or 0) - float(b.get(col, 0) or 0)) > 1e-8:
|
| 479 |
+
changed = True
|
| 480 |
+
break
|
| 481 |
+
except Exception:
|
| 482 |
+
pass
|
| 483 |
+
flags.append("Different" if changed else "No summary-level difference")
|
| 484 |
+
df["Hydraulic Difference"] = flags
|
| 485 |
+
return df
|
| 486 |
+
|
| 487 |
+
|
| 488 |
+
def deterministic_comparison_analysis(comparison: pd.DataFrame) -> str:
|
| 489 |
+
"""Create a conservative, deterministic scenario-comparison narrative."""
|
| 490 |
+
if comparison is None or comparison.empty:
|
| 491 |
+
return "No scenario comparison data are available."
|
| 492 |
+
df = comparison.copy()
|
| 493 |
+
base = df[df["Scenario ID"].astype(str) == "BASE_MODEL"]
|
| 494 |
+
if base.empty:
|
| 495 |
+
base = df.iloc[[0]]
|
| 496 |
+
b = base.iloc[0]
|
| 497 |
+
metrics = [
|
| 498 |
+
("Peak Subcatchment Runoff", "peak subcatchment runoff"),
|
| 499 |
+
("Peak Link Flow", "peak link flow"),
|
| 500 |
+
("Maximum Link Velocity", "maximum link velocity"),
|
| 501 |
+
("Maximum Modelled Depth Ratio", "maximum modelled depth ratio"),
|
| 502 |
+
("Maximum Node Flooding", "maximum node flooding"),
|
| 503 |
+
("Maximum Node Inflow", "maximum node inflow"),
|
| 504 |
+
("Maximum Storage Depth", "maximum storage depth"),
|
| 505 |
+
("Maximum Storage Volume", "maximum storage volume"),
|
| 506 |
+
]
|
| 507 |
+
lines = [
|
| 508 |
+
"The comparison uses the current uploaded model as the reference case. Differences are calculated from deterministic simulation summaries and are intended for preliminary engineering review.",
|
| 509 |
+
"",
|
| 510 |
+
]
|
| 511 |
+
# Guard: if every base metric is zero while at least one scenario is
|
| 512 |
+
# non-zero, the base record almost certainly did not carry usable
|
| 513 |
+
# results (e.g. it was built from an empty or malformed results dict).
|
| 514 |
+
# Deltas quoted against a zeroed base are misleading, so say so
|
| 515 |
+
# explicitly instead of presenting "+X relative to base" as fact.
|
| 516 |
+
def _metric_value(record, col):
|
| 517 |
+
try:
|
| 518 |
+
return abs(float(record.get(col, 0) or 0))
|
| 519 |
+
except Exception:
|
| 520 |
+
return 0.0
|
| 521 |
+
base_all_zero = all(_metric_value(b, col) <= 1e-12 for col, _ in metrics)
|
| 522 |
+
any_scenario_nonzero = any(
|
| 523 |
+
_metric_value(row, col) > 1e-12
|
| 524 |
+
for _, row in df[df["Scenario ID"].astype(str) != str(b.get("Scenario ID", "BASE_MODEL"))].iterrows()
|
| 525 |
+
for col, _ in metrics
|
| 526 |
+
)
|
| 527 |
+
if base_all_zero and any_scenario_nonzero:
|
| 528 |
+
lines.insert(0, (
|
| 529 |
+
"CAUTION: All reference-case metrics are zero while scenario results are non-zero. "
|
| 530 |
+
"The base record does not appear to contain usable simulation results; differences below "
|
| 531 |
+
"are NOT valid deltas against the uploaded model. Re-simulate or rebuild the base record "
|
| 532 |
+
"before relying on this comparison."
|
| 533 |
+
))
|
| 534 |
+
lines.insert(1, "")
|
| 535 |
+
scenarios = df[df["Scenario ID"].astype(str) != str(b.get("Scenario ID", "BASE_MODEL"))]
|
| 536 |
+
if scenarios.empty:
|
| 537 |
+
lines.append("No alternative scenario has been completed.")
|
| 538 |
+
return "\n".join(lines)
|
| 539 |
+
for _, row in scenarios.iterrows():
|
| 540 |
+
sid = row.get("Scenario ID", "Scenario")
|
| 541 |
+
name = row.get("Scenario Name", sid)
|
| 542 |
+
lines.append(f"{name} ({sid}):")
|
| 543 |
+
changes=[]
|
| 544 |
+
no_conduits = int(row.get("Conduit Count", 0) or 0) == 0
|
| 545 |
+
for col,label in metrics:
|
| 546 |
+
if no_conduits and col in {"Maximum Link Velocity", "Maximum Modelled Depth Ratio"}:
|
| 547 |
+
changes.append(f"- {label}: Not applicable — no conduits in model")
|
| 548 |
+
continue
|
| 549 |
+
try:
|
| 550 |
+
bv=float(b.get(col,0) or 0); sv=float(row.get(col,0) or 0)
|
| 551 |
+
except Exception:
|
| 552 |
+
continue
|
| 553 |
+
delta=sv-bv
|
| 554 |
+
if abs(bv)>1e-12:
|
| 555 |
+
pct=100*delta/abs(bv)
|
| 556 |
+
changes.append(f"- {label}: {sv:.4g} ({delta:+.4g}; {pct:+.1f}% relative to base)")
|
| 557 |
+
else:
|
| 558 |
+
changes.append(f"- {label}: {sv:.4g} ({delta:+.4g} relative to base)")
|
| 559 |
+
lines.extend(changes)
|
| 560 |
+
if str(row.get("Hydraulic Difference", "")) == "No summary-level difference":
|
| 561 |
+
lines.append("- No difference was detected in the reported summary metrics. Check the input-change status and detailed time series before treating this as a distinct hydraulic alternative.")
|
| 562 |
+
if str(row.get("Input Changed", "")) == "No":
|
| 563 |
+
lines.append("- The final scenario input is identical to the base-model input; identical results are expected.")
|
| 564 |
+
if float(row.get("Maximum Node Flooding",0) or 0)>0:
|
| 565 |
+
lines.append("- Modelled node flooding is present and requires review.")
|
| 566 |
+
if abs(float(row.get("Runoff Error (%)",0) or 0))>1 or abs(float(row.get("Flow Error (%)",0) or 0))>1:
|
| 567 |
+
lines.append("- Numerical continuity exceeds 1% for at least one reported balance and requires review.")
|
| 568 |
+
lines.append("")
|
| 569 |
+
lines.append("The comparison does not establish design adequacy or compliance. The responsible engineer must review model changes, storm sources, controlling elements, physical feasibility, and applicable project criteria.")
|
| 570 |
+
return "\n".join(lines).strip()
|
| 571 |
+
|
| 572 |
+
def comparison_dataframe(records: Iterable[dict[str, Any]]) -> pd.DataFrame:
|
| 573 |
+
rows = [r.get("summary", {}) for r in records]
|
| 574 |
+
return pd.DataFrame(rows)
|
| 575 |
+
|
| 576 |
+
|
| 577 |
+
def build_scenario_package(records: Iterable[dict[str, Any]]) -> bytes:
|
| 578 |
+
records = list(records)
|
| 579 |
+
buffer = BytesIO()
|
| 580 |
+
with zipfile.ZipFile(buffer, "w", compression=zipfile.ZIP_DEFLATED) as zf:
|
| 581 |
+
comparison = comparison_dataframe(records)
|
| 582 |
+
zf.writestr("scenario_comparison.csv", comparison.to_csv(index=False))
|
| 583 |
+
manifest = []
|
| 584 |
+
for rec in records:
|
| 585 |
+
sid = safe_scenario_id(rec.get("definition", {}).get("scenario_id", "scenario"))
|
| 586 |
+
base = f"scenarios/{sid}"
|
| 587 |
+
editable_clone = rec.get("files", {}).get("editable_clone", b"")
|
| 588 |
+
if editable_clone:
|
| 589 |
+
zf.writestr(f"{base}/{sid}_editable_clone_source.inp", editable_clone)
|
| 590 |
+
zf.writestr(f"{base}/{sid}.inp", rec.get("files", {}).get("inp", b""))
|
| 591 |
+
zf.writestr(f"{base}/{sid}.rpt", rec.get("files", {}).get("rpt", b""))
|
| 592 |
+
out_bytes = rec.get("files", {}).get("out", b"")
|
| 593 |
+
if out_bytes:
|
| 594 |
+
zf.writestr(f"{base}/{sid}.out", out_bytes)
|
| 595 |
+
zf.writestr(f"{base}/scenario_definition.json", json.dumps(rec.get("definition", {}), indent=2, default=str))
|
| 596 |
+
zf.writestr(f"{base}/scenario_manifest.json", json.dumps(rec.get("manifest", {}), indent=2, default=str))
|
| 597 |
+
manifest.append(rec.get("manifest", {}))
|
| 598 |
+
zf.writestr("scenario_register.json", json.dumps(manifest, indent=2, default=str))
|
| 599 |
+
zf.writestr("README.txt", "Rev22.1 preliminary scenario package. Editable clone source files are preserved separately from final scenario input files. Generated storms and parameter changes require professional verification before design use or municipal submission.\n")
|
| 600 |
+
return buffer.getvalue()
|
server.py
ADDED
|
@@ -0,0 +1,231 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""SWMM Analysis MCP Server — dual-surface deployment for Hugging Face Spaces.
|
| 2 |
+
|
| 3 |
+
Surfaces
|
| 4 |
+
--------
|
| 5 |
+
1. MCP (Streamable HTTP) at /mcp
|
| 6 |
+
For MCP-native clients: Claude Desktop & claude.ai custom connectors,
|
| 7 |
+
Gemini CLI/clients, ChatGPT connectors, LangChain MCP adapters, n8n MCP
|
| 8 |
+
Client node, Flowise/Langflow MCP tools, Codex CLI. Stateless HTTP +
|
| 9 |
+
JSON responses for maximum client compatibility; DNS-rebinding
|
| 10 |
+
protection disabled (required behind the HF Spaces proxy).
|
| 11 |
+
|
| 12 |
+
2. REST/OpenAPI at /api/* (schema at /openapi.json)
|
| 13 |
+
Every registry tool as POST /api/tool/{name}; used by Custom GPT
|
| 14 |
+
Actions, n8n HTTP Request nodes, plain webhooks, and anything that
|
| 15 |
+
speaks OpenAPI rather than MCP.
|
| 16 |
+
|
| 17 |
+
3. Built-in agent at POST /api/agent (also MCP tool `agent_analyze`)
|
| 18 |
+
A multi-provider LLM tool-loop over the same registry, for callers that
|
| 19 |
+
want one natural-language endpoint (see agent.py).
|
| 20 |
+
|
| 21 |
+
4. GET /files/{session_id}/{filename} — generated report downloads.
|
| 22 |
+
|
| 23 |
+
Both surfaces dispatch to the same registry in tools.py, so behaviour is
|
| 24 |
+
identical on every platform.
|
| 25 |
+
"""
|
| 26 |
+
from __future__ import annotations
|
| 27 |
+
|
| 28 |
+
import contextlib
|
| 29 |
+
import inspect
|
| 30 |
+
import json
|
| 31 |
+
import os
|
| 32 |
+
from pathlib import Path
|
| 33 |
+
from typing import Any
|
| 34 |
+
|
| 35 |
+
from fastapi import Body, FastAPI, HTTPException
|
| 36 |
+
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
|
| 37 |
+
|
| 38 |
+
from mcp.server.fastmcp import FastMCP
|
| 39 |
+
from mcp.server.transport_security import TransportSecuritySettings
|
| 40 |
+
|
| 41 |
+
import agent as agent_module
|
| 42 |
+
from sessions import SESSION_ROOT, STORE
|
| 43 |
+
from tools import TOOL_REGISTRY
|
| 44 |
+
|
| 45 |
+
SERVER_NAME = "swmm-analysis"
|
| 46 |
+
SERVER_VERSION = "1.0.0 (engine Rev 23.2)"
|
| 47 |
+
|
| 48 |
+
# ---------------------------------------------------------------------------
|
| 49 |
+
# MCP surface
|
| 50 |
+
# ---------------------------------------------------------------------------
|
| 51 |
+
# Known HF-Spaces deployment fixes, learned the hard way on prior servers:
|
| 52 |
+
# 1. TransportSecuritySettings(enable_dns_rebinding_protection=False):
|
| 53 |
+
# the Spaces proxy rewrites Host headers and rebinding protection
|
| 54 |
+
# rejects every request without this.
|
| 55 |
+
# 2. The MCP session manager lifespan MUST be wired into the outer FastAPI
|
| 56 |
+
# app (dropping it silently breaks streamable HTTP after startup).
|
| 57 |
+
# 3. README.md needs its YAML front-matter header or the Space won't build.
|
| 58 |
+
mcp = FastMCP(
|
| 59 |
+
SERVER_NAME,
|
| 60 |
+
instructions=(
|
| 61 |
+
"Deterministic EPA-SWMM stormwater analysis: upload a .inp model, run the "
|
| 62 |
+
"crash-isolated simulation, then query results, Calgary-style screening, "
|
| 63 |
+
"QA/QC findings, engine-report reconciliation, controlled scenarios, and "
|
| 64 |
+
"SWMR draft reports. Typical flow: upload_model -> run_simulation -> "
|
| 65 |
+
"analysis tools with the returned session_id. All outputs are preliminary "
|
| 66 |
+
"engineering screening, not professional determinations."
|
| 67 |
+
),
|
| 68 |
+
stateless_http=True,
|
| 69 |
+
json_response=True,
|
| 70 |
+
transport_security=TransportSecuritySettings(enable_dns_rebinding_protection=False),
|
| 71 |
+
)
|
| 72 |
+
|
| 73 |
+
for _name, _fn in TOOL_REGISTRY.items():
|
| 74 |
+
mcp.tool()(_fn)
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
@mcp.tool()
|
| 78 |
+
def agent_analyze(question: str, provider: str = "anthropic", model: str = "",
|
| 79 |
+
session_id: str = "", api_key: str = "", base_url: str = "") -> dict:
|
| 80 |
+
"""Ask the built-in agent a natural-language question; it plans and runs the
|
| 81 |
+
SWMM tools itself and returns an answer plus a full tool audit trail.
|
| 82 |
+
Providers: anthropic, openai, gemini, groq, mistral, local (keys via Space
|
| 83 |
+
secrets or api_key argument). MCP clients normally drive tools directly —
|
| 84 |
+
use this when you want server-side orchestration."""
|
| 85 |
+
return agent_module.run_agent(
|
| 86 |
+
question=question, provider=provider, model=model or None,
|
| 87 |
+
api_key=api_key or None, base_url=base_url or None,
|
| 88 |
+
session_id=session_id or None)
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
mcp_app = mcp.streamable_http_app()
|
| 92 |
+
|
| 93 |
+
# ---------------------------------------------------------------------------
|
| 94 |
+
# FastAPI app with the MCP lifespan wired in (fix #2)
|
| 95 |
+
# ---------------------------------------------------------------------------
|
| 96 |
+
|
| 97 |
+
@contextlib.asynccontextmanager
|
| 98 |
+
async def lifespan(app: FastAPI):
|
| 99 |
+
async with contextlib.AsyncExitStack() as stack:
|
| 100 |
+
await stack.enter_async_context(mcp.session_manager.run())
|
| 101 |
+
yield
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
app = FastAPI(
|
| 105 |
+
title="SWMM Analysis MCP Server",
|
| 106 |
+
version=SERVER_VERSION,
|
| 107 |
+
description="Dual-surface (MCP + REST) EPA-SWMM analysis server with a built-in multi-provider agent. "
|
| 108 |
+
"MCP endpoint: /mcp. Engine: OpenSWMM in a crash-isolated worker, Calgary screening, "
|
| 109 |
+
"deterministic QA/QC, .rpt reconciliation, scenarios, SWMR reporting.",
|
| 110 |
+
lifespan=lifespan,
|
| 111 |
+
)
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
# ---------------------------------------------------------------------------
|
| 115 |
+
# REST surface — one endpoint per tool, plus a generic dispatcher
|
| 116 |
+
# ---------------------------------------------------------------------------
|
| 117 |
+
|
| 118 |
+
def _tool_meta(name: str, fn) -> dict:
|
| 119 |
+
sig = inspect.signature(fn)
|
| 120 |
+
return {
|
| 121 |
+
"name": name,
|
| 122 |
+
"description": (fn.__doc__ or "").strip(),
|
| 123 |
+
"parameters": {
|
| 124 |
+
p: {"required": prm.default is inspect.Parameter.empty,
|
| 125 |
+
"default": None if prm.default is inspect.Parameter.empty else prm.default}
|
| 126 |
+
for p, prm in sig.parameters.items()
|
| 127 |
+
},
|
| 128 |
+
"rest": f"POST /api/tool/{name}",
|
| 129 |
+
}
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
@app.get("/api/tools")
|
| 133 |
+
def rest_list_tools() -> dict:
|
| 134 |
+
"""List all tools with parameter metadata (machine-readable)."""
|
| 135 |
+
return {"server": SERVER_NAME, "version": SERVER_VERSION,
|
| 136 |
+
"tools": [_tool_meta(n, f) for n, f in TOOL_REGISTRY.items()]}
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
@app.post("/api/tool/{tool_name}")
|
| 140 |
+
def rest_call_tool(tool_name: str, payload: dict = Body(default={})) -> JSONResponse:
|
| 141 |
+
"""Invoke any registry tool. Body = the tool's keyword arguments as JSON."""
|
| 142 |
+
fn = TOOL_REGISTRY.get(tool_name)
|
| 143 |
+
if fn is None:
|
| 144 |
+
raise HTTPException(404, f"Unknown tool '{tool_name}'. See /api/tools.")
|
| 145 |
+
try:
|
| 146 |
+
result = fn(**(payload or {}))
|
| 147 |
+
except (KeyError, ValueError, TypeError) as exc:
|
| 148 |
+
raise HTTPException(400, str(exc))
|
| 149 |
+
except Exception as exc:
|
| 150 |
+
raise HTTPException(500, f"{type(exc).__name__}: {exc}")
|
| 151 |
+
return JSONResponse(json.loads(json.dumps(result, default=str)))
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
@app.post("/api/agent")
|
| 155 |
+
def rest_agent(payload: dict = Body(...)) -> JSONResponse:
|
| 156 |
+
"""Built-in agent endpoint.
|
| 157 |
+
|
| 158 |
+
Body: {"question": str, "provider": "anthropic|openai|gemini|groq|mistral|local",
|
| 159 |
+
"model": str?, "api_key": str?, "base_url": str?, "session_id": str?,
|
| 160 |
+
"inp_content": str?, "allow_report": bool?}
|
| 161 |
+
"""
|
| 162 |
+
question = payload.get("question", "").strip()
|
| 163 |
+
if not question:
|
| 164 |
+
raise HTTPException(400, "'question' is required.")
|
| 165 |
+
try:
|
| 166 |
+
result = agent_module.run_agent(
|
| 167 |
+
question=question,
|
| 168 |
+
provider=payload.get("provider", "anthropic"),
|
| 169 |
+
model=payload.get("model") or None,
|
| 170 |
+
api_key=payload.get("api_key") or None,
|
| 171 |
+
base_url=payload.get("base_url") or None,
|
| 172 |
+
session_id=payload.get("session_id") or None,
|
| 173 |
+
inp_content=payload.get("inp_content") or None,
|
| 174 |
+
allow_report=bool(payload.get("allow_report", False)),
|
| 175 |
+
)
|
| 176 |
+
except ValueError as exc:
|
| 177 |
+
raise HTTPException(400, str(exc))
|
| 178 |
+
except Exception as exc:
|
| 179 |
+
raise HTTPException(502, f"Agent/provider error: {type(exc).__name__}: {exc}")
|
| 180 |
+
return JSONResponse(json.loads(json.dumps(result, default=str)))
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
@app.get("/files/{session_id}/{filename}")
|
| 184 |
+
def serve_file(session_id: str, filename: str) -> FileResponse:
|
| 185 |
+
"""Download generated report artifacts."""
|
| 186 |
+
safe_session = Path(session_id).name
|
| 187 |
+
safe_file = Path(filename).name
|
| 188 |
+
path = (SESSION_ROOT / safe_session / "outputs" / safe_file).resolve()
|
| 189 |
+
if not str(path).startswith(str(SESSION_ROOT.resolve())) or not path.exists():
|
| 190 |
+
raise HTTPException(404, "File not found (sessions expire; regenerate the report).")
|
| 191 |
+
return FileResponse(path, filename=safe_file)
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
@app.get("/health")
|
| 195 |
+
def health() -> dict:
|
| 196 |
+
return {"status": "ok", "server": SERVER_NAME, "version": SERVER_VERSION,
|
| 197 |
+
"tools": len(TOOL_REGISTRY) + 1, "sessions": len(STORE.list()),
|
| 198 |
+
"mcp_endpoint": "/mcp", "openapi": "/openapi.json"}
|
| 199 |
+
|
| 200 |
+
|
| 201 |
+
@app.get("/", response_class=HTMLResponse)
|
| 202 |
+
def index() -> str:
|
| 203 |
+
tool_rows = "".join(
|
| 204 |
+
f"<tr><td><code>{n}</code></td><td>{(f.__doc__ or '').strip().splitlines()[0]}</td></tr>"
|
| 205 |
+
for n, f in TOOL_REGISTRY.items())
|
| 206 |
+
return f"""<!doctype html><html><head><title>SWMM Analysis MCP Server</title>
|
| 207 |
+
<style>body{{font-family:system-ui;max-width:900px;margin:2rem auto;padding:0 1rem;color:#222}}
|
| 208 |
+
code{{background:#f2f2f2;padding:1px 5px;border-radius:4px}}table{{border-collapse:collapse;width:100%}}
|
| 209 |
+
td,th{{border:1px solid #ddd;padding:6px 10px;text-align:left;font-size:14px}}h1{{color:#0a4d8c}}</style></head>
|
| 210 |
+
<body><h1>🌧️ SWMM Analysis MCP Server</h1>
|
| 211 |
+
<p>Deterministic EPA-SWMM stormwater analysis (engine Rev 23.2) with dual MCP + REST surfaces
|
| 212 |
+
and a built-in multi-provider agent. All results are preliminary engineering screening.</p>
|
| 213 |
+
<ul>
|
| 214 |
+
<li><b>MCP endpoint (Streamable HTTP):</b> <code><this-space-url>/mcp</code></li>
|
| 215 |
+
<li><b>REST:</b> <code>POST /api/tool/{{name}}</code> — catalog at <a href="/api/tools">/api/tools</a>,
|
| 216 |
+
schema at <a href="/openapi.json">/openapi.json</a></li>
|
| 217 |
+
<li><b>Agent:</b> <code>POST /api/agent</code> (anthropic · openai · gemini · groq · mistral · local)</li>
|
| 218 |
+
<li><b>Health:</b> <a href="/health">/health</a></li>
|
| 219 |
+
</ul>
|
| 220 |
+
<h3>Tools ({len(TOOL_REGISTRY) + 1})</h3><table><tr><th>Tool</th><th>Purpose</th></tr>{tool_rows}
|
| 221 |
+
<tr><td><code>agent_analyze</code></td><td>Server-side agent loop over all tools (MCP + REST).</td></tr></table>
|
| 222 |
+
<p>Typical flow: <code>upload_model</code> → <code>run_simulation</code> → analysis tools with the returned
|
| 223 |
+
<code>session_id</code>. See the README for per-platform connection instructions.</p></body></html>"""
|
| 224 |
+
|
| 225 |
+
|
| 226 |
+
# Mount MCP last so explicit routes take priority.
|
| 227 |
+
app.mount("/", mcp_app)
|
| 228 |
+
|
| 229 |
+
if __name__ == "__main__":
|
| 230 |
+
import uvicorn
|
| 231 |
+
uvicorn.run(app, host="0.0.0.0", port=int(os.environ.get("PORT", "7860")))
|
sessions.py
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Session store for the SWMM MCP server.
|
| 2 |
+
|
| 3 |
+
Each uploaded model gets a session with its own working directory under
|
| 4 |
+
/tmp/swmm_sessions. Sessions expire after SESSION_TTL_HOURS (default 6) and
|
| 5 |
+
are swept lazily on access. State is process-local: the HF Space runs a
|
| 6 |
+
single uvicorn worker, matching the stateful-session conclusion from the
|
| 7 |
+
WNTR MCP architecture work (one backend, session-ID state).
|
| 8 |
+
"""
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import os
|
| 12 |
+
import shutil
|
| 13 |
+
import threading
|
| 14 |
+
import time
|
| 15 |
+
import uuid
|
| 16 |
+
from pathlib import Path
|
| 17 |
+
from typing import Any
|
| 18 |
+
|
| 19 |
+
SESSION_ROOT = Path(os.environ.get("SWMM_SESSION_ROOT", "/tmp/swmm_sessions"))
|
| 20 |
+
SESSION_TTL_S = float(os.environ.get("SESSION_TTL_HOURS", "6")) * 3600.0
|
| 21 |
+
MAX_SESSIONS = int(os.environ.get("MAX_SESSIONS", "40"))
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
class Session:
|
| 25 |
+
def __init__(self, session_id: str, workdir: Path):
|
| 26 |
+
self.id = session_id
|
| 27 |
+
self.workdir = workdir
|
| 28 |
+
self.created = time.time()
|
| 29 |
+
self.touched = time.time()
|
| 30 |
+
self.data: dict[str, Any] = {}
|
| 31 |
+
|
| 32 |
+
def touch(self) -> None:
|
| 33 |
+
self.touched = time.time()
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
class SessionStore:
|
| 37 |
+
def __init__(self) -> None:
|
| 38 |
+
self._sessions: dict[str, Session] = {}
|
| 39 |
+
self._lock = threading.Lock()
|
| 40 |
+
SESSION_ROOT.mkdir(parents=True, exist_ok=True)
|
| 41 |
+
|
| 42 |
+
def create(self) -> Session:
|
| 43 |
+
with self._lock:
|
| 44 |
+
self._sweep_locked()
|
| 45 |
+
if len(self._sessions) >= MAX_SESSIONS:
|
| 46 |
+
oldest = min(self._sessions.values(), key=lambda s: s.touched)
|
| 47 |
+
self._drop_locked(oldest.id)
|
| 48 |
+
sid = uuid.uuid4().hex[:12]
|
| 49 |
+
workdir = SESSION_ROOT / sid
|
| 50 |
+
workdir.mkdir(parents=True, exist_ok=True)
|
| 51 |
+
session = Session(sid, workdir)
|
| 52 |
+
self._sessions[sid] = session
|
| 53 |
+
return session
|
| 54 |
+
|
| 55 |
+
def get(self, session_id: str) -> Session:
|
| 56 |
+
with self._lock:
|
| 57 |
+
self._sweep_locked()
|
| 58 |
+
session = self._sessions.get(str(session_id))
|
| 59 |
+
if session is None:
|
| 60 |
+
raise KeyError(
|
| 61 |
+
f"Unknown or expired session '{session_id}'. Call upload_model first "
|
| 62 |
+
f"(sessions expire after {SESSION_TTL_S/3600:.0f} h of inactivity)."
|
| 63 |
+
)
|
| 64 |
+
session.touch()
|
| 65 |
+
return session
|
| 66 |
+
|
| 67 |
+
def list(self) -> list[dict[str, Any]]:
|
| 68 |
+
with self._lock:
|
| 69 |
+
self._sweep_locked()
|
| 70 |
+
return [
|
| 71 |
+
{
|
| 72 |
+
"session_id": s.id,
|
| 73 |
+
"model": s.data.get("filename"),
|
| 74 |
+
"simulated": bool(s.data.get("results")),
|
| 75 |
+
"age_minutes": round((time.time() - s.created) / 60.0, 1),
|
| 76 |
+
}
|
| 77 |
+
for s in self._sessions.values()
|
| 78 |
+
]
|
| 79 |
+
|
| 80 |
+
def drop(self, session_id: str) -> bool:
|
| 81 |
+
with self._lock:
|
| 82 |
+
return self._drop_locked(str(session_id))
|
| 83 |
+
|
| 84 |
+
# -- internal --
|
| 85 |
+
def _drop_locked(self, session_id: str) -> bool:
|
| 86 |
+
session = self._sessions.pop(session_id, None)
|
| 87 |
+
if session is None:
|
| 88 |
+
return False
|
| 89 |
+
shutil.rmtree(session.workdir, ignore_errors=True)
|
| 90 |
+
return True
|
| 91 |
+
|
| 92 |
+
def _sweep_locked(self) -> None:
|
| 93 |
+
now = time.time()
|
| 94 |
+
for sid in [s for s, v in self._sessions.items() if now - v.touched > SESSION_TTL_S]:
|
| 95 |
+
self._drop_locked(sid)
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
STORE = SessionStore()
|
smoke_test.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Deployment smoke test. Run against a live server:
|
| 2 |
+
python smoke_test.py http://127.0.0.1:7860 (or the Space URL)
|
| 3 |
+
Verifies: health, REST tool call, full MCP workflow, regression pin (>=26/27
|
| 4 |
+
links reconciled on Kincora), report download, agent 400-without-key.
|
| 5 |
+
Requires a local Kincora_Phase_2.inp or any valid .inp passed as argv[2].
|
| 6 |
+
"""
|
| 7 |
+
import asyncio, json, sys
|
| 8 |
+
import httpx
|
| 9 |
+
from mcp import ClientSession
|
| 10 |
+
from mcp.client.streamable_http import streamablehttp_client
|
| 11 |
+
|
| 12 |
+
BASE = sys.argv[1].rstrip("/") if len(sys.argv) > 1 else "http://127.0.0.1:7860"
|
| 13 |
+
INP = sys.argv[2] if len(sys.argv) > 2 else "Kincora_Phase_2.inp"
|
| 14 |
+
|
| 15 |
+
async def main():
|
| 16 |
+
h = httpx.get(f"{BASE}/health", timeout=30).json()
|
| 17 |
+
assert h["status"] == "ok", h
|
| 18 |
+
print("health:", h["server"], h["version"], "| tools", h["tools"])
|
| 19 |
+
r = httpx.post(f"{BASE}/api/tool/list_sessions", json={}, timeout=30)
|
| 20 |
+
assert r.status_code == 200
|
| 21 |
+
print("REST surface: OK")
|
| 22 |
+
inp = open(INP, encoding="utf-8", errors="replace").read()
|
| 23 |
+
async with streamablehttp_client(f"{BASE}/mcp") as (read, write, _):
|
| 24 |
+
async with ClientSession(read, write) as s:
|
| 25 |
+
await s.initialize()
|
| 26 |
+
tools = await s.list_tools()
|
| 27 |
+
assert len(tools.tools) >= 16
|
| 28 |
+
up = json.loads((await s.call_tool("upload_model",
|
| 29 |
+
{"inp_content": inp, "filename": INP})).content[0].text)
|
| 30 |
+
sid = up["session_id"]
|
| 31 |
+
run = json.loads((await s.call_tool("run_simulation", {"session_id": sid})).content[0].text)
|
| 32 |
+
recon = run["rpt_reconciliation"]
|
| 33 |
+
print(f"MCP workflow: session {sid} | recon {recon.get('ok')}/{recon.get('links_checked')}")
|
| 34 |
+
if "Kincora" in INP:
|
| 35 |
+
assert recon.get("ok", 0) >= 26, "REGRESSION PIN FAILED"
|
| 36 |
+
rep = json.loads((await s.call_tool("generate_report",
|
| 37 |
+
{"session_id": sid, "project_name": "Smoke Test"})).content[0].text)
|
| 38 |
+
dl = httpx.get(f"{BASE}{rep['files']['docx']}", timeout=60)
|
| 39 |
+
assert dl.status_code == 200 and len(dl.content) > 30000
|
| 40 |
+
print("report + download: OK", len(dl.content), "bytes")
|
| 41 |
+
await s.call_tool("close_session", {"session_id": sid})
|
| 42 |
+
r = httpx.post(f"{BASE}/api/agent", json={"question": "x", "provider": "anthropic"}, timeout=30)
|
| 43 |
+
print("agent (no key expected 400 unless secret set):", r.status_code)
|
| 44 |
+
print("\nSMOKE TEST: PASS")
|
| 45 |
+
|
| 46 |
+
asyncio.run(main())
|
sql_agent.py
ADDED
|
@@ -0,0 +1,390 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Safe provider-agnostic SQL retrieval agent for the complete SWMM SQLite store.
|
| 2 |
+
|
| 3 |
+
The language model produces a constrained JSON retrieval plan. This module validates
|
| 4 |
+
that plan and executes read-only SELECT statements only. No model-generated SQL is
|
| 5 |
+
executed directly.
|
| 6 |
+
"""
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import json
|
| 10 |
+
import math
|
| 11 |
+
import re
|
| 12 |
+
import sqlite3
|
| 13 |
+
from dataclasses import dataclass
|
| 14 |
+
from typing import Any
|
| 15 |
+
|
| 16 |
+
import pandas as pd
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
_ALLOWED_FILTER_OPS = {
|
| 20 |
+
"eq": "=", "ne": "!=", "gt": ">", "gte": ">=", "lt": "<", "lte": "<=",
|
| 21 |
+
"like": "LIKE", "in": "IN", "between": "BETWEEN",
|
| 22 |
+
"is_null": "IS NULL", "not_null": "IS NOT NULL",
|
| 23 |
+
}
|
| 24 |
+
_ALLOWED_AGGS = {"MAX", "MIN", "AVG", "SUM", "COUNT"}
|
| 25 |
+
_MAX_ACTIONS = 6
|
| 26 |
+
_MAX_ROWS_PER_ACTION = 120
|
| 27 |
+
_MAX_CONTEXT_CHARS = 45_000
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
@dataclass
|
| 31 |
+
class RetrievalResult:
|
| 32 |
+
context: str
|
| 33 |
+
audit: list[dict[str, Any]]
|
| 34 |
+
plan: dict[str, Any]
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
class SafeSQLAgent:
|
| 38 |
+
"""Validate and execute bounded, read-only retrieval plans against a ResultDatabase."""
|
| 39 |
+
|
| 40 |
+
def __init__(self, result_db: Any) -> None:
|
| 41 |
+
self.db = result_db
|
| 42 |
+
self.conn: sqlite3.Connection = result_db.connection
|
| 43 |
+
self._tables = self._read_tables()
|
| 44 |
+
self._columns = {table: self._read_columns(table) for table in self._tables}
|
| 45 |
+
|
| 46 |
+
@staticmethod
|
| 47 |
+
def _quote_identifier(name: str) -> str:
|
| 48 |
+
return '"' + name.replace('"', '""') + '"'
|
| 49 |
+
|
| 50 |
+
def _read_tables(self) -> list[str]:
|
| 51 |
+
rows = self.conn.execute(
|
| 52 |
+
"SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name"
|
| 53 |
+
).fetchall()
|
| 54 |
+
return [str(r[0]) for r in rows]
|
| 55 |
+
|
| 56 |
+
def _read_columns(self, table: str) -> list[str]:
|
| 57 |
+
if table not in self._tables:
|
| 58 |
+
return []
|
| 59 |
+
rows = self.conn.execute(f"PRAGMA table_info({self._quote_identifier(table)})").fetchall()
|
| 60 |
+
return [str(r[1]) for r in rows]
|
| 61 |
+
|
| 62 |
+
def _row_count(self, table: str) -> int:
|
| 63 |
+
try:
|
| 64 |
+
return int(self.conn.execute(
|
| 65 |
+
f"SELECT COUNT(*) FROM {self._quote_identifier(table)}"
|
| 66 |
+
).fetchone()[0])
|
| 67 |
+
except Exception:
|
| 68 |
+
return 0
|
| 69 |
+
|
| 70 |
+
@staticmethod
|
| 71 |
+
def _tokens(text: str) -> set[str]:
|
| 72 |
+
return {t.lower() for t in re.findall(r"[A-Za-z0-9_:.\-]+", text) if len(t) > 1}
|
| 73 |
+
|
| 74 |
+
def schema_context(self, question: str, max_tables: int = 16) -> str:
|
| 75 |
+
"""Return a compact, question-ranked schema catalogue for the planning call."""
|
| 76 |
+
q_tokens = self._tokens(question)
|
| 77 |
+
synonyms = {
|
| 78 |
+
"node": {"node", "junction", "outfall", "storage", "manhole", "flood", "head", "depth"},
|
| 79 |
+
"link": {"link", "conduit", "pipe", "pump", "weir", "orifice", "outlet", "flow", "velocity", "capacity"},
|
| 80 |
+
"subcatchment": {"subcatchment", "catchment", "runoff", "rainfall", "infiltration", "hydrology"},
|
| 81 |
+
"input": {"input", "option", "roughness", "diameter", "length", "invert", "curve", "pattern", "control"},
|
| 82 |
+
"timeseries": {"time", "timeseries", "duration", "when", "peak", "first", "last", "hour"},
|
| 83 |
+
}
|
| 84 |
+
expanded = set(q_tokens)
|
| 85 |
+
for _, words in synonyms.items():
|
| 86 |
+
if q_tokens & words:
|
| 87 |
+
expanded |= words
|
| 88 |
+
|
| 89 |
+
always = {
|
| 90 |
+
"simulation_metadata", "simulation_warnings", "node_summary", "link_summary",
|
| 91 |
+
"subcatchment_summary", "model_input_section_catalog",
|
| 92 |
+
}
|
| 93 |
+
scored: list[tuple[float, str]] = []
|
| 94 |
+
for table in self._tables:
|
| 95 |
+
hay = self._tokens(table + " " + " ".join(self._columns[table]))
|
| 96 |
+
score = float(len(expanded & hay) * 5)
|
| 97 |
+
for token in expanded:
|
| 98 |
+
if token in table.lower():
|
| 99 |
+
score += 3
|
| 100 |
+
score += sum(1 for c in self._columns[table] if token in c.lower()) * 0.5
|
| 101 |
+
if table in always:
|
| 102 |
+
score += 4
|
| 103 |
+
if table.endswith("_timeseries") and (expanded & synonyms["timeseries"]):
|
| 104 |
+
score += 5
|
| 105 |
+
if table.startswith("inp_") and (expanded & synonyms["input"]):
|
| 106 |
+
score += 5
|
| 107 |
+
scored.append((score, table))
|
| 108 |
+
|
| 109 |
+
selected = [t for _, t in sorted(scored, reverse=True)[:max_tables]]
|
| 110 |
+
for table in always:
|
| 111 |
+
if table in self._tables and table not in selected:
|
| 112 |
+
selected.append(table)
|
| 113 |
+
selected = selected[:max_tables]
|
| 114 |
+
|
| 115 |
+
lines = ["AVAILABLE SQLITE TABLES (question-ranked):"]
|
| 116 |
+
for table in selected:
|
| 117 |
+
cols = self._columns[table]
|
| 118 |
+
count = self._row_count(table)
|
| 119 |
+
lines.append(f"- {table} ({count} rows): {', '.join(cols)}")
|
| 120 |
+
lines.append(
|
| 121 |
+
"The complete database may contain additional inp_<section> tables. "
|
| 122 |
+
"Use model_input_lines to search exact source text when a section is not listed."
|
| 123 |
+
)
|
| 124 |
+
return "\n".join(lines)
|
| 125 |
+
|
| 126 |
+
def planner_system_prompt(self, schema_context: str) -> str:
|
| 127 |
+
return f"""You are a retrieval planner for an EPA SWMM SQLite database.
|
| 128 |
+
Return ONLY valid JSON. Do not answer the engineering question.
|
| 129 |
+
Create the smallest read-only retrieval plan that supplies enough evidence for a later engineering answer.
|
| 130 |
+
Never emit SQL. Use only the operations and fields below.
|
| 131 |
+
|
| 132 |
+
{schema_context}
|
| 133 |
+
|
| 134 |
+
JSON format:
|
| 135 |
+
{{
|
| 136 |
+
"reasoning_summary": "brief retrieval rationale",
|
| 137 |
+
"actions": [
|
| 138 |
+
{{
|
| 139 |
+
"operation": "select",
|
| 140 |
+
"table": "table_name",
|
| 141 |
+
"columns": ["column1", "column2"],
|
| 142 |
+
"filters": [{{"column":"column1","op":"eq|ne|gt|gte|lt|lte|like|in|between|is_null|not_null","value": "value or list"}}],
|
| 143 |
+
"order_by": [{{"column":"column1","direction":"asc|desc"}}],
|
| 144 |
+
"limit": 40,
|
| 145 |
+
"label": "descriptive result label"
|
| 146 |
+
}},
|
| 147 |
+
{{
|
| 148 |
+
"operation": "aggregate",
|
| 149 |
+
"table": "table_name",
|
| 150 |
+
"group_by": ["optional_column"],
|
| 151 |
+
"metrics": [{{"function":"MAX|MIN|AVG|SUM|COUNT","column":"column_or_*","alias":"metric_name"}}],
|
| 152 |
+
"filters": [],
|
| 153 |
+
"order_by": [{{"column":"metric_alias_or_group_column","direction":"desc"}}],
|
| 154 |
+
"limit": 40,
|
| 155 |
+
"label": "descriptive result label"
|
| 156 |
+
}},
|
| 157 |
+
{{
|
| 158 |
+
"operation": "search_input",
|
| 159 |
+
"term": "text to find in raw SWMM input lines",
|
| 160 |
+
"section": "optional SWMM section name without brackets",
|
| 161 |
+
"limit": 40,
|
| 162 |
+
"label": "descriptive result label"
|
| 163 |
+
}},
|
| 164 |
+
{{
|
| 165 |
+
"operation": "describe_table",
|
| 166 |
+
"table": "table_name",
|
| 167 |
+
"label": "table structure"
|
| 168 |
+
}}
|
| 169 |
+
]
|
| 170 |
+
}}
|
| 171 |
+
|
| 172 |
+
Rules:
|
| 173 |
+
- Maximum 6 actions and normally 1-4 actions.
|
| 174 |
+
- Use exact asset IDs appearing in the user's question as equality filters.
|
| 175 |
+
- Prefer summary/aggregate retrieval for broad questions and detailed time-series only for named assets or explicit temporal questions.
|
| 176 |
+
- For first/last/peak questions, use ordering and a small limit or an aggregate.
|
| 177 |
+
- Retrieve model inputs from inp_* tables or model_input_lines.
|
| 178 |
+
- Never request full tables. Keep each limit at or below 120.
|
| 179 |
+
- Use engineering evidence from both input and output tables when the question asks for diagnosis or recommendations.
|
| 180 |
+
"""
|
| 181 |
+
|
| 182 |
+
@staticmethod
|
| 183 |
+
def parse_plan(text: str) -> dict[str, Any]:
|
| 184 |
+
cleaned = text.strip()
|
| 185 |
+
cleaned = re.sub(r"^```(?:json)?\s*", "", cleaned, flags=re.IGNORECASE)
|
| 186 |
+
cleaned = re.sub(r"\s*```$", "", cleaned)
|
| 187 |
+
start, end = cleaned.find("{"), cleaned.rfind("}")
|
| 188 |
+
if start >= 0 and end > start:
|
| 189 |
+
cleaned = cleaned[start:end + 1]
|
| 190 |
+
plan = json.loads(cleaned)
|
| 191 |
+
if not isinstance(plan, dict):
|
| 192 |
+
raise ValueError("Planner response must be a JSON object")
|
| 193 |
+
actions = plan.get("actions", [])
|
| 194 |
+
if not isinstance(actions, list):
|
| 195 |
+
raise ValueError("Planner actions must be a list")
|
| 196 |
+
plan["actions"] = actions[:_MAX_ACTIONS]
|
| 197 |
+
return plan
|
| 198 |
+
|
| 199 |
+
def _validate_table(self, table: str) -> str:
|
| 200 |
+
if table not in self._tables:
|
| 201 |
+
raise ValueError(f"Unknown table: {table}")
|
| 202 |
+
return table
|
| 203 |
+
|
| 204 |
+
def _validate_column(self, table: str, column: str, *, allow_star: bool = False) -> str:
|
| 205 |
+
if allow_star and column == "*":
|
| 206 |
+
return column
|
| 207 |
+
if column not in self._columns[table]:
|
| 208 |
+
raise ValueError(f"Unknown column {column!r} in {table}")
|
| 209 |
+
return column
|
| 210 |
+
|
| 211 |
+
def _build_filters(self, table: str, filters: Any) -> tuple[str, list[Any]]:
|
| 212 |
+
if not filters:
|
| 213 |
+
return "", []
|
| 214 |
+
clauses: list[str] = []
|
| 215 |
+
params: list[Any] = []
|
| 216 |
+
for item in list(filters)[:12]:
|
| 217 |
+
if not isinstance(item, dict):
|
| 218 |
+
continue
|
| 219 |
+
col = self._validate_column(table, str(item.get("column", "")))
|
| 220 |
+
op_key = str(item.get("op", "eq")).lower()
|
| 221 |
+
if op_key not in _ALLOWED_FILTER_OPS:
|
| 222 |
+
raise ValueError(f"Unsupported filter operation: {op_key}")
|
| 223 |
+
sql_op = _ALLOWED_FILTER_OPS[op_key]
|
| 224 |
+
qcol = self._quote_identifier(col)
|
| 225 |
+
value = item.get("value")
|
| 226 |
+
if op_key in {"is_null", "not_null"}:
|
| 227 |
+
clauses.append(f"{qcol} {sql_op}")
|
| 228 |
+
elif op_key == "in":
|
| 229 |
+
values = value if isinstance(value, list) else [value]
|
| 230 |
+
values = values[:50]
|
| 231 |
+
if not values:
|
| 232 |
+
clauses.append("1=0")
|
| 233 |
+
else:
|
| 234 |
+
clauses.append(f"{qcol} IN ({','.join('?' for _ in values)})")
|
| 235 |
+
params.extend(values)
|
| 236 |
+
elif op_key == "between":
|
| 237 |
+
values = value if isinstance(value, list) else []
|
| 238 |
+
if len(values) != 2:
|
| 239 |
+
raise ValueError("between requires exactly two values")
|
| 240 |
+
clauses.append(f"{qcol} BETWEEN ? AND ?")
|
| 241 |
+
params.extend(values)
|
| 242 |
+
else:
|
| 243 |
+
clauses.append(f"{qcol} {sql_op} ?")
|
| 244 |
+
params.append(value)
|
| 245 |
+
return (" WHERE " + " AND ".join(clauses)) if clauses else "", params
|
| 246 |
+
|
| 247 |
+
def _build_order(self, table: str, order_by: Any, aliases: set[str] | None = None) -> str:
|
| 248 |
+
if not order_by:
|
| 249 |
+
return ""
|
| 250 |
+
aliases = aliases or set()
|
| 251 |
+
parts: list[str] = []
|
| 252 |
+
for item in list(order_by)[:4]:
|
| 253 |
+
if not isinstance(item, dict):
|
| 254 |
+
continue
|
| 255 |
+
col = str(item.get("column", ""))
|
| 256 |
+
if col not in aliases:
|
| 257 |
+
self._validate_column(table, col)
|
| 258 |
+
direction = "DESC" if str(item.get("direction", "asc")).lower() == "desc" else "ASC"
|
| 259 |
+
parts.append(f"{self._quote_identifier(col)} {direction}")
|
| 260 |
+
return " ORDER BY " + ", ".join(parts) if parts else ""
|
| 261 |
+
|
| 262 |
+
@staticmethod
|
| 263 |
+
def _clean_frame(df: pd.DataFrame) -> pd.DataFrame:
|
| 264 |
+
out = df.copy()
|
| 265 |
+
for col in out.columns:
|
| 266 |
+
out[col] = out[col].map(
|
| 267 |
+
lambda v: None if isinstance(v, float) and (math.isnan(v) or math.isinf(v)) else v
|
| 268 |
+
)
|
| 269 |
+
return out
|
| 270 |
+
|
| 271 |
+
def _execute_select(self, action: dict[str, Any]) -> tuple[pd.DataFrame, str]:
|
| 272 |
+
table = self._validate_table(str(action.get("table", "")))
|
| 273 |
+
requested = action.get("columns") or self._columns[table]
|
| 274 |
+
columns = [self._validate_column(table, str(c)) for c in requested]
|
| 275 |
+
columns = columns[:24]
|
| 276 |
+
where_sql, params = self._build_filters(table, action.get("filters"))
|
| 277 |
+
order_sql = self._build_order(table, action.get("order_by"))
|
| 278 |
+
limit = min(max(int(action.get("limit", 40)), 1), _MAX_ROWS_PER_ACTION)
|
| 279 |
+
sql = (
|
| 280 |
+
"SELECT " + ", ".join(self._quote_identifier(c) for c in columns) +
|
| 281 |
+
f" FROM {self._quote_identifier(table)}" + where_sql + order_sql + " LIMIT ?"
|
| 282 |
+
)
|
| 283 |
+
df = pd.read_sql_query(sql, self.conn, params=(*params, limit))
|
| 284 |
+
return self._clean_frame(df), sql
|
| 285 |
+
|
| 286 |
+
def _execute_aggregate(self, action: dict[str, Any]) -> tuple[pd.DataFrame, str]:
|
| 287 |
+
table = self._validate_table(str(action.get("table", "")))
|
| 288 |
+
group_by = [self._validate_column(table, str(c)) for c in (action.get("group_by") or [])][:6]
|
| 289 |
+
metric_exprs: list[str] = []
|
| 290 |
+
aliases: set[str] = set()
|
| 291 |
+
for metric in (action.get("metrics") or [])[:12]:
|
| 292 |
+
if not isinstance(metric, dict):
|
| 293 |
+
continue
|
| 294 |
+
fn = str(metric.get("function", "")).upper()
|
| 295 |
+
if fn not in _ALLOWED_AGGS:
|
| 296 |
+
raise ValueError(f"Unsupported aggregate: {fn}")
|
| 297 |
+
col = str(metric.get("column", "*"))
|
| 298 |
+
self._validate_column(table, col, allow_star=(fn == "COUNT"))
|
| 299 |
+
alias = re.sub(r"[^A-Za-z0-9_]+", "_", str(metric.get("alias") or f"{fn.lower()}_{col}"))
|
| 300 |
+
aliases.add(alias)
|
| 301 |
+
expr_col = "*" if col == "*" else self._quote_identifier(col)
|
| 302 |
+
metric_exprs.append(f"{fn}({expr_col}) AS {self._quote_identifier(alias)}")
|
| 303 |
+
if not metric_exprs:
|
| 304 |
+
metric_exprs = ['COUNT(*) AS "row_count"']
|
| 305 |
+
aliases.add("row_count")
|
| 306 |
+
select_parts = [self._quote_identifier(c) for c in group_by] + metric_exprs
|
| 307 |
+
where_sql, params = self._build_filters(table, action.get("filters"))
|
| 308 |
+
group_sql = " GROUP BY " + ", ".join(self._quote_identifier(c) for c in group_by) if group_by else ""
|
| 309 |
+
order_sql = self._build_order(table, action.get("order_by"), aliases=aliases)
|
| 310 |
+
limit = min(max(int(action.get("limit", 40)), 1), _MAX_ROWS_PER_ACTION)
|
| 311 |
+
sql = (
|
| 312 |
+
"SELECT " + ", ".join(select_parts) + f" FROM {self._quote_identifier(table)}" +
|
| 313 |
+
where_sql + group_sql + order_sql + " LIMIT ?"
|
| 314 |
+
)
|
| 315 |
+
df = pd.read_sql_query(sql, self.conn, params=(*params, limit))
|
| 316 |
+
return self._clean_frame(df), sql
|
| 317 |
+
|
| 318 |
+
def _execute_search_input(self, action: dict[str, Any]) -> tuple[pd.DataFrame, str]:
|
| 319 |
+
term = str(action.get("term", "")).strip()
|
| 320 |
+
if not term:
|
| 321 |
+
raise ValueError("search_input requires a term")
|
| 322 |
+
section = str(action.get("section", "")).strip().upper()
|
| 323 |
+
limit = min(max(int(action.get("limit", 40)), 1), _MAX_ROWS_PER_ACTION)
|
| 324 |
+
sql = (
|
| 325 |
+
"SELECT line_no, section_name, section_row_no, raw_text FROM model_input_lines "
|
| 326 |
+
"WHERE lower(raw_text) LIKE lower(?)"
|
| 327 |
+
)
|
| 328 |
+
params: list[Any] = [f"%{term}%"]
|
| 329 |
+
if section:
|
| 330 |
+
sql += " AND upper(section_name)=?"
|
| 331 |
+
params.append(section)
|
| 332 |
+
sql += " ORDER BY line_no LIMIT ?"
|
| 333 |
+
params.append(limit)
|
| 334 |
+
df = pd.read_sql_query(sql, self.conn, params=params)
|
| 335 |
+
return self._clean_frame(df), sql
|
| 336 |
+
|
| 337 |
+
def _execute_describe(self, action: dict[str, Any]) -> tuple[pd.DataFrame, str]:
|
| 338 |
+
table = self._validate_table(str(action.get("table", "")))
|
| 339 |
+
rows = self.conn.execute(f"PRAGMA table_info({self._quote_identifier(table)})").fetchall()
|
| 340 |
+
df = pd.DataFrame(rows, columns=["cid", "name", "type", "notnull", "default_value", "primary_key"])
|
| 341 |
+
df["row_count"] = self._row_count(table)
|
| 342 |
+
return df, f"PRAGMA table_info({table})"
|
| 343 |
+
|
| 344 |
+
def execute_plan(self, plan: dict[str, Any]) -> RetrievalResult:
|
| 345 |
+
contexts: list[str] = []
|
| 346 |
+
audit: list[dict[str, Any]] = []
|
| 347 |
+
used_chars = 0
|
| 348 |
+
for index, action in enumerate(plan.get("actions", [])[:_MAX_ACTIONS], start=1):
|
| 349 |
+
if not isinstance(action, dict):
|
| 350 |
+
continue
|
| 351 |
+
operation = str(action.get("operation", "select")).lower()
|
| 352 |
+
label = str(action.get("label") or f"Retrieval {index}")
|
| 353 |
+
try:
|
| 354 |
+
if operation == "select":
|
| 355 |
+
df, sql = self._execute_select(action)
|
| 356 |
+
elif operation == "aggregate":
|
| 357 |
+
df, sql = self._execute_aggregate(action)
|
| 358 |
+
elif operation == "search_input":
|
| 359 |
+
df, sql = self._execute_search_input(action)
|
| 360 |
+
elif operation == "describe_table":
|
| 361 |
+
df, sql = self._execute_describe(action)
|
| 362 |
+
else:
|
| 363 |
+
raise ValueError(f"Unsupported operation: {operation}")
|
| 364 |
+
csv_text = df.to_csv(index=False)
|
| 365 |
+
block = f"=== {label} ===\n{csv_text}"
|
| 366 |
+
remaining = _MAX_CONTEXT_CHARS - used_chars
|
| 367 |
+
if remaining <= 0:
|
| 368 |
+
break
|
| 369 |
+
if len(block) > remaining:
|
| 370 |
+
block = block[:remaining] + "\n[retrieval context truncated]"
|
| 371 |
+
contexts.append(block)
|
| 372 |
+
used_chars += len(block)
|
| 373 |
+
audit.append({
|
| 374 |
+
"action": index,
|
| 375 |
+
"label": label,
|
| 376 |
+
"operation": operation,
|
| 377 |
+
"table": action.get("table", "model_input_lines" if operation == "search_input" else ""),
|
| 378 |
+
"rows_returned": int(len(df)),
|
| 379 |
+
"status": "ok",
|
| 380 |
+
})
|
| 381 |
+
except Exception as exc:
|
| 382 |
+
audit.append({
|
| 383 |
+
"action": index,
|
| 384 |
+
"label": label,
|
| 385 |
+
"operation": operation,
|
| 386 |
+
"table": action.get("table", ""),
|
| 387 |
+
"rows_returned": 0,
|
| 388 |
+
"status": f"error: {exc}",
|
| 389 |
+
})
|
| 390 |
+
return RetrievalResult(context="\n\n".join(contexts), audit=audit, plan=plan)
|
swmm_core.py
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Crash-isolated deterministic OpenSWMM simulation core.
|
| 2 |
+
|
| 3 |
+
The OpenSWMM engine is a native extension. It is never loaded into the
|
| 4 |
+
long-lived Streamlit process. Each simulation runs in a short-lived worker
|
| 5 |
+
process and returns only serialisable Python data.
|
| 6 |
+
"""
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import os
|
| 10 |
+
import pickle
|
| 11 |
+
import subprocess
|
| 12 |
+
import sys
|
| 13 |
+
import tempfile
|
| 14 |
+
from pathlib import Path
|
| 15 |
+
from typing import Any
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
class SwmmWorkerError(RuntimeError):
|
| 19 |
+
"""Raised when the isolated OpenSWMM worker cannot complete."""
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def run_swmm(
|
| 24 |
+
inp_path: str | Path,
|
| 25 |
+
rpt_path: str | Path | None = None,
|
| 26 |
+
out_path: str | Path | None = None,
|
| 27 |
+
timeout_s: int = 900,
|
| 28 |
+
) -> dict[str, Any]:
|
| 29 |
+
"""Run OpenSWMM in an isolated subprocess.
|
| 30 |
+
|
| 31 |
+
Isolation prevents a native OpenSWMM segmentation fault from terminating
|
| 32 |
+
Streamlit. The worker deliberately uses ``os._exit`` after serialising its
|
| 33 |
+
result so native-library finalisers cannot crash the parent application.
|
| 34 |
+
"""
|
| 35 |
+
inp = Path(inp_path).resolve()
|
| 36 |
+
if not inp.is_file():
|
| 37 |
+
raise FileNotFoundError(f"SWMM input file not found: {inp}")
|
| 38 |
+
|
| 39 |
+
rpt = Path(rpt_path).resolve() if rpt_path else inp.with_suffix(".rpt")
|
| 40 |
+
out = Path(out_path).resolve() if out_path else inp.with_suffix(".out")
|
| 41 |
+
worker = Path(__file__).with_name("swmm_worker.py")
|
| 42 |
+
if not worker.is_file():
|
| 43 |
+
raise FileNotFoundError(f"OpenSWMM worker not found: {worker}")
|
| 44 |
+
|
| 45 |
+
fd, result_name = tempfile.mkstemp(prefix="swmm_result_", suffix=".pkl")
|
| 46 |
+
os.close(fd)
|
| 47 |
+
result_file = Path(result_name)
|
| 48 |
+
|
| 49 |
+
worker_python = os.environ.get("SWMM_WORKER_PYTHON", "/opt/swmm-venv/bin/python")
|
| 50 |
+
if not Path(worker_python).is_file():
|
| 51 |
+
raise FileNotFoundError(
|
| 52 |
+
f"Isolated OpenSWMM interpreter not found: {worker_python}"
|
| 53 |
+
)
|
| 54 |
+
|
| 55 |
+
cmd = [
|
| 56 |
+
worker_python,
|
| 57 |
+
"-u",
|
| 58 |
+
str(worker),
|
| 59 |
+
"--inp", str(inp),
|
| 60 |
+
"--rpt", str(rpt),
|
| 61 |
+
"--out", str(out),
|
| 62 |
+
"--result", str(result_file),
|
| 63 |
+
]
|
| 64 |
+
|
| 65 |
+
try:
|
| 66 |
+
completed = subprocess.run(
|
| 67 |
+
cmd,
|
| 68 |
+
stdout=subprocess.PIPE,
|
| 69 |
+
stderr=subprocess.PIPE,
|
| 70 |
+
text=True,
|
| 71 |
+
timeout=timeout_s,
|
| 72 |
+
check=False,
|
| 73 |
+
env={**os.environ, "PYTHONUNBUFFERED": "1"},
|
| 74 |
+
)
|
| 75 |
+
|
| 76 |
+
payload = None
|
| 77 |
+
if result_file.exists() and result_file.stat().st_size:
|
| 78 |
+
with result_file.open("rb") as f:
|
| 79 |
+
payload = pickle.load(f)
|
| 80 |
+
|
| 81 |
+
if isinstance(payload, dict) and payload.get("ok"):
|
| 82 |
+
results = payload["results"]
|
| 83 |
+
results.setdefault("metadata", {})["worker_stdout"] = completed.stdout[-4000:]
|
| 84 |
+
results["metadata"]["worker_stderr"] = completed.stderr[-4000:]
|
| 85 |
+
results["metadata"]["worker_exit_code"] = completed.returncode
|
| 86 |
+
return results
|
| 87 |
+
|
| 88 |
+
detail = ""
|
| 89 |
+
if isinstance(payload, dict):
|
| 90 |
+
detail = payload.get("error", "")
|
| 91 |
+
if not detail:
|
| 92 |
+
detail = completed.stderr.strip() or completed.stdout.strip()
|
| 93 |
+
if completed.returncode in (-11, 139):
|
| 94 |
+
detail = (
|
| 95 |
+
"The OpenSWMM worker encountered a native segmentation fault. "
|
| 96 |
+
"The Streamlit process remained protected. " + detail
|
| 97 |
+
).strip()
|
| 98 |
+
raise SwmmWorkerError(
|
| 99 |
+
f"OpenSWMM worker failed with exit code {completed.returncode}. {detail}".strip()
|
| 100 |
+
)
|
| 101 |
+
except subprocess.TimeoutExpired as exc:
|
| 102 |
+
raise SwmmWorkerError(
|
| 103 |
+
f"OpenSWMM simulation exceeded the {timeout_s}-second timeout."
|
| 104 |
+
) from exc
|
| 105 |
+
finally:
|
| 106 |
+
result_file.unlink(missing_ok=True)
|
swmm_worker.py
ADDED
|
@@ -0,0 +1,207 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""One-shot OpenSWMM worker. Do not import this module from Streamlit."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import argparse
|
| 5 |
+
import os
|
| 6 |
+
import pickle
|
| 7 |
+
import traceback
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
from typing import Any
|
| 10 |
+
|
| 11 |
+
import numpy as np
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def _as_float_list(values: Any) -> list[float]:
|
| 15 |
+
return np.asarray(values, dtype=float).copy().tolist()
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def _enum_name(value: Any) -> str:
|
| 19 |
+
return getattr(value, "name", str(value))
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def _safe_float(getter, default: float = 0.0) -> float:
|
| 23 |
+
try:
|
| 24 |
+
return float(getter())
|
| 25 |
+
except Exception:
|
| 26 |
+
return default
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def simulate(inp_path: str, rpt_path: str, out_path: str) -> dict[str, Any]:
|
| 30 |
+
from openswmm.engine import Solver
|
| 31 |
+
|
| 32 |
+
node_ts: dict[str, dict[str, Any]] = {}
|
| 33 |
+
link_ts: dict[str, dict[str, Any]] = {}
|
| 34 |
+
sub_ts: dict[str, dict[str, Any]] = {}
|
| 35 |
+
times: list[Any] = []
|
| 36 |
+
warnings: list[dict[str, Any]] = []
|
| 37 |
+
|
| 38 |
+
with Solver(inp_path, rpt_path, out_path) as solver:
|
| 39 |
+
solver.set_warning_callback(
|
| 40 |
+
lambda code, message: warnings.append(
|
| 41 |
+
{"code": int(code), "message": str(message)}
|
| 42 |
+
)
|
| 43 |
+
)
|
| 44 |
+
|
| 45 |
+
nodes = solver.nodes
|
| 46 |
+
links = solver.links
|
| 47 |
+
subs = solver.subcatchments
|
| 48 |
+
|
| 49 |
+
node_ids = [str(nodes.get_id(i)) for i in range(len(nodes))]
|
| 50 |
+
link_ids = [str(links.get_id(i)) for i in range(len(links))]
|
| 51 |
+
sub_ids = [str(subs.get_id(i)) for i in range(len(subs))]
|
| 52 |
+
|
| 53 |
+
for node in nodes:
|
| 54 |
+
node_ts[str(node.id)] = {
|
| 55 |
+
"depth": [], "flooding": [], "inflow": [], "head": [],
|
| 56 |
+
"outflow": [], "volume": [],
|
| 57 |
+
"invert_elevation": float(node.invert_elev),
|
| 58 |
+
"full_depth": float(node.max_depth),
|
| 59 |
+
}
|
| 60 |
+
|
| 61 |
+
for link in links:
|
| 62 |
+
geom1 = 0.0
|
| 63 |
+
try:
|
| 64 |
+
geom1 = float(link.xsect.geom1)
|
| 65 |
+
except Exception:
|
| 66 |
+
try:
|
| 67 |
+
geom1 = float(link.xsect.geometry[0])
|
| 68 |
+
except Exception:
|
| 69 |
+
pass
|
| 70 |
+
link_ts[str(link.id)] = {
|
| 71 |
+
"flow": [], "depth": [], "velocity": [], "volume": [],
|
| 72 |
+
"capacity": [], "length": float(link.length),
|
| 73 |
+
"roughness": float(link.roughness), "diameter": geom1,
|
| 74 |
+
}
|
| 75 |
+
|
| 76 |
+
for sub in subs:
|
| 77 |
+
sub_ts[str(sub.id)] = {"runoff": [], "rainfall": [], "infil": []}
|
| 78 |
+
|
| 79 |
+
for _elapsed in solver.steps():
|
| 80 |
+
times.append(solver.current_datetime)
|
| 81 |
+
|
| 82 |
+
node_values = {
|
| 83 |
+
"depth": _as_float_list(nodes.depths),
|
| 84 |
+
"flooding": _as_float_list(nodes.overflows),
|
| 85 |
+
"inflow": _as_float_list(nodes.inflows),
|
| 86 |
+
"head": _as_float_list(nodes.heads),
|
| 87 |
+
"outflow": _as_float_list(nodes.outflows),
|
| 88 |
+
"volume": _as_float_list(nodes.volumes),
|
| 89 |
+
}
|
| 90 |
+
for i, node_id in enumerate(node_ids):
|
| 91 |
+
for key, values in node_values.items():
|
| 92 |
+
node_ts[node_id][key].append(values[i])
|
| 93 |
+
|
| 94 |
+
link_values = {
|
| 95 |
+
"flow": _as_float_list(links.flows),
|
| 96 |
+
"depth": _as_float_list(links.depths),
|
| 97 |
+
"velocity": _as_float_list(links.velocities),
|
| 98 |
+
"volume": _as_float_list(links.volumes),
|
| 99 |
+
"capacity": _as_float_list(links.capacities),
|
| 100 |
+
}
|
| 101 |
+
for i, link_id in enumerate(link_ids):
|
| 102 |
+
for key, values in link_values.items():
|
| 103 |
+
link_ts[link_id][key].append(values[i])
|
| 104 |
+
|
| 105 |
+
if sub_ids:
|
| 106 |
+
sub_values = {
|
| 107 |
+
"runoff": _as_float_list(subs.runoffs),
|
| 108 |
+
"rainfall": _as_float_list(subs.rainfalls),
|
| 109 |
+
"infil": _as_float_list(subs.infils),
|
| 110 |
+
}
|
| 111 |
+
for i, sub_id in enumerate(sub_ids):
|
| 112 |
+
for key, values in sub_values.items():
|
| 113 |
+
sub_ts[sub_id][key].append(values[i])
|
| 114 |
+
|
| 115 |
+
# Recompute link velocity as |flow| / (volume / length).
|
| 116 |
+
# Rationale: the bulk `links.velocities` API array was found to
|
| 117 |
+
# disagree with the engine's own .rpt Link Flow Summary (5-12% on
|
| 118 |
+
# circular pipes; understated up to ~6x on IRREGULAR transect
|
| 119 |
+
# channels), while |Q|*L/volume reproduces the .rpt values to ~1%
|
| 120 |
+
# for both pipes and channels (validated against a SWMM 5.0.022
|
| 121 |
+
# reference run of the same model). The raw API series is kept as
|
| 122 |
+
# "velocity_api" for auditability. Zero-length links (OUTLET/DUMMY)
|
| 123 |
+
# carry no meaningful velocity and are reported as zero.
|
| 124 |
+
for link_id in link_ids:
|
| 125 |
+
ts = link_ts[link_id]
|
| 126 |
+
length = float(ts.get("length", 0.0) or 0.0)
|
| 127 |
+
flows = ts["flow"]
|
| 128 |
+
volumes = ts["volume"]
|
| 129 |
+
ts["velocity_api"] = ts["velocity"]
|
| 130 |
+
if length > 0.0:
|
| 131 |
+
derived = []
|
| 132 |
+
for q, vol in zip(flows, volumes):
|
| 133 |
+
if vol > 1e-9:
|
| 134 |
+
v = abs(q) * length / vol
|
| 135 |
+
derived.append(v if np.isfinite(v) else 0.0)
|
| 136 |
+
else:
|
| 137 |
+
derived.append(0.0)
|
| 138 |
+
ts["velocity"] = derived
|
| 139 |
+
else:
|
| 140 |
+
ts["velocity"] = [0.0] * len(flows)
|
| 141 |
+
|
| 142 |
+
mb = solver.mass_balance
|
| 143 |
+
diag = mb.routing_diagnostics
|
| 144 |
+
metadata = {
|
| 145 |
+
"flow_units": _enum_name(solver.flow_units),
|
| 146 |
+
"system_units": str(solver.unit_system),
|
| 147 |
+
# OpenSWMM returns continuity errors as fractions; the .rpt and
|
| 148 |
+
# every downstream consumer (UI banner, report thresholds,
|
| 149 |
+
# calgary_rules continuity_*_pct) express them in PERCENT.
|
| 150 |
+
# Convert at the source so a -1.93% error reads as -1.93, not
|
| 151 |
+
# -0.0193 (which silently defeated the 0.5/1.0% thresholds).
|
| 152 |
+
"runoff_error": float(mb.runoff_continuity_error) * 100.0,
|
| 153 |
+
"flow_error": float(mb.routing_continuity_error) * 100.0,
|
| 154 |
+
"quality_error": _safe_float(lambda: mb.quality_continuity_error) * 100.0,
|
| 155 |
+
"start_time": solver.start_datetime,
|
| 156 |
+
"end_time": solver.end_datetime,
|
| 157 |
+
"routing_steps": int(diag.n_steps),
|
| 158 |
+
"not_converged_steps": int(diag.n_steps_not_converged),
|
| 159 |
+
"pct_not_converged": float(diag.pct_not_converged),
|
| 160 |
+
"avg_routing_step_s": float(diag.avg_time_step),
|
| 161 |
+
"warnings": warnings,
|
| 162 |
+
"report_path": rpt_path,
|
| 163 |
+
"output_path": out_path,
|
| 164 |
+
}
|
| 165 |
+
|
| 166 |
+
return {
|
| 167 |
+
"node_ts": node_ts,
|
| 168 |
+
"link_ts": link_ts,
|
| 169 |
+
"sub_ts": sub_ts,
|
| 170 |
+
"times": times,
|
| 171 |
+
"metadata": metadata,
|
| 172 |
+
}
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
def write_payload(path: Path, payload: dict[str, Any]) -> None:
|
| 176 |
+
temp = path.with_suffix(path.suffix + ".tmp")
|
| 177 |
+
with temp.open("wb") as f:
|
| 178 |
+
pickle.dump(payload, f, protocol=pickle.HIGHEST_PROTOCOL)
|
| 179 |
+
f.flush()
|
| 180 |
+
os.fsync(f.fileno())
|
| 181 |
+
os.replace(temp, path)
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
def main() -> int:
|
| 185 |
+
parser = argparse.ArgumentParser()
|
| 186 |
+
parser.add_argument("--inp", required=True)
|
| 187 |
+
parser.add_argument("--rpt", required=True)
|
| 188 |
+
parser.add_argument("--out", required=True)
|
| 189 |
+
parser.add_argument("--result", required=True)
|
| 190 |
+
args = parser.parse_args()
|
| 191 |
+
result_path = Path(args.result)
|
| 192 |
+
|
| 193 |
+
try:
|
| 194 |
+
results = simulate(args.inp, args.rpt, args.out)
|
| 195 |
+
write_payload(result_path, {"ok": True, "results": results})
|
| 196 |
+
# Bypass Python/native-extension finalisers. This is intentional.
|
| 197 |
+
os._exit(0)
|
| 198 |
+
except BaseException as exc:
|
| 199 |
+
write_payload(result_path, {
|
| 200 |
+
"ok": False,
|
| 201 |
+
"error": f"{type(exc).__name__}: {exc}\n{traceback.format_exc()}",
|
| 202 |
+
})
|
| 203 |
+
os._exit(1)
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
if __name__ == "__main__":
|
| 207 |
+
main()
|
tools.py
ADDED
|
@@ -0,0 +1,404 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""SWMM analysis tool registry.
|
| 2 |
+
|
| 3 |
+
Every capability is a plain typed function registered in TOOL_REGISTRY.
|
| 4 |
+
The MCP surface, the REST surface, and the internal agent all dispatch to
|
| 5 |
+
these same functions, so behaviour is identical regardless of platform.
|
| 6 |
+
|
| 7 |
+
Epistemics: all screening results are deterministic and distinguish
|
| 8 |
+
"screening" from "criterion"; nothing here is a professional engineering
|
| 9 |
+
determination. Outputs are bounded (row/point limits) so they remain usable
|
| 10 |
+
as LLM tool results.
|
| 11 |
+
"""
|
| 12 |
+
from __future__ import annotations
|
| 13 |
+
|
| 14 |
+
import base64
|
| 15 |
+
import binascii
|
| 16 |
+
import json
|
| 17 |
+
import math
|
| 18 |
+
from pathlib import Path
|
| 19 |
+
from typing import Any, Callable
|
| 20 |
+
|
| 21 |
+
import pandas as pd
|
| 22 |
+
|
| 23 |
+
import model_pipeline as mp
|
| 24 |
+
import rpt_reconciliation as rr
|
| 25 |
+
from calgary_rules import (
|
| 26 |
+
CalgaryCriteria,
|
| 27 |
+
apply_storage_classification,
|
| 28 |
+
criteria_register,
|
| 29 |
+
infer_design_event,
|
| 30 |
+
)
|
| 31 |
+
from preliminary_design_assistant import build_deterministic_findings, findings_dataframe
|
| 32 |
+
from results_db import ResultDatabase
|
| 33 |
+
from sessions import STORE
|
| 34 |
+
from sql_agent import SafeSQLAgent
|
| 35 |
+
from swmm_core import run_swmm
|
| 36 |
+
|
| 37 |
+
MAX_ROWS = 60
|
| 38 |
+
MAX_TS_POINTS = 200
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def _df_records(df: pd.DataFrame | None, limit: int = MAX_ROWS) -> dict[str, Any]:
|
| 42 |
+
if df is None or df.empty:
|
| 43 |
+
return {"rows": [], "row_count": 0, "truncated": False}
|
| 44 |
+
clean = df.replace({float("nan"): None})
|
| 45 |
+
return {
|
| 46 |
+
"rows": json.loads(clean.head(limit).to_json(orient="records")),
|
| 47 |
+
"row_count": int(len(df)),
|
| 48 |
+
"truncated": bool(len(df) > limit),
|
| 49 |
+
}
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def _require_results(session) -> None:
|
| 53 |
+
if not session.data.get("results"):
|
| 54 |
+
raise ValueError(f"Session '{session.id}' has no simulation results yet. Call run_simulation first.")
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
# ---------------------------------------------------------------------------
|
| 58 |
+
# Model lifecycle
|
| 59 |
+
# ---------------------------------------------------------------------------
|
| 60 |
+
|
| 61 |
+
def upload_model(inp_content: str, filename: str = "model.inp") -> dict:
|
| 62 |
+
"""Upload an EPA SWMM .inp model (raw text or base64) and create a session.
|
| 63 |
+
|
| 64 |
+
Returns a session_id used by every other tool, plus element counts.
|
| 65 |
+
"""
|
| 66 |
+
text = inp_content
|
| 67 |
+
if "[" not in inp_content[:2000]: # likely base64
|
| 68 |
+
try:
|
| 69 |
+
text = base64.b64decode(inp_content, validate=True).decode("utf-8", errors="replace")
|
| 70 |
+
except (binascii.Error, ValueError):
|
| 71 |
+
pass
|
| 72 |
+
if "[OPTIONS]" not in text.upper() and "[JUNCTIONS]" not in text.upper():
|
| 73 |
+
raise ValueError("Content does not look like a SWMM .inp file (no [OPTIONS]/[JUNCTIONS] section).")
|
| 74 |
+
session = STORE.create()
|
| 75 |
+
safe_name = Path(filename).name or "model.inp"
|
| 76 |
+
if not safe_name.lower().endswith(".inp"):
|
| 77 |
+
safe_name += ".inp"
|
| 78 |
+
inp_path = session.workdir / safe_name
|
| 79 |
+
inp_path.write_text(text, encoding="utf-8")
|
| 80 |
+
sections = mp.parse_inp_sections(str(inp_path))
|
| 81 |
+
session.data.update({"filename": safe_name, "inp_path": str(inp_path), "sections": sections})
|
| 82 |
+
counts = {name: len(rows) for name, rows in sections.items()
|
| 83 |
+
if name in ("JUNCTIONS", "OUTFALLS", "STORAGE", "CONDUITS", "PUMPS", "WEIRS",
|
| 84 |
+
"ORIFICES", "OUTLETS", "SUBCATCHMENTS", "RAINGAGES", "TIMESERIES")}
|
| 85 |
+
gages = [row[0] for row in sections.get("RAINGAGES", []) if row]
|
| 86 |
+
return {
|
| 87 |
+
"session_id": session.id,
|
| 88 |
+
"filename": safe_name,
|
| 89 |
+
"element_counts": counts,
|
| 90 |
+
"rain_gages": gages,
|
| 91 |
+
"design_event_inference": infer_design_event(gages) if gages else None,
|
| 92 |
+
"next_step": "Call run_simulation with this session_id.",
|
| 93 |
+
}
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def run_simulation(session_id: str) -> dict:
|
| 97 |
+
"""Run the model in the crash-isolated OpenSWMM worker and build summaries.
|
| 98 |
+
|
| 99 |
+
Also runs the deterministic worker-vs-.rpt reconciliation cross-check.
|
| 100 |
+
"""
|
| 101 |
+
session = STORE.get(session_id)
|
| 102 |
+
inp_path = session.data.get("inp_path")
|
| 103 |
+
if not inp_path:
|
| 104 |
+
raise ValueError("Session has no uploaded model.")
|
| 105 |
+
results = run_swmm(inp_path)
|
| 106 |
+
md = results["metadata"]
|
| 107 |
+
sections = session.data["sections"]
|
| 108 |
+
node_df = mp.build_node_summary(results["node_ts"], mp.parse_node_types(sections), 0.001, 0.9)
|
| 109 |
+
link_df = mp.build_link_summary(results["link_ts"], mp.parse_link_topology(sections),
|
| 110 |
+
mp.parse_conduit_geometry(sections), 0.9, 3.0)
|
| 111 |
+
sub_df = mp.build_sub_summary(results["sub_ts"], mp.parse_subcatchment_attrs(sections),
|
| 112 |
+
results.get("times"), md.get("flow_units", "CMS"))
|
| 113 |
+
db = ResultDatabase(str(session.workdir / "results.sqlite"))
|
| 114 |
+
db.load(node_df, link_df, sub_df, inp_path=inp_path, results=results)
|
| 115 |
+
|
| 116 |
+
recon = {"verdict": "Not performed"}
|
| 117 |
+
recon_links = recon_nodes = recon_cont = None
|
| 118 |
+
rpt_path = md.get("report_path")
|
| 119 |
+
if rpt_path and Path(str(rpt_path)).exists():
|
| 120 |
+
recon_links = rr.reconcile_links(link_df, rpt_path)
|
| 121 |
+
recon_nodes = rr.reconcile_nodes(node_df, rpt_path)
|
| 122 |
+
recon_cont = rr.reconcile_continuity(md, rpt_path)
|
| 123 |
+
recon = rr.reconciliation_summary(recon_links)
|
| 124 |
+
|
| 125 |
+
session.data.update({
|
| 126 |
+
"results": results, "node_df": node_df, "link_df": link_df, "sub_df": sub_df,
|
| 127 |
+
"db": db, "recon_links": recon_links, "recon_nodes": recon_nodes,
|
| 128 |
+
"recon_continuity": recon_cont, "recon_summary": recon,
|
| 129 |
+
})
|
| 130 |
+
flooded_col = next((c for c in node_df.columns if c.startswith("Peak Flooding (")), None)
|
| 131 |
+
flooded = int((pd.to_numeric(node_df[flooded_col], errors="coerce").fillna(0) > 0.001).sum()) if flooded_col else 0
|
| 132 |
+
return {
|
| 133 |
+
"session_id": session.id,
|
| 134 |
+
"simulation": "completed",
|
| 135 |
+
"flow_units": md.get("flow_units"),
|
| 136 |
+
"runoff_continuity_error_pct": round(float(md.get("runoff_error", 0.0)), 3),
|
| 137 |
+
"flow_continuity_error_pct": round(float(md.get("flow_error", 0.0)), 3),
|
| 138 |
+
"warnings": (results.get("warnings") or md.get("warnings") or [])[:10],
|
| 139 |
+
"flooded_nodes": flooded,
|
| 140 |
+
"rpt_reconciliation": recon,
|
| 141 |
+
"note": "Values are model results, not engineering determinations.",
|
| 142 |
+
}
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def list_sessions() -> dict:
|
| 146 |
+
"""List active sessions (id, model filename, simulated flag, age)."""
|
| 147 |
+
return {"sessions": STORE.list()}
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
def close_session(session_id: str) -> dict:
|
| 151 |
+
"""Delete a session and its working files."""
|
| 152 |
+
return {"session_id": session_id, "deleted": STORE.drop(session_id)}
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
# ---------------------------------------------------------------------------
|
| 156 |
+
# Results
|
| 157 |
+
# ---------------------------------------------------------------------------
|
| 158 |
+
|
| 159 |
+
def get_node_results(session_id: str, node_type: str = "", sort_by: str = "Depth Ratio",
|
| 160 |
+
limit: int = 20) -> dict:
|
| 161 |
+
"""Node result summary. Optional node_type filter (junction/storage/outfall);
|
| 162 |
+
sorted descending by sort_by column (default Depth Ratio)."""
|
| 163 |
+
session = STORE.get(session_id)
|
| 164 |
+
_require_results(session)
|
| 165 |
+
df = session.data["node_df"]
|
| 166 |
+
if node_type:
|
| 167 |
+
df = df[df["Type"].astype(str).str.lower() == node_type.lower()]
|
| 168 |
+
if sort_by in df.columns:
|
| 169 |
+
df = df.sort_values(sort_by, ascending=False)
|
| 170 |
+
return _df_records(df, min(int(limit), MAX_ROWS))
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
def get_link_results(session_id: str, sort_by: str = "Peak Velocity (m/s)", limit: int = 20) -> dict:
|
| 174 |
+
"""Link result summary sorted descending by sort_by (default peak velocity)."""
|
| 175 |
+
session = STORE.get(session_id)
|
| 176 |
+
_require_results(session)
|
| 177 |
+
df = session.data["link_df"]
|
| 178 |
+
if sort_by in df.columns:
|
| 179 |
+
df = df.sort_values(sort_by, ascending=False)
|
| 180 |
+
return _df_records(df, min(int(limit), MAX_ROWS))
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
def get_subcatchment_results(session_id: str, limit: int = 30) -> dict:
|
| 184 |
+
"""Subcatchment runoff summary."""
|
| 185 |
+
session = STORE.get(session_id)
|
| 186 |
+
_require_results(session)
|
| 187 |
+
return _df_records(session.data["sub_df"], min(int(limit), MAX_ROWS))
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
def get_timeseries(session_id: str, object_type: str, object_id: str, variable: str) -> dict:
|
| 191 |
+
"""Bounded time series for one object.
|
| 192 |
+
|
| 193 |
+
object_type: node|link|subcatchment. Variables — node: depth, flooding,
|
| 194 |
+
inflow, head, outflow, volume; link: flow, depth, velocity, volume,
|
| 195 |
+
capacity; subcatchment: runoff, rainfall, infil. Series longer than 200
|
| 196 |
+
points are decimated evenly (peaks preserved via max-in-bucket).
|
| 197 |
+
"""
|
| 198 |
+
session = STORE.get(session_id)
|
| 199 |
+
_require_results(session)
|
| 200 |
+
results = session.data["results"]
|
| 201 |
+
key = {"node": "node_ts", "link": "link_ts", "subcatchment": "sub_ts"}.get(object_type.lower())
|
| 202 |
+
if key is None:
|
| 203 |
+
raise ValueError("object_type must be node, link, or subcatchment")
|
| 204 |
+
store = results[key]
|
| 205 |
+
if object_id not in store:
|
| 206 |
+
raise ValueError(f"Unknown {object_type} '{object_id}'. Known: {sorted(store)[:25]}")
|
| 207 |
+
series = store[object_id].get(variable)
|
| 208 |
+
if not isinstance(series, list):
|
| 209 |
+
available = [k for k, v in store[object_id].items() if isinstance(v, list)]
|
| 210 |
+
raise ValueError(f"Unknown variable '{variable}'. Available: {available}")
|
| 211 |
+
times = results.get("times", [])
|
| 212 |
+
n = len(series)
|
| 213 |
+
if n > MAX_TS_POINTS:
|
| 214 |
+
bucket = math.ceil(n / MAX_TS_POINTS)
|
| 215 |
+
points = []
|
| 216 |
+
for i in range(0, n, bucket):
|
| 217 |
+
chunk = series[i:i + bucket]
|
| 218 |
+
j = i + max(range(len(chunk)), key=lambda k: abs(chunk[k]))
|
| 219 |
+
points.append({"t": str(times[j]) if j < len(times) else j, "v": round(float(series[j]), 6)})
|
| 220 |
+
else:
|
| 221 |
+
points = [{"t": str(times[i]) if i < len(times) else i, "v": round(float(v), 6)}
|
| 222 |
+
for i, v in enumerate(series)]
|
| 223 |
+
return {"object_id": object_id, "variable": variable, "n_source_points": n,
|
| 224 |
+
"decimated": n > MAX_TS_POINTS, "peak": round(float(max(series, key=abs, default=0.0)), 6),
|
| 225 |
+
"points": points}
|
| 226 |
+
|
| 227 |
+
|
| 228 |
+
def query_results(session_id: str, plan: dict | str) -> dict:
|
| 229 |
+
"""Execute a validated JSON retrieval plan against the bounded result DB.
|
| 230 |
+
|
| 231 |
+
Plan format: {"actions":[{"type":"select"|"aggregate","table":...,
|
| 232 |
+
"columns":[...], "filters":[{"column","op","value"}], "order_by":[...],
|
| 233 |
+
"limit":N, "aggregations":[{"agg","column"}]}]}. Use get_table_catalog
|
| 234 |
+
for table/column names. Read-only; invalid plans degrade gracefully.
|
| 235 |
+
"""
|
| 236 |
+
session = STORE.get(session_id)
|
| 237 |
+
_require_results(session)
|
| 238 |
+
if isinstance(plan, str):
|
| 239 |
+
plan = json.loads(plan)
|
| 240 |
+
agent = SafeSQLAgent(session.data["db"])
|
| 241 |
+
result = agent.execute_plan(plan)
|
| 242 |
+
return {"context": result.context[:12000]}
|
| 243 |
+
|
| 244 |
+
|
| 245 |
+
def get_table_catalog(session_id: str) -> dict:
|
| 246 |
+
"""List queryable tables (results + complete tokenized INP) for query_results."""
|
| 247 |
+
session = STORE.get(session_id)
|
| 248 |
+
_require_results(session)
|
| 249 |
+
return _df_records(session.data["db"].table_catalog(), 60)
|
| 250 |
+
|
| 251 |
+
|
| 252 |
+
# ---------------------------------------------------------------------------
|
| 253 |
+
# Screening and review
|
| 254 |
+
# ---------------------------------------------------------------------------
|
| 255 |
+
|
| 256 |
+
def calgary_screening(session_id: str) -> dict:
|
| 257 |
+
"""Screen results against City-of-Calgary-style criteria.
|
| 258 |
+
|
| 259 |
+
Velocity screen (3.0 m/s advisory / 4.0 m/s critical), storage
|
| 260 |
+
classification (trap-low vs pond heuristics), and the criteria register.
|
| 261 |
+
SCREENING ONLY — thresholds must be confirmed by the responsible engineer.
|
| 262 |
+
"""
|
| 263 |
+
session = STORE.get(session_id)
|
| 264 |
+
_require_results(session)
|
| 265 |
+
crit = CalgaryCriteria()
|
| 266 |
+
link_df = session.data["link_df"]
|
| 267 |
+
node_df = session.data["node_df"]
|
| 268 |
+
vel_col = next((c for c in link_df.columns if c.startswith("Peak Velocity")), None)
|
| 269 |
+
lv = link_df[["Link ID", vel_col, "Depth Ratio"]].copy()
|
| 270 |
+
lv["Screen"] = lv[vel_col].apply(
|
| 271 |
+
lambda v: "CRITICAL > 4.0" if v > 4.0 else ("Advisory > 3.0" if v > 3.0 else "OK"))
|
| 272 |
+
flagged = lv[lv["Screen"] != "OK"].sort_values(vel_col, ascending=False)
|
| 273 |
+
storage = node_df[node_df["Type"].astype(str).str.lower() == "storage"].copy()
|
| 274 |
+
storage_class = apply_storage_classification(storage, crit, "m") if not storage.empty else pd.DataFrame()
|
| 275 |
+
return {
|
| 276 |
+
"velocity_screen_flagged": _df_records(flagged, 30),
|
| 277 |
+
"storage_classification": _df_records(storage_class, 30),
|
| 278 |
+
"criteria_register": _df_records(criteria_register(crit), 40),
|
| 279 |
+
"status": "Screening only — criteria applicability requires engineer confirmation.",
|
| 280 |
+
}
|
| 281 |
+
|
| 282 |
+
|
| 283 |
+
def preliminary_design_review(session_id: str) -> dict:
|
| 284 |
+
"""Deterministic QA/QC findings register (topology, hydrology, screening)
|
| 285 |
+
merged with the worker-vs-.rpt reconciliation findings (RPT-###)."""
|
| 286 |
+
session = STORE.get(session_id)
|
| 287 |
+
_require_results(session)
|
| 288 |
+
inp_text = Path(session.data["inp_path"]).read_text(encoding="utf-8", errors="replace")
|
| 289 |
+
findings = build_deterministic_findings(
|
| 290 |
+
inp_text=inp_text, node_summary=session.data["node_df"],
|
| 291 |
+
link_summary=session.data["link_df"], sub_summary=session.data["sub_df"],
|
| 292 |
+
metadata=session.data["results"]["metadata"],
|
| 293 |
+
simulation_completed=True, output_results_available=True)
|
| 294 |
+
recon_findings = rr.reconciliation_findings(
|
| 295 |
+
session.data.get("recon_links"), session.data.get("recon_nodes"),
|
| 296 |
+
session.data.get("recon_continuity"))
|
| 297 |
+
all_findings = list(findings) + list(recon_findings)
|
| 298 |
+
session.data["findings"] = all_findings
|
| 299 |
+
return {"findings": _df_records(findings_dataframe(all_findings), 60),
|
| 300 |
+
"rpt_reconciliation": session.data.get("recon_summary", {})}
|
| 301 |
+
|
| 302 |
+
|
| 303 |
+
def get_reconciliation(session_id: str) -> dict:
|
| 304 |
+
"""Worker-vs-.rpt reconciliation detail: flagged links and continuity check.
|
| 305 |
+
For flagged links, .rpt values are authoritative for screening."""
|
| 306 |
+
session = STORE.get(session_id)
|
| 307 |
+
_require_results(session)
|
| 308 |
+
lr = session.data.get("recon_links")
|
| 309 |
+
flagged = lr[lr["Overall Status"] != "OK"] if lr is not None and not lr.empty else pd.DataFrame()
|
| 310 |
+
return {"summary": session.data.get("recon_summary", {}),
|
| 311 |
+
"flagged_links": _df_records(flagged, 40),
|
| 312 |
+
"continuity": _df_records(session.data.get("recon_continuity"), 5)}
|
| 313 |
+
|
| 314 |
+
|
| 315 |
+
# ---------------------------------------------------------------------------
|
| 316 |
+
# Scenarios and reporting
|
| 317 |
+
# ---------------------------------------------------------------------------
|
| 318 |
+
|
| 319 |
+
def run_scenario(session_id: str, scenario_name: str,
|
| 320 |
+
conduit_diameter_overrides: dict | str | None = None,
|
| 321 |
+
rainfall_multiplier: float | None = None) -> dict:
|
| 322 |
+
"""Clone the base model, apply controlled changes, re-simulate, compare.
|
| 323 |
+
|
| 324 |
+
conduit_diameter_overrides: {"link_id": new_diameter_m}. The base model is
|
| 325 |
+
never mutated; comparisons quote deterministic summary deltas.
|
| 326 |
+
"""
|
| 327 |
+
import scenario_manager as sm
|
| 328 |
+
session = STORE.get(session_id)
|
| 329 |
+
_require_results(session)
|
| 330 |
+
if isinstance(conduit_diameter_overrides, str) and conduit_diameter_overrides:
|
| 331 |
+
conduit_diameter_overrides = json.loads(conduit_diameter_overrides)
|
| 332 |
+
kwargs: dict[str, Any] = {}
|
| 333 |
+
if conduit_diameter_overrides:
|
| 334 |
+
kwargs["conduit_diameter_overrides"] = {str(k): float(v) for k, v in conduit_diameter_overrides.items()}
|
| 335 |
+
if rainfall_multiplier is not None:
|
| 336 |
+
kwargs["rainfall_multiplier"] = float(rainfall_multiplier)
|
| 337 |
+
scen_id = f"scn_{len(session.data.setdefault('scenarios', [])) + 1}"
|
| 338 |
+
definition = sm.ScenarioDefinition(scenario_id=scen_id, scenario_name=scenario_name, **kwargs)
|
| 339 |
+
record = sm.run_scenario(session.data["inp_path"], definition,
|
| 340 |
+
work_dir=str(session.workdir / "scenarios"))
|
| 341 |
+
base_record = sm.base_model_record(session.data["results"])
|
| 342 |
+
session.data["scenarios"].append(record)
|
| 343 |
+
comparison = sm.comparison_with_base(base_record, session.data["scenarios"])
|
| 344 |
+
narrative = sm.deterministic_comparison_analysis(comparison)
|
| 345 |
+
session.data["scenario_comparison"] = comparison
|
| 346 |
+
keep = [c for c in comparison.columns if comparison[c].dtype != object or c in
|
| 347 |
+
("Scenario ID", "Scenario Name", "Simulation Status", "Velocity Link")]
|
| 348 |
+
return {"scenario_id": scen_id, "comparison": _df_records(comparison[keep], 20),
|
| 349 |
+
"deterministic_analysis": narrative[:6000]}
|
| 350 |
+
|
| 351 |
+
|
| 352 |
+
def generate_report(session_id: str, project_name: str, client: str = "",
|
| 353 |
+
consultant: str = "", prepared_by: str = "",
|
| 354 |
+
outline_plan_no: str = "") -> dict:
|
| 355 |
+
"""Generate the Calgary-style SWMR draft package (docx + audit zip).
|
| 356 |
+
|
| 357 |
+
Returns download paths served by this Space at /files/{session_id}/{name}.
|
| 358 |
+
The draft-readiness score honestly reflects missing project information.
|
| 359 |
+
"""
|
| 360 |
+
from report_engine import ReportMetadata, generate_report_package
|
| 361 |
+
session = STORE.get(session_id)
|
| 362 |
+
_require_results(session)
|
| 363 |
+
meta = ReportMetadata(project_name=project_name, client=client, consultant=consultant,
|
| 364 |
+
prepared_by=prepared_by, outline_plan_no=outline_plan_no)
|
| 365 |
+
findings = session.data.get("findings") or []
|
| 366 |
+
pkg = generate_report_package(
|
| 367 |
+
metadata=meta, inp_sections=session.data["sections"],
|
| 368 |
+
node_summary=session.data["node_df"], link_summary=session.data["link_df"],
|
| 369 |
+
sub_summary=session.data["sub_df"],
|
| 370 |
+
simulation_metadata=session.data["results"]["metadata"],
|
| 371 |
+
result_db_bytes=session.data["db"].export_bytes(),
|
| 372 |
+
preliminary_review_artifacts={
|
| 373 |
+
"findings": findings, "status": "Preliminary",
|
| 374 |
+
"manifest": {"rpt_reconciliation": session.data.get("recon_summary", {})},
|
| 375 |
+
} if findings else None)
|
| 376 |
+
outputs = session.workdir / "outputs"
|
| 377 |
+
outputs.mkdir(exist_ok=True)
|
| 378 |
+
files = {}
|
| 379 |
+
for key, blob in pkg.items():
|
| 380 |
+
if isinstance(blob, (bytes, bytearray)):
|
| 381 |
+
name = pkg.get(f"{key}_name") if isinstance(pkg.get(f"{key}_name"), str) else f"{key}.bin"
|
| 382 |
+
if key == "docx":
|
| 383 |
+
name = f"{project_name.replace(' ', '_')}_SWMR_Draft.docx"
|
| 384 |
+
elif key == "zip":
|
| 385 |
+
name = f"{project_name.replace(' ', '_')}_SWMR_Package.zip"
|
| 386 |
+
(outputs / name).write_bytes(blob)
|
| 387 |
+
files[key] = f"/files/{session.id}/{name}"
|
| 388 |
+
return {"files": files, "size_bytes": {k: len(v) for k, v in pkg.items() if isinstance(v, (bytes, bytearray))},
|
| 389 |
+
"note": "Draft for engineering review — not an issued document."}
|
| 390 |
+
|
| 391 |
+
|
| 392 |
+
# ---------------------------------------------------------------------------
|
| 393 |
+
# Registry
|
| 394 |
+
# ---------------------------------------------------------------------------
|
| 395 |
+
|
| 396 |
+
TOOL_REGISTRY: dict[str, Callable[..., dict]] = {
|
| 397 |
+
fn.__name__: fn for fn in [
|
| 398 |
+
upload_model, run_simulation, list_sessions, close_session,
|
| 399 |
+
get_node_results, get_link_results, get_subcatchment_results,
|
| 400 |
+
get_timeseries, query_results, get_table_catalog,
|
| 401 |
+
calgary_screening, preliminary_design_review, get_reconciliation,
|
| 402 |
+
run_scenario, generate_report,
|
| 403 |
+
]
|
| 404 |
+
}
|
worker-requirements.txt
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
numpy==1.26.4
|
| 2 |
+
openswmm==6.0.0a2
|