Spaces:
Build error
Build error
Upload folder using huggingface_hub
#74
by petter2025 - opened
- Dockerfile +3 -60
- README.md +7 -21
- alembic/versions/a1f3c9d2e6b7_create_api_keys_table.py +0 -58
- alembic/versions/e4a7c1f9b3d2_create_onchain_rationales_table.py +0 -62
- app/api/routes_admin.py +49 -194
- app/api/routes_governance.py +7 -214
- app/api/routes_history.py +3 -4
- app/api/routes_incidents.py +13 -15
- app/api/routes_intents.py +4 -10
- app/api/routes_memory.py +2 -3
- app/api/routes_onchain.py +0 -149
- app/api/routes_payments.py +13 -47
- app/api/routes_pricing.py +44 -22
- app/api/routes_risk.py +14 -10
- app/api/routes_users.py +3 -4
- app/api/webhooks.py +13 -34
- app/core/config.py +0 -1
- app/core/storage.py +2 -16
- app/core/usage_tracker.py +62 -426
- app/database/models_intents.py +65 -6
- app/database/models_onchain.py +0 -54
- app/main.py +6 -113
- deploy/kubernetes/arf-api/secret.yaml +1 -14
- docs/authentication.md +10 -40
- docs/disaster-recovery.md +396 -0
- docs/pilot-readiness.md +125 -0
- docs/regulatory-mapping.md +212 -0
- docs/security-assessment.md +186 -0
- render.yaml +0 -2
- requirements.txt +1 -6
- tests/conftest.py +0 -7
- tests/test_deps.py +1 -75
- tests/test_governance.py +0 -37
- tests/test_healing_endpoint.py +0 -28
- tests/test_history.py +6 -2
- tests/test_payments.py +11 -45
- tests/test_risk.py +1 -5
- tests/test_routes_admin.py +0 -126
- tests/test_routes_governance_execute.py +0 -352
- tests/test_routes_onchain.py +0 -88
- tests/test_routes_pricing.py +0 -47
- tests/test_tracker_binding.py +0 -95
- tests/test_usage_tracker.py +0 -128
- tests/test_webhooks.py +26 -95
Dockerfile
CHANGED
|
@@ -1,64 +1,7 @@
|
|
| 1 |
-
|
| 2 |
-
# ---- deps stage: needs git + a credentialed clone of the private ARF repos
|
| 3 |
-
# (agentic_reliability_framework, ARF-Bayesian-Pricing-Calculator).
|
| 4 |
-
# This stage is discarded after build -- the credential never reaches
|
| 5 |
-
# the final image's layers, env, or git config. ----
|
| 6 |
-
#
|
| 7 |
-
# GH_PAT is read via a BuildKit secret mount, not `ARG` -- an ARG's value is
|
| 8 |
-
# printed in plaintext as part of the logged RUN command that uses it (this
|
| 9 |
-
# is exactly how a real, live token ended up visible in a Render deploy log
|
| 10 |
-
# this session). A secret mount's value is never written to a log line or
|
| 11 |
-
# an image layer. REQUIRES a matching setup step in Render's dashboard
|
| 12 |
-
# before this will build: Render's Docker service settings -> Secret Files
|
| 13 |
-
# -> add a file named exactly `gh_pat` containing the token value (nothing
|
| 14 |
-
# else in the file). The old `GH_PAT` environment variable is no longer
|
| 15 |
-
# read by this Dockerfile and can be removed once this is confirmed working.
|
| 16 |
-
FROM python:3.12-slim AS deps
|
| 17 |
RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/*
|
| 18 |
-
RUN --mount=type=secret,id=gh_pat,dst=/etc/secrets/gh_pat \
|
| 19 |
-
git config --global url."https://$(cat /etc/secrets/gh_pat)@github.com/".insteadOf "https://github.com/"
|
| 20 |
-
RUN python -m venv /opt/venv
|
| 21 |
-
ENV PATH="/opt/venv/bin:$PATH"
|
| 22 |
WORKDIR /app
|
| 23 |
COPY requirements.txt .
|
| 24 |
-
|
| 25 |
-
# transitively by sentence-transformers (for agentic_reliability_framework's
|
| 26 |
-
# RAG/semantic-memory features) and, left to the default PyPI index, resolves
|
| 27 |
-
# to the CUDA-enabled build (nvidia-cusparselt, cuda-toolkit, nvidia-nccl, ...)
|
| 28 |
-
# even though this service runs on CPU-only Render instances. That variant's
|
| 29 |
-
# extra weight is a real contributor to out-of-memory deploy failures.
|
| 30 |
-
#
|
| 31 |
-
# A separate `pip install torch==... --index-url .../cpu` RUN before this one
|
| 32 |
-
# does NOT work: it's a distinct resolve that only knows about the CPU wheel;
|
| 33 |
-
# the very next `pip install -r requirements.txt`, seeing no --index-url, only
|
| 34 |
-
# has the default PyPI index in view and re-resolves torch from there,
|
| 35 |
-
# silently replacing the CPU build with the CUDA one at the same version
|
| 36 |
-
# number (confirmed happening in a real deploy -- final `pip install` log
|
| 37 |
-
# showed plain `torch-2.13.0` plus the full nvidia/cuda-toolkit/triton stack,
|
| 38 |
-
# not `torch-2.13.0+cpu`). Putting torch and -r requirements.txt in one
|
| 39 |
-
# `pip install` call, with the CPU wheelhouse as the primary --index-url and
|
| 40 |
-
# PyPI as --extra-index-url, makes it a single resolve: torch is satisfied
|
| 41 |
-
# from the CPU index and nothing later re-derives a different build for it.
|
| 42 |
-
# Version pinned to 2.13.0 to match exactly what pip's resolver already chose
|
| 43 |
-
# for this dependency tree (confirmed available on the CPU index for
|
| 44 |
-
# cp312/manylinux before pinning it here, not assumed).
|
| 45 |
-
RUN pip install --no-cache-dir \
|
| 46 |
-
--index-url https://download.pytorch.org/whl/cpu \
|
| 47 |
-
--extra-index-url https://pypi.org/simple \
|
| 48 |
-
torch==2.13.0 \
|
| 49 |
-
-r requirements.txt
|
| 50 |
-
|
| 51 |
-
# ---- final stage: just the built venv + app code, no git, no credential ----
|
| 52 |
-
FROM python:3.12-slim
|
| 53 |
-
COPY --from=deps /opt/venv /opt/venv
|
| 54 |
-
ENV PATH="/opt/venv/bin:$PATH"
|
| 55 |
-
WORKDIR /app
|
| 56 |
COPY . .
|
| 57 |
-
|
| 58 |
-
# expands with a real shell interpreting the command; exec form passes
|
| 59 |
-
# arguments literally with no variable substitution at all. Render injects
|
| 60 |
-
# PORT and expects the app to bind to it (its deploy log explicitly failed
|
| 61 |
-
# port-scanning for it: "Bind your service to at least one port"); the
|
| 62 |
-
# Hugging Face Space mirror sets no such variable and expects the
|
| 63 |
-
# conventional default, 7860. One image, correct on both targets.
|
| 64 |
-
CMD uvicorn app.main:app --host 0.0.0.0 --port ${PORT:-7860}
|
|
|
|
| 1 |
+
FROM python:3.12-slim
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2 |
RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/*
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
WORKDIR /app
|
| 4 |
COPY requirements.txt .
|
| 5 |
+
RUN pip install --no-cache-dir -r requirements.txt
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6 |
COPY . .
|
| 7 |
+
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "7860"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
README.md
CHANGED
|
@@ -1,31 +1,16 @@
|
|
| 1 |
-
---
|
| 2 |
-
title: ARF API
|
| 3 |
-
emoji: 🛡️
|
| 4 |
-
colorFrom: blue
|
| 5 |
-
colorTo: gray
|
| 6 |
-
sdk: docker
|
| 7 |
-
pinned: false
|
| 8 |
-
---
|
| 9 |
-
|
| 10 |
# arf-api
|
| 11 |
|
| 12 |
ARF API Control Plane (FastAPI)
|
| 13 |
|
| 14 |
## Live Demo
|
| 15 |
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
Face Space below is a secondary, publicly-browsable mirror of the same code, not the primary
|
| 19 |
-
integration target -- point real pilot/customer integrations at Render once its URL is
|
| 20 |
-
confirmed live, not at the Space URL.
|
| 21 |
-
|
| 22 |
-
- **HF Space (public mirror)**: [https://arf-ai-agentic-reliability-framework-api.hf.space](https://arf-ai-agentic-reliability-framework-api.hf.space)
|
| 23 |
- **Interactive Documentation**: [https://arf-ai-agentic-reliability-framework-api.hf.space/docs](https://arf-ai-agentic-reliability-framework-api.hf.space/docs)
|
| 24 |
|
| 25 |
## Quick Start (Local Development)
|
| 26 |
|
| 27 |
1. **Install dependencies**:
|
| 28 |
-
|
| 29 |
```bash
|
| 30 |
pip install -r requirements.txt
|
| 31 |
```
|
|
@@ -39,11 +24,9 @@ ARF_HMC_MODEL – path to HMC model JSON (default: models/hmc_model.json)
|
|
| 39 |
|
| 40 |
ARF_USE_HYPERPRIORS – true/false
|
| 41 |
|
| 42 |
-
API_KEY –
|
| 43 |
```
|
| 44 |
|
| 45 |
-
The settings that actually gate access are `ARF_INTERNAL_API_KEY` and `ARF_ADMIN_API_KEY`, not `API_KEY` above — see [docs/authentication.md](docs/authentication.md) for what's actually enforced and what isn't yet.
|
| 46 |
-
|
| 47 |
3. **Run the app locally**:
|
| 48 |
|
| 49 |
```bash
|
|
@@ -106,7 +89,9 @@ curl -X POST "http://localhost:8000/api/v1/v1/incidents/evaluate" -H "Content-
|
|
| 106 |
"effect": -90,
|
| 107 |
"explanation_text": "If we apply restart_container instead of no_action, latency would change from 600.00 to 510.00 (Δ = -90.00). Based on heuristic causal model.",
|
| 108 |
"is_model_based": false,
|
| 109 |
-
"warnings": [
|
|
|
|
|
|
|
| 110 |
},
|
| 111 |
"utility_decision": {
|
| 112 |
"best_action": "restart_container",
|
|
@@ -133,3 +118,4 @@ Notes
|
|
| 133 |
|
| 134 |
- The governance endpoints use an in-process `RiskEngine` initialized at startup.
|
| 135 |
- Outcomes are recorded via `POST /api/v1/intents/outcome` (`app/api/routes_governance.py`), tenant-scoped and auth-protected.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
# arf-api
|
| 2 |
|
| 3 |
ARF API Control Plane (FastAPI)
|
| 4 |
|
| 5 |
## Live Demo
|
| 6 |
|
| 7 |
+
The API is deployed and accessible at:
|
| 8 |
+
- **Base URL**: [https://arf-ai-agentic-reliability-framework-api.hf.space](https://arf-ai-agentic-reliability-framework-api.hf.space)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
- **Interactive Documentation**: [https://arf-ai-agentic-reliability-framework-api.hf.space/docs](https://arf-ai-agentic-reliability-framework-api.hf.space/docs)
|
| 10 |
|
| 11 |
## Quick Start (Local Development)
|
| 12 |
|
| 13 |
1. **Install dependencies**:
|
|
|
|
| 14 |
```bash
|
| 15 |
pip install -r requirements.txt
|
| 16 |
```
|
|
|
|
| 24 |
|
| 25 |
ARF_USE_HYPERPRIORS – true/false
|
| 26 |
|
| 27 |
+
API_KEY – optional (currently not enforced)
|
| 28 |
```
|
| 29 |
|
|
|
|
|
|
|
| 30 |
3. **Run the app locally**:
|
| 31 |
|
| 32 |
```bash
|
|
|
|
| 89 |
"effect": -90,
|
| 90 |
"explanation_text": "If we apply restart_container instead of no_action, latency would change from 600.00 to 510.00 (Δ = -90.00). Based on heuristic causal model.",
|
| 91 |
"is_model_based": false,
|
| 92 |
+
"warnings": [
|
| 93 |
+
"Using heuristic causal model (no fitted SCM)."
|
| 94 |
+
]
|
| 95 |
},
|
| 96 |
"utility_decision": {
|
| 97 |
"best_action": "restart_container",
|
|
|
|
| 118 |
|
| 119 |
- The governance endpoints use an in-process `RiskEngine` initialized at startup.
|
| 120 |
- Outcomes are recorded via `POST /api/v1/intents/outcome` (`app/api/routes_governance.py`), tenant-scoped and auth-protected.
|
| 121 |
+
|
alembic/versions/a1f3c9d2e6b7_create_api_keys_table.py
DELETED
|
@@ -1,58 +0,0 @@
|
|
| 1 |
-
"""create api_keys table (moves API key storage off ephemeral SQLite)
|
| 2 |
-
|
| 3 |
-
api_keys previously lived only in a SQLite file per service (arf_usage.db),
|
| 4 |
-
which is wiped on every Render deploy/restart on the Free plan and was also
|
| 5 |
-
independently duplicated between arf-api and arf-gateway. This creates the
|
| 6 |
-
durable, single source of truth in Postgres. Rows are hashed at rest
|
| 7 |
-
(pepper-HMAC lookup_hash + salted key_hash) -- no plaintext key column,
|
| 8 |
-
since there is no pre-pepper data to migrate here (confirmed with the user:
|
| 9 |
-
existing SQLite api_keys rows in both services are safe to discard, keys get
|
| 10 |
-
reissued via POST /admin/keys).
|
| 11 |
-
|
| 12 |
-
Revision ID: a1f3c9d2e6b7
|
| 13 |
-
Revises: d36deffe7fa2
|
| 14 |
-
Create Date: 2026-08-24 00:00:00.000000
|
| 15 |
-
|
| 16 |
-
"""
|
| 17 |
-
from typing import Sequence, Union
|
| 18 |
-
|
| 19 |
-
from alembic import op
|
| 20 |
-
import sqlalchemy as sa
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
# revision identifiers, used by Alembic.
|
| 24 |
-
revision: str = 'a1f3c9d2e6b7'
|
| 25 |
-
down_revision: Union[str, Sequence[str], None] = 'd36deffe7fa2'
|
| 26 |
-
branch_labels: Union[str, Sequence[str], None] = None
|
| 27 |
-
depends_on: Union[str, Sequence[str], None] = None
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
def upgrade() -> None:
|
| 31 |
-
"""Upgrade schema."""
|
| 32 |
-
# No FK to tenants.id here, deliberately -- the pre-existing SQLite
|
| 33 |
-
# schema this replaces never enforced one either (seeded/demo keys via
|
| 34 |
-
# ARF_API_KEYS may reference synthetic tenant_ids that don't have a
|
| 35 |
-
# tenants row), and adding one now would be a behavior change beyond
|
| 36 |
-
# this migration's scope.
|
| 37 |
-
op.create_table(
|
| 38 |
-
'api_keys',
|
| 39 |
-
sa.Column('id', sa.Integer(), nullable=False),
|
| 40 |
-
sa.Column('tenant_id', sa.String(length=64), nullable=False),
|
| 41 |
-
sa.Column('tier', sa.String(length=32), nullable=False),
|
| 42 |
-
sa.Column('created_at', sa.DateTime(), nullable=False),
|
| 43 |
-
sa.Column('last_used_at', sa.DateTime(), nullable=True),
|
| 44 |
-
sa.Column('is_active', sa.Boolean(), nullable=False, server_default=sa.true()),
|
| 45 |
-
sa.Column('salt', sa.String(length=64), nullable=False),
|
| 46 |
-
sa.Column('key_hash', sa.String(length=64), nullable=False),
|
| 47 |
-
sa.Column('lookup_hash', sa.String(length=64), nullable=False),
|
| 48 |
-
sa.PrimaryKeyConstraint('id'),
|
| 49 |
-
)
|
| 50 |
-
op.create_index(op.f('ix_api_keys_tenant_id'), 'api_keys', ['tenant_id'], unique=False)
|
| 51 |
-
op.create_unique_constraint('uq_api_keys_lookup_hash', 'api_keys', ['lookup_hash'])
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
def downgrade() -> None:
|
| 55 |
-
"""Downgrade schema."""
|
| 56 |
-
op.drop_constraint('uq_api_keys_lookup_hash', 'api_keys', type_='unique')
|
| 57 |
-
op.drop_index(op.f('ix_api_keys_tenant_id'), table_name='api_keys')
|
| 58 |
-
op.drop_table('api_keys')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
alembic/versions/e4a7c1f9b3d2_create_onchain_rationales_table.py
DELETED
|
@@ -1,62 +0,0 @@
|
|
| 1 |
-
"""create onchain_rationales table
|
| 2 |
-
|
| 3 |
-
Stores the plaintext preimage of an anchored `RiskAttestation.rationale_hash`
|
| 4 |
-
(arf-onchain's RiskAttestationRegistry / enterprise's
|
| 5 |
-
arf_enterprise.onchain.attestation). The chain only ever holds the hash --
|
| 6 |
-
see models_onchain.py's module docstring for why the text has to live here,
|
| 7 |
-
keyed by that same hash, or the anchored hash is unverifiable.
|
| 8 |
-
|
| 9 |
-
Revision ID: e4a7c1f9b3d2
|
| 10 |
-
Revises: a1f3c9d2e6b7
|
| 11 |
-
Create Date: 2026-09-07 00:00:00.000000
|
| 12 |
-
|
| 13 |
-
"""
|
| 14 |
-
|
| 15 |
-
from typing import Sequence, Union
|
| 16 |
-
|
| 17 |
-
from alembic import op
|
| 18 |
-
import sqlalchemy as sa
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
# revision identifiers, used by Alembic.
|
| 22 |
-
revision: str = "e4a7c1f9b3d2"
|
| 23 |
-
down_revision: Union[str, Sequence[str], None] = "a1f3c9d2e6b7"
|
| 24 |
-
branch_labels: Union[str, Sequence[str], None] = None
|
| 25 |
-
depends_on: Union[str, Sequence[str], None] = None
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
def upgrade() -> None:
|
| 29 |
-
"""Upgrade schema."""
|
| 30 |
-
op.create_table(
|
| 31 |
-
"onchain_rationales",
|
| 32 |
-
sa.Column("id", sa.String(length=64), nullable=False),
|
| 33 |
-
sa.Column("rationale_hash", sa.String(length=66), nullable=False),
|
| 34 |
-
sa.Column("rationale", sa.Text(), nullable=False),
|
| 35 |
-
sa.Column("agent_address", sa.String(length=42), nullable=True),
|
| 36 |
-
sa.Column("evaluator_address", sa.String(length=42), nullable=True),
|
| 37 |
-
sa.Column("created_at", sa.DateTime(), nullable=False),
|
| 38 |
-
sa.PrimaryKeyConstraint("id"),
|
| 39 |
-
)
|
| 40 |
-
op.create_index(
|
| 41 |
-
op.f("ix_onchain_rationales_rationale_hash"),
|
| 42 |
-
"onchain_rationales",
|
| 43 |
-
["rationale_hash"],
|
| 44 |
-
unique=True,
|
| 45 |
-
)
|
| 46 |
-
op.create_index(
|
| 47 |
-
op.f("ix_onchain_rationales_created_at"),
|
| 48 |
-
"onchain_rationales",
|
| 49 |
-
["created_at"],
|
| 50 |
-
unique=False,
|
| 51 |
-
)
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
def downgrade() -> None:
|
| 55 |
-
"""Downgrade schema."""
|
| 56 |
-
op.drop_index(
|
| 57 |
-
op.f("ix_onchain_rationales_created_at"), table_name="onchain_rationales"
|
| 58 |
-
)
|
| 59 |
-
op.drop_index(
|
| 60 |
-
op.f("ix_onchain_rationales_rationale_hash"), table_name="onchain_rationales"
|
| 61 |
-
)
|
| 62 |
-
op.drop_table("onchain_rationales")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
app/api/routes_admin.py
CHANGED
|
@@ -2,18 +2,14 @@
|
|
| 2 |
Admin API endpoints for API key management and audit logs.
|
| 3 |
These endpoints should be protected (e.g., by an admin API key) in production.
|
| 4 |
"""
|
| 5 |
-
from fastapi import APIRouter, Depends, HTTPException, Query, Path, Body
|
| 6 |
from pydantic import BaseModel
|
| 7 |
from typing import Optional
|
| 8 |
from datetime import datetime
|
| 9 |
import os
|
| 10 |
import secrets
|
| 11 |
import uuid
|
| 12 |
-
from
|
| 13 |
-
from app.api.deps import get_db
|
| 14 |
-
from app.core import usage_tracker
|
| 15 |
-
from app.core.usage_tracker import Tier
|
| 16 |
-
from app.database.models_intents import TenantDB
|
| 17 |
|
| 18 |
router = APIRouter(prefix="/admin", tags=["admin"])
|
| 19 |
# Admin key must be supplied via environment; there is no default. Fail closed
|
|
@@ -31,8 +27,6 @@ def verify_admin(admin_key: str = Query(..., alias="admin_key")):
|
|
| 31 |
|
| 32 |
class CreateKeyRequest(BaseModel):
|
| 33 |
tier: str
|
| 34 |
-
tenant_id: Optional[str] = None # attach to an existing tenant; omit to create a new one
|
| 35 |
-
org_name: Optional[str] = None # used only when creating a new tenant
|
| 36 |
|
| 37 |
|
| 38 |
class UpdateTierRequest(BaseModel):
|
|
@@ -40,141 +34,75 @@ class UpdateTierRequest(BaseModel):
|
|
| 40 |
|
| 41 |
|
| 42 |
@router.post("/keys", dependencies=[Depends(verify_admin)])
|
| 43 |
-
async def create_api_key(req: CreateKeyRequest
|
| 44 |
-
# Previously called get_or_create_api_key(new_key, tier_enum) -- since
|
| 45 |
-
# that function's signature is (key, tenant_id, tier=FREE), tier_enum
|
| 46 |
-
# was silently accepted as tenant_id and every key of the same tier
|
| 47 |
-
# collided onto one bogus tenant_id (e.g. every "free" key sharing
|
| 48 |
-
# tenant_id="free"). tenant_id gates real per-tenant isolation
|
| 49 |
-
# elsewhere (BetaStateDB, IntentDB, decision audit log), so this was a
|
| 50 |
-
# cross-tenant data bug, not just a mislabeled field.
|
| 51 |
if req.tier not in [t.value for t in Tier]:
|
| 52 |
raise HTTPException(
|
| 53 |
status_code=400, detail=f"Invalid tier. Must be one of {[t.value for t in Tier]}")
|
| 54 |
-
tier_enum = Tier(req.tier)
|
| 55 |
-
|
| 56 |
-
tenant_id = req.tenant_id
|
| 57 |
-
if tenant_id:
|
| 58 |
-
if not db.query(TenantDB).filter(TenantDB.id == tenant_id).first():
|
| 59 |
-
raise HTTPException(status_code=404, detail=f"Tenant {tenant_id} not found")
|
| 60 |
-
else:
|
| 61 |
-
tenant_id = str(uuid.uuid4())
|
| 62 |
-
db.add(TenantDB(
|
| 63 |
-
id=tenant_id,
|
| 64 |
-
name=req.org_name or "Default Organization",
|
| 65 |
-
created_at=datetime.utcnow(),
|
| 66 |
-
created_by="admin",
|
| 67 |
-
))
|
| 68 |
-
db.commit()
|
| 69 |
-
|
| 70 |
new_key = f"sk_live_{uuid.uuid4().hex[:24]}"
|
| 71 |
-
|
| 72 |
-
|
|
|
|
| 73 |
|
| 74 |
|
| 75 |
@router.get("/keys", dependencies=[Depends(verify_admin)])
|
| 76 |
async def list_api_keys(limit: int = 100, offset: int = 0):
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
tier/deactivate endpoints below. `current_month_usage` is not shown
|
| 81 |
-
here: `monthly_counts` is intentionally still keyed by the raw API key
|
| 82 |
-
(arf-gateway depends on reading it that way), which this endpoint no
|
| 83 |
-
longer has -- query `/admin/keys/{api_key}/audit` with the real key for
|
| 84 |
-
per-key usage/audit history instead.
|
| 85 |
-
"""
|
| 86 |
-
with usage_tracker.tracker._get_pg_conn() as conn:
|
| 87 |
-
rows = usage_tracker.tracker._pg_execute(
|
| 88 |
-
conn,
|
| 89 |
-
"SELECT lookup_hash, tier, created_at, last_used_at, is_active FROM api_keys "
|
| 90 |
-
"ORDER BY created_at DESC LIMIT %s OFFSET %s",
|
| 91 |
(limit, offset)
|
| 92 |
-
).fetchall()
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
"
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 104 |
return {"keys": keys, "total": len(keys)}
|
| 105 |
|
| 106 |
|
| 107 |
-
@router.patch("/keys/{
|
| 108 |
async def update_key_tier(
|
| 109 |
-
|
| 110 |
req: UpdateTierRequest = Body(...),
|
| 111 |
):
|
| 112 |
if req.tier not in [t.value for t in Tier]:
|
| 113 |
raise HTTPException(
|
| 114 |
status_code=400, detail=f"Invalid tier. Must be one of {[t.value for t in Tier]}")
|
| 115 |
-
with
|
| 116 |
-
row =
|
| 117 |
-
|
| 118 |
if not row:
|
| 119 |
-
conn.rollback()
|
| 120 |
raise HTTPException(status_code=404, detail="API key not found")
|
| 121 |
-
|
| 122 |
-
|
| 123 |
conn.commit()
|
| 124 |
return {"message": f"Tier updated to {req.tier}"}
|
| 125 |
|
| 126 |
|
| 127 |
-
@router.
|
| 128 |
-
async def rotate_api_key(
|
| 129 |
-
key_id: str = Path(..., description="The key_id from GET /admin/keys (not the raw API key)"),
|
| 130 |
-
):
|
| 131 |
-
"""Atomically deactivate a key and issue a new one on the same tenant
|
| 132 |
-
and tier -- the single action a leaked/compromised key actually needs.
|
| 133 |
-
Doing this as get-old + deactivate + create separately (the only option
|
| 134 |
-
before this endpoint existed) risks losing track of the tenant_id
|
| 135 |
-
partway through, or leaving the old key active if a later step fails.
|
| 136 |
-
The new plaintext key is returned exactly once, like create_api_key's."""
|
| 137 |
-
with usage_tracker.tracker._get_pg_conn() as conn:
|
| 138 |
-
old_row = usage_tracker.tracker._pg_execute(
|
| 139 |
-
conn, "SELECT tenant_id, tier FROM api_keys WHERE lookup_hash = %s", (key_id,)
|
| 140 |
-
).fetchone()
|
| 141 |
-
if not old_row:
|
| 142 |
-
conn.rollback()
|
| 143 |
-
raise HTTPException(status_code=404, detail="API key not found")
|
| 144 |
-
|
| 145 |
-
new_key = f"sk_live_{uuid.uuid4().hex[:24]}"
|
| 146 |
-
salt = secrets.token_hex(16)
|
| 147 |
-
usage_tracker.tracker._pg_execute(
|
| 148 |
-
conn, "UPDATE api_keys SET is_active = false WHERE lookup_hash = %s", (key_id,))
|
| 149 |
-
usage_tracker.tracker._pg_execute(
|
| 150 |
-
conn,
|
| 151 |
-
"INSERT INTO api_keys "
|
| 152 |
-
"(tenant_id, tier, created_at, is_active, salt, key_hash, lookup_hash) "
|
| 153 |
-
"VALUES (%s, %s, %s, %s, %s, %s, %s)",
|
| 154 |
-
(old_row["tenant_id"], old_row["tier"], datetime.utcnow(), True,
|
| 155 |
-
salt, usage_tracker.tracker._salted_hash(new_key, salt), usage_tracker.tracker._lookup_hash(new_key)),
|
| 156 |
-
)
|
| 157 |
-
conn.commit()
|
| 158 |
-
|
| 159 |
-
return {
|
| 160 |
-
"api_key": new_key,
|
| 161 |
-
"tenant_id": old_row["tenant_id"],
|
| 162 |
-
"tier": old_row["tier"],
|
| 163 |
-
"deactivated_key_id": key_id,
|
| 164 |
-
}
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
@router.delete("/keys/{key_id}", dependencies=[Depends(verify_admin)])
|
| 168 |
async def deactivate_api_key(
|
| 169 |
-
|
| 170 |
-
with
|
| 171 |
-
row =
|
| 172 |
-
|
| 173 |
if not row:
|
| 174 |
-
conn.rollback()
|
| 175 |
raise HTTPException(status_code=404, detail="API key not found")
|
| 176 |
-
|
| 177 |
-
|
| 178 |
conn.commit()
|
| 179 |
return {"message": "API key deactivated"}
|
| 180 |
|
|
@@ -188,23 +116,21 @@ async def get_audit_logs(
|
|
| 188 |
):
|
| 189 |
start = datetime.fromisoformat(start_date) if start_date else None
|
| 190 |
end = datetime.fromisoformat(end_date) if end_date else None
|
| 191 |
-
logs =
|
| 192 |
return {"api_key": api_key, "logs": logs}
|
| 193 |
|
| 194 |
|
| 195 |
@router.get("/stats", dependencies=[Depends(verify_admin)])
|
| 196 |
async def get_global_stats():
|
| 197 |
-
with
|
| 198 |
-
total_keys =
|
| 199 |
-
|
| 200 |
-
pg_conn.commit()
|
| 201 |
-
with usage_tracker.tracker._get_conn() as conn:
|
| 202 |
total_requests = conn.execute(
|
| 203 |
"SELECT COUNT(*) FROM usage_log").fetchone()[0]
|
| 204 |
by_tier = conn.execute(
|
| 205 |
"SELECT tier, COUNT(*) as count FROM usage_log GROUP BY tier"
|
| 206 |
).fetchall()
|
| 207 |
-
month =
|
| 208 |
current_month_requests = conn.execute(
|
| 209 |
"SELECT SUM(count) FROM monthly_counts WHERE year_month = ?", (month,)
|
| 210 |
).fetchone()[0] or 0
|
|
@@ -214,74 +140,3 @@ async def get_global_stats():
|
|
| 214 |
"current_month_evaluations": current_month_requests,
|
| 215 |
"by_tier": [{"tier": row[0], "count": row[1]} for row in by_tier],
|
| 216 |
}
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
# ---------------------------------------------------------------------------
|
| 220 |
-
# Enterprise execution approvals (v4.3.4, opt-in -- see routes_governance.py's
|
| 221 |
-
# POST /intents/{id}/execute). Without these, the durable approval ledger has
|
| 222 |
-
# no way to actually be resolved through the API at all -- present but
|
| 223 |
-
# unusable. app.state.approval_store is None whenever ARF_ENABLE_EXECUTION
|
| 224 |
-
# is unset/false or arf_enterprise isn't installed (see main.py lifespan).
|
| 225 |
-
# ---------------------------------------------------------------------------
|
| 226 |
-
|
| 227 |
-
class ResolveApprovalRequest(BaseModel):
|
| 228 |
-
approved: bool
|
| 229 |
-
note: Optional[str] = None
|
| 230 |
-
|
| 231 |
-
|
| 232 |
-
def _require_approval_store(request: Request):
|
| 233 |
-
approval_store = getattr(request.app.state, "approval_store", None)
|
| 234 |
-
if approval_store is None:
|
| 235 |
-
raise HTTPException(
|
| 236 |
-
status_code=501,
|
| 237 |
-
detail="Enterprise execution approvals are not enabled on this deployment "
|
| 238 |
-
"(ARF_ENABLE_EXECUTION unset, or arf_enterprise is not installed)",
|
| 239 |
-
)
|
| 240 |
-
return approval_store
|
| 241 |
-
|
| 242 |
-
|
| 243 |
-
@router.get("/executions/pending", dependencies=[Depends(verify_admin)])
|
| 244 |
-
async def list_pending_executions(request: Request, limit: int = 100, offset: int = 0):
|
| 245 |
-
approval_store = _require_approval_store(request)
|
| 246 |
-
pending = approval_store.list_pending(limit=limit, offset=offset)
|
| 247 |
-
return {
|
| 248 |
-
"pending": [
|
| 249 |
-
{
|
| 250 |
-
"approval_id": r.id,
|
| 251 |
-
"decision_id": r.decision_id,
|
| 252 |
-
"intent_id": r.intent_id,
|
| 253 |
-
"level": r.level,
|
| 254 |
-
"approval_required": r.approval_required,
|
| 255 |
-
"requested_at": r.requested_at.isoformat(),
|
| 256 |
-
}
|
| 257 |
-
for r in pending
|
| 258 |
-
],
|
| 259 |
-
"total": len(pending),
|
| 260 |
-
}
|
| 261 |
-
|
| 262 |
-
|
| 263 |
-
@router.post("/executions/{approval_id}/resolve", dependencies=[Depends(verify_admin)])
|
| 264 |
-
async def resolve_execution_approval(
|
| 265 |
-
request: Request,
|
| 266 |
-
req: ResolveApprovalRequest,
|
| 267 |
-
approval_id: str = Path(..., description="The approval_id from POST /intents/{id}/execute's 202 response"),
|
| 268 |
-
):
|
| 269 |
-
# resolved_by is a fixed "admin" constant, not derived from admin_key in
|
| 270 |
-
# any way -- even a prefix of that secret has no business being
|
| 271 |
-
# persisted into a database row a GET endpoint can read back. Matches
|
| 272 |
-
# create_api_key's existing created_by="admin" pattern above: this
|
| 273 |
-
# codebase has one shared admin credential, not per-admin identity, so
|
| 274 |
-
# there's nothing more specific to record.
|
| 275 |
-
approval_store = _require_approval_store(request)
|
| 276 |
-
resolved = approval_store.resolve(
|
| 277 |
-
approval_id, approved=req.approved, resolved_by="admin", note=req.note
|
| 278 |
-
)
|
| 279 |
-
if not resolved:
|
| 280 |
-
raise HTTPException(
|
| 281 |
-
status_code=404,
|
| 282 |
-
detail="Approval not found or already resolved",
|
| 283 |
-
)
|
| 284 |
-
return {
|
| 285 |
-
"message": f"Approval {'approved' if req.approved else 'rejected'}",
|
| 286 |
-
"approval_id": approval_id,
|
| 287 |
-
}
|
|
|
|
| 2 |
Admin API endpoints for API key management and audit logs.
|
| 3 |
These endpoints should be protected (e.g., by an admin API key) in production.
|
| 4 |
"""
|
| 5 |
+
from fastapi import APIRouter, Depends, HTTPException, Query, Path, Body
|
| 6 |
from pydantic import BaseModel
|
| 7 |
from typing import Optional
|
| 8 |
from datetime import datetime
|
| 9 |
import os
|
| 10 |
import secrets
|
| 11 |
import uuid
|
| 12 |
+
from app.core.usage_tracker import tracker, Tier
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
|
| 14 |
router = APIRouter(prefix="/admin", tags=["admin"])
|
| 15 |
# Admin key must be supplied via environment; there is no default. Fail closed
|
|
|
|
| 27 |
|
| 28 |
class CreateKeyRequest(BaseModel):
|
| 29 |
tier: str
|
|
|
|
|
|
|
| 30 |
|
| 31 |
|
| 32 |
class UpdateTierRequest(BaseModel):
|
|
|
|
| 34 |
|
| 35 |
|
| 36 |
@router.post("/keys", dependencies=[Depends(verify_admin)])
|
| 37 |
+
async def create_api_key(req: CreateKeyRequest):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
if req.tier not in [t.value for t in Tier]:
|
| 39 |
raise HTTPException(
|
| 40 |
status_code=400, detail=f"Invalid tier. Must be one of {[t.value for t in Tier]}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 41 |
new_key = f"sk_live_{uuid.uuid4().hex[:24]}"
|
| 42 |
+
tier_enum = Tier(req.tier)
|
| 43 |
+
tracker.get_or_create_api_key(new_key, tier_enum)
|
| 44 |
+
return {"api_key": new_key, "tier": req.tier}
|
| 45 |
|
| 46 |
|
| 47 |
@router.get("/keys", dependencies=[Depends(verify_admin)])
|
| 48 |
async def list_api_keys(limit: int = 100, offset: int = 0):
|
| 49 |
+
with tracker._get_conn() as conn:
|
| 50 |
+
rows = conn.execute(
|
| 51 |
+
"SELECT key, tier, created_at, last_used_at, is_active FROM api_keys ORDER BY created_at DESC LIMIT ? OFFSET ?", # noqa: E501
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 52 |
(limit, offset)
|
| 53 |
+
).fetchall() # noqa: E501
|
| 54 |
+
keys = []
|
| 55 |
+
for row in rows:
|
| 56 |
+
month = tracker._get_month_key()
|
| 57 |
+
usage_row = conn.execute(
|
| 58 |
+
"SELECT count FROM monthly_counts WHERE api_key = ? AND year_month = ?",
|
| 59 |
+
(row["key"], month)
|
| 60 |
+
).fetchone()
|
| 61 |
+
usage = usage_row["count"] if usage_row else 0
|
| 62 |
+
keys.append(
|
| 63 |
+
{
|
| 64 |
+
"key": row["key"],
|
| 65 |
+
"tier": row["tier"],
|
| 66 |
+
"created_at": datetime.fromtimestamp(
|
| 67 |
+
row["created_at"]).isoformat(),
|
| 68 |
+
"last_used_at": datetime.fromtimestamp(
|
| 69 |
+
row["last_used_at"]).isoformat() if row["last_used_at"] else None,
|
| 70 |
+
"is_active": bool(
|
| 71 |
+
row["is_active"]),
|
| 72 |
+
"current_month_usage": usage,
|
| 73 |
+
})
|
| 74 |
return {"keys": keys, "total": len(keys)}
|
| 75 |
|
| 76 |
|
| 77 |
+
@router.patch("/keys/{api_key}/tier", dependencies=[Depends(verify_admin)])
|
| 78 |
async def update_key_tier(
|
| 79 |
+
api_key: str = Path(..., description="The API key to update"),
|
| 80 |
req: UpdateTierRequest = Body(...),
|
| 81 |
):
|
| 82 |
if req.tier not in [t.value for t in Tier]:
|
| 83 |
raise HTTPException(
|
| 84 |
status_code=400, detail=f"Invalid tier. Must be one of {[t.value for t in Tier]}")
|
| 85 |
+
with tracker._get_conn() as conn:
|
| 86 |
+
row = conn.execute(
|
| 87 |
+
"SELECT key FROM api_keys WHERE key = ?", (api_key,)).fetchone()
|
| 88 |
if not row:
|
|
|
|
| 89 |
raise HTTPException(status_code=404, detail="API key not found")
|
| 90 |
+
conn.execute("UPDATE api_keys SET tier = ? WHERE key = ?",
|
| 91 |
+
(req.tier, api_key))
|
| 92 |
conn.commit()
|
| 93 |
return {"message": f"Tier updated to {req.tier}"}
|
| 94 |
|
| 95 |
|
| 96 |
+
@router.delete("/keys/{api_key}", dependencies=[Depends(verify_admin)])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 97 |
async def deactivate_api_key(
|
| 98 |
+
api_key: str = Path(..., description="The API key to deactivate")):
|
| 99 |
+
with tracker._get_conn() as conn:
|
| 100 |
+
row = conn.execute(
|
| 101 |
+
"SELECT key FROM api_keys WHERE key = ?", (api_key,)).fetchone()
|
| 102 |
if not row:
|
|
|
|
| 103 |
raise HTTPException(status_code=404, detail="API key not found")
|
| 104 |
+
conn.execute(
|
| 105 |
+
"UPDATE api_keys SET is_active = 0 WHERE key = ?", (api_key,))
|
| 106 |
conn.commit()
|
| 107 |
return {"message": "API key deactivated"}
|
| 108 |
|
|
|
|
| 116 |
):
|
| 117 |
start = datetime.fromisoformat(start_date) if start_date else None
|
| 118 |
end = datetime.fromisoformat(end_date) if end_date else None
|
| 119 |
+
logs = tracker.get_audit_logs(api_key, start, end, limit)
|
| 120 |
return {"api_key": api_key, "logs": logs}
|
| 121 |
|
| 122 |
|
| 123 |
@router.get("/stats", dependencies=[Depends(verify_admin)])
|
| 124 |
async def get_global_stats():
|
| 125 |
+
with tracker._get_conn() as conn:
|
| 126 |
+
total_keys = conn.execute(
|
| 127 |
+
"SELECT COUNT(*) FROM api_keys WHERE is_active = 1").fetchone()[0]
|
|
|
|
|
|
|
| 128 |
total_requests = conn.execute(
|
| 129 |
"SELECT COUNT(*) FROM usage_log").fetchone()[0]
|
| 130 |
by_tier = conn.execute(
|
| 131 |
"SELECT tier, COUNT(*) as count FROM usage_log GROUP BY tier"
|
| 132 |
).fetchall()
|
| 133 |
+
month = tracker._get_month_key()
|
| 134 |
current_month_requests = conn.execute(
|
| 135 |
"SELECT SUM(count) FROM monthly_counts WHERE year_month = ?", (month,)
|
| 136 |
).fetchone()[0] or 0
|
|
|
|
| 140 |
"current_month_evaluations": current_month_requests,
|
| 141 |
"by_tier": [{"tier": row[0], "count": row[1]} for row in by_tier],
|
| 142 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
app/api/routes_governance.py
CHANGED
|
@@ -24,12 +24,10 @@ intents and healing decisions. It integrates:
|
|
| 24 |
|
| 25 |
from fastapi import APIRouter, Depends, HTTPException, Request, BackgroundTasks, Header
|
| 26 |
from fastapi.encoders import jsonable_encoder
|
| 27 |
-
from fastapi.responses import JSONResponse
|
| 28 |
from sqlalchemy.orm import Session
|
| 29 |
from pydantic import BaseModel
|
| 30 |
import uuid
|
| 31 |
import logging
|
| 32 |
-
import os
|
| 33 |
import time
|
| 34 |
import datetime
|
| 35 |
from typing import Optional, Dict, Any, List
|
|
@@ -42,7 +40,7 @@ from app.services.outcome_service import record_outcome
|
|
| 42 |
from app.api.deps import get_db, get_skill_registry, verify_internal_key # <-- v4.3.2
|
| 43 |
from app.database.session import SessionLocal
|
| 44 |
from app.core.usage_tracker import enforce_quota # <-- tenant resolution
|
| 45 |
-
from app.database.models_intents import DecisionAuditLogDB
|
| 46 |
from agentic_reliability_framework.core.models.event import ReliabilityEvent
|
| 47 |
from agentic_reliability_framework.core.governance.policies import (
|
| 48 |
PolicyEvaluator,
|
|
@@ -69,57 +67,6 @@ except ImportError:
|
|
| 69 |
RUST_AVAILABLE = False
|
| 70 |
ExecutionLadder = None
|
| 71 |
|
| 72 |
-
# ===== ENTERPRISE EXECUTOR (optional) =====
|
| 73 |
-
# `arf_enterprise` is not in requirements.txt -- it's a proprietary,
|
| 74 |
-
# private-repo package, and unlike agentic_reliability_framework/
|
| 75 |
-
# arf-pricing-calculator (plain git+https URLs with no visible credential
|
| 76 |
-
# setup in the Dockerfile) I can't confirm Render's build can actually
|
| 77 |
-
# clone it. Stays fully optional/try-except-guarded, same shape as
|
| 78 |
-
# RUST_AVAILABLE above, deliberately not added as a hard dependency this
|
| 79 |
-
# session -- see POST /intents/{id}/execute below and .env.example for what
|
| 80 |
-
# an operator needs to do to actually turn this on.
|
| 81 |
-
try:
|
| 82 |
-
from arf_enterprise.executor import EnterpriseExecutor
|
| 83 |
-
from arf_enterprise.actuators.fake import FakeCloudActuator
|
| 84 |
-
from arf_enterprise.config import EnterpriseConfig
|
| 85 |
-
from arf_enterprise.store import ApprovalStore, PostgresStore
|
| 86 |
-
from arf_enterprise.exceptions import (
|
| 87 |
-
ExecutionError as EnterpriseExecutionError,
|
| 88 |
-
PendingApprovalError,
|
| 89 |
-
SafetyError as EnterpriseSafetyError,
|
| 90 |
-
)
|
| 91 |
-
ENTERPRISE_EXECUTOR_AVAILABLE = True
|
| 92 |
-
except ImportError:
|
| 93 |
-
ENTERPRISE_EXECUTOR_AVAILABLE = False
|
| 94 |
-
EnterpriseExecutor = None
|
| 95 |
-
FakeCloudActuator = None
|
| 96 |
-
EnterpriseConfig = None
|
| 97 |
-
ApprovalStore = None
|
| 98 |
-
PostgresStore = None
|
| 99 |
-
EnterpriseExecutionError = None
|
| 100 |
-
PendingApprovalError = None
|
| 101 |
-
EnterpriseSafetyError = None
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
def _trusted_signing_keys() -> List[str]:
|
| 105 |
-
"""Parse ARF_TRUSTED_SIGNING_KEYS into a list of hex fingerprints.
|
| 106 |
-
|
| 107 |
-
Comma-separated, matching EnterpriseConfig.from_env's own parsing.
|
| 108 |
-
Splitting matters: the variable holds N fingerprints, and passing the
|
| 109 |
-
raw string as a single-element list would register the literal
|
| 110 |
-
"abc,def" as one key -- so neither real key would be trusted, and an
|
| 111 |
-
unset variable would register the empty string as trusted rather than
|
| 112 |
-
trusting nothing.
|
| 113 |
-
|
| 114 |
-
An empty result is the correct fail-closed state: the ladder then
|
| 115 |
-
rejects every signed intent, which is loud and safe.
|
| 116 |
-
"""
|
| 117 |
-
raw = os.getenv("ARF_TRUSTED_SIGNING_KEYS", "")
|
| 118 |
-
return [k.strip() for k in raw.split(",") if k.strip()]
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
ARF_ENABLE_EXECUTION = os.getenv("ARF_ENABLE_EXECUTION", "false").lower() == "true"
|
| 122 |
-
|
| 123 |
# ===== OPEN TELEMETRY =====
|
| 124 |
try:
|
| 125 |
from opentelemetry import trace
|
|
@@ -146,22 +93,6 @@ class OutcomeRequest(BaseModel):
|
|
| 146 |
skill_version: Optional[int] = None
|
| 147 |
|
| 148 |
|
| 149 |
-
class ExecuteIntentRequest(BaseModel):
|
| 150 |
-
"""The client already received `healing_intent` in /intents/evaluate's
|
| 151 |
-
response -- resubmitting it here (rather than this endpoint trying to
|
| 152 |
-
reconstruct a signed, action/component/parameters-bearing intent from
|
| 153 |
-
IntentDB's stored oss_payload, which doesn't carry the signature) is
|
| 154 |
-
the cheapest correct design; IntentDB is used only to confirm the
|
| 155 |
-
intent exists and belongs to the caller's tenant."""
|
| 156 |
-
healing_intent: Dict[str, Any]
|
| 157 |
-
human_approved: bool = False
|
| 158 |
-
admin_approved: bool = False
|
| 159 |
-
# v4.3.1: optional skill provenance, forwarded to record_outcome the
|
| 160 |
-
# same way OutcomeRequest already does.
|
| 161 |
-
skill_id: Optional[str] = None
|
| 162 |
-
skill_version: Optional[int] = None
|
| 163 |
-
|
| 164 |
-
|
| 165 |
class HealingDecisionRequest(BaseModel):
|
| 166 |
event: ReliabilityEvent
|
| 167 |
# v4.3.1: optional skill context for Bayesian utility
|
|
@@ -307,7 +238,7 @@ async def evaluate_intent_endpoint(
|
|
| 307 |
|
| 308 |
record = UsageRecord(
|
| 309 |
api_key=api_key,
|
| 310 |
-
tier=
|
| 311 |
timestamp=start_time,
|
| 312 |
endpoint="/api/v1/intents/evaluate",
|
| 313 |
request_body=intent_req.model_dump(),
|
|
@@ -435,144 +366,7 @@ async def evaluate_intent_endpoint(
|
|
| 435 |
span.set_status(Status(StatusCode.ERROR, error_msg))
|
| 436 |
span.record_exception(e)
|
| 437 |
span.end()
|
| 438 |
-
raise HTTPException(status_code=500, detail=
|
| 439 |
-
|
| 440 |
-
|
| 441 |
-
# --------------------------------------------------------------------------
|
| 442 |
-
# Endpoint: execute a previously evaluated intent (v4.3.4, opt-in)
|
| 443 |
-
# --------------------------------------------------------------------------
|
| 444 |
-
@router.post("/intents/{deterministic_id}/execute")
|
| 445 |
-
async def execute_intent_endpoint(
|
| 446 |
-
request: Request,
|
| 447 |
-
deterministic_id: str,
|
| 448 |
-
exec_req: ExecuteIntentRequest,
|
| 449 |
-
db: Session = Depends(get_db),
|
| 450 |
-
skill_registry=Depends(get_skill_registry),
|
| 451 |
-
quota: dict = Depends(enforce_quota), # tenant resolved from authenticated API key
|
| 452 |
-
):
|
| 453 |
-
"""
|
| 454 |
-
Execute a previously evaluated healing intent through
|
| 455 |
-
arf_enterprise.EnterpriseExecutor -- gate re-check (Rust ladder),
|
| 456 |
-
optional durable approval, actuation, and independent read-back
|
| 457 |
-
verification, whose *verified* result (not a client self-report) feeds
|
| 458 |
-
record_outcome the same way /intents/outcome already does.
|
| 459 |
-
|
| 460 |
-
Off by default (ARF_ENABLE_EXECUTION unset or false) and a 501 if the
|
| 461 |
-
arf_enterprise package isn't importable -- this is new, opt-in
|
| 462 |
-
capability, not a replacement for the existing advisory-only flow.
|
| 463 |
-
Uses FakeCloudActuator regardless of what's configured elsewhere:
|
| 464 |
-
selecting a real cloud actuator (which provider, which credentials) is
|
| 465 |
-
a deliberate later step for whoever actually deploys against real
|
| 466 |
-
infrastructure, not something this endpoint defaults into.
|
| 467 |
-
"""
|
| 468 |
-
if not ENTERPRISE_EXECUTOR_AVAILABLE:
|
| 469 |
-
raise HTTPException(
|
| 470 |
-
status_code=501,
|
| 471 |
-
detail="arf_enterprise is not installed on this deployment; execution is unavailable",
|
| 472 |
-
)
|
| 473 |
-
if not ARF_ENABLE_EXECUTION:
|
| 474 |
-
raise HTTPException(
|
| 475 |
-
status_code=501,
|
| 476 |
-
detail="Execution is not enabled (set ARF_ENABLE_EXECUTION=true to opt in)",
|
| 477 |
-
)
|
| 478 |
-
|
| 479 |
-
tenant_id = quota["tenant_id"]
|
| 480 |
-
|
| 481 |
-
# Existence + tenant-ownership check only -- the healing_intent to
|
| 482 |
-
# execute comes from the request body (see ExecuteIntentRequest), not
|
| 483 |
-
# reconstructed from what's stored here.
|
| 484 |
-
intent_row = db.query(IntentDB).filter(
|
| 485 |
-
IntentDB.deterministic_id == deterministic_id,
|
| 486 |
-
IntentDB.tenant_id == tenant_id,
|
| 487 |
-
).one_or_none()
|
| 488 |
-
if not intent_row:
|
| 489 |
-
raise HTTPException(status_code=404, detail=f"Intent not found: {deterministic_id}")
|
| 490 |
-
|
| 491 |
-
risk_engine = request.app.state.risk_engine
|
| 492 |
-
|
| 493 |
-
def _on_verified_outcome(intent: Dict[str, Any], verified_success: bool, context: Dict[str, Any]) -> None:
|
| 494 |
-
try:
|
| 495 |
-
record_outcome(
|
| 496 |
-
db=db,
|
| 497 |
-
tenant_id=tenant_id,
|
| 498 |
-
deterministic_id=deterministic_id,
|
| 499 |
-
success=verified_success,
|
| 500 |
-
recorded_by="enterprise_executor",
|
| 501 |
-
notes=f"Auto-recorded from verified execution. Observed: {context.get('observed')}",
|
| 502 |
-
risk_engine=risk_engine,
|
| 503 |
-
skill_id=exec_req.skill_id,
|
| 504 |
-
skill_version=exec_req.skill_version,
|
| 505 |
-
skill_registry=skill_registry,
|
| 506 |
-
)
|
| 507 |
-
except Exception:
|
| 508 |
-
# Execution already happened by the time this fires -- a failure
|
| 509 |
-
# here must not be raised back through EnterpriseExecutor.execute()
|
| 510 |
-
# (which would misreport a real actuation as failed). Logged so
|
| 511 |
-
# the risk-engine-not-updated case is visible, not silent.
|
| 512 |
-
logger.exception(
|
| 513 |
-
"Failed to record verified outcome for intent %s after execution",
|
| 514 |
-
deterministic_id,
|
| 515 |
-
)
|
| 516 |
-
|
| 517 |
-
# Singleton initialised once at startup (main.py lifespan) rather than
|
| 518 |
-
# constructed fresh per request -- None here means either
|
| 519 |
-
# ARF_ENABLE_EXECUTION wasn't set (unreachable, checked above) or the
|
| 520 |
-
# ledger failed to initialise at startup, in which case boolean-trust
|
| 521 |
-
# mode is the documented fallback (see EnterpriseExecutor.execute).
|
| 522 |
-
approval_store = getattr(request.app.state, "approval_store", None)
|
| 523 |
-
|
| 524 |
-
# A narrow config carrying only the trust anchor. Deliberately NOT
|
| 525 |
-
# EnterpriseConfig.from_env(), which would also pick up ARF_CLOUD,
|
| 526 |
-
# ARF_MAX_BLAST_RADIUS, ARF_ENFORCE_BUSINESS_HOURS and the audit/safety
|
| 527 |
-
# toggles -- turning on guardrails and audit logging as a side effect of
|
| 528 |
-
# enabling signing is a behaviour change nobody asked for. Everything
|
| 529 |
-
# other than the trusted keys stays on today's defaults.
|
| 530 |
-
#
|
| 531 |
-
# Without this the executor got EnterpriseConfig() with an empty
|
| 532 |
-
# trusted_signing_keys, so the ladder trusted no keys and rejected every
|
| 533 |
-
# signed intent as "Untrusted signing key" -- the whole execute path was
|
| 534 |
-
# unreachable regardless of ARF_ENABLE_EXECUTION.
|
| 535 |
-
trusted_keys = _trusted_signing_keys()
|
| 536 |
-
if not trusted_keys:
|
| 537 |
-
logger.warning(
|
| 538 |
-
"ARF_TRUSTED_SIGNING_KEYS is unset or empty; the execution ladder "
|
| 539 |
-
"trusts no signing keys and will reject every signed intent. Set it "
|
| 540 |
-
"to the hex fingerprint(s) of the key(s) permitted to sign intents."
|
| 541 |
-
)
|
| 542 |
-
|
| 543 |
-
executor = EnterpriseExecutor(
|
| 544 |
-
config=EnterpriseConfig(trusted_signing_keys=trusted_keys),
|
| 545 |
-
actuator=FakeCloudActuator(),
|
| 546 |
-
approval_store=approval_store,
|
| 547 |
-
on_verified_outcome=_on_verified_outcome,
|
| 548 |
-
)
|
| 549 |
-
|
| 550 |
-
try:
|
| 551 |
-
result = await executor.execute(
|
| 552 |
-
exec_req.healing_intent,
|
| 553 |
-
human_approved=exec_req.human_approved,
|
| 554 |
-
admin_approved=exec_req.admin_approved,
|
| 555 |
-
)
|
| 556 |
-
return result
|
| 557 |
-
except PendingApprovalError as e:
|
| 558 |
-
return JSONResponse(
|
| 559 |
-
status_code=202,
|
| 560 |
-
content={
|
| 561 |
-
"status": "pending_approval",
|
| 562 |
-
"approval_id": e.approval_id,
|
| 563 |
-
"level": e.level,
|
| 564 |
-
"approval_required": e.approval_required,
|
| 565 |
-
"detail": str(e),
|
| 566 |
-
},
|
| 567 |
-
)
|
| 568 |
-
except (EnterpriseExecutionError, EnterpriseSafetyError) as e:
|
| 569 |
-
# A legitimate "did not execute" outcome (ladder denial, safety
|
| 570 |
-
# constraint, verification mismatch) -- not a server bug, so not a
|
| 571 |
-
# 5xx.
|
| 572 |
-
raise HTTPException(status_code=422, detail=str(e))
|
| 573 |
-
except Exception:
|
| 574 |
-
logger.exception("Unexpected error in execute_intent_endpoint")
|
| 575 |
-
raise HTTPException(status_code=500, detail="Internal server error")
|
| 576 |
|
| 577 |
|
| 578 |
# --------------------------------------------------------------------------
|
|
@@ -619,9 +413,8 @@ async def record_outcome_endpoint(
|
|
| 619 |
logger.warning(f"Failed to update pricing buffer for intent {outcome.deterministic_id}: {e}")
|
| 620 |
|
| 621 |
return {"message": "Outcome recorded", "outcome_id": outcome_record.id}
|
| 622 |
-
except Exception:
|
| 623 |
-
|
| 624 |
-
raise HTTPException(status_code=500, detail="Internal server error")
|
| 625 |
|
| 626 |
|
| 627 |
# --------------------------------------------------------------------------
|
|
@@ -661,7 +454,7 @@ async def evaluate_healing_decision_endpoint(
|
|
| 661 |
|
| 662 |
record = UsageRecord(
|
| 663 |
api_key=api_key,
|
| 664 |
-
tier=
|
| 665 |
timestamp=start_time,
|
| 666 |
endpoint="/api/v1/healing/evaluate",
|
| 667 |
request_body=decision_req.model_dump(),
|
|
@@ -760,4 +553,4 @@ async def evaluate_healing_decision_endpoint(
|
|
| 760 |
span.set_status(Status(StatusCode.ERROR, error_msg))
|
| 761 |
span.record_exception(e)
|
| 762 |
span.end()
|
| 763 |
-
raise HTTPException(status_code=500, detail=
|
|
|
|
| 24 |
|
| 25 |
from fastapi import APIRouter, Depends, HTTPException, Request, BackgroundTasks, Header
|
| 26 |
from fastapi.encoders import jsonable_encoder
|
|
|
|
| 27 |
from sqlalchemy.orm import Session
|
| 28 |
from pydantic import BaseModel
|
| 29 |
import uuid
|
| 30 |
import logging
|
|
|
|
| 31 |
import time
|
| 32 |
import datetime
|
| 33 |
from typing import Optional, Dict, Any, List
|
|
|
|
| 40 |
from app.api.deps import get_db, get_skill_registry, verify_internal_key # <-- v4.3.2
|
| 41 |
from app.database.session import SessionLocal
|
| 42 |
from app.core.usage_tracker import enforce_quota # <-- tenant resolution
|
| 43 |
+
from app.database.models_intents import DecisionAuditLogDB
|
| 44 |
from agentic_reliability_framework.core.models.event import ReliabilityEvent
|
| 45 |
from agentic_reliability_framework.core.governance.policies import (
|
| 46 |
PolicyEvaluator,
|
|
|
|
| 67 |
RUST_AVAILABLE = False
|
| 68 |
ExecutionLadder = None
|
| 69 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 70 |
# ===== OPEN TELEMETRY =====
|
| 71 |
try:
|
| 72 |
from opentelemetry import trace
|
|
|
|
| 93 |
skill_version: Optional[int] = None
|
| 94 |
|
| 95 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 96 |
class HealingDecisionRequest(BaseModel):
|
| 97 |
event: ReliabilityEvent
|
| 98 |
# v4.3.1: optional skill context for Bayesian utility
|
|
|
|
| 238 |
|
| 239 |
record = UsageRecord(
|
| 240 |
api_key=api_key,
|
| 241 |
+
tier=None,
|
| 242 |
timestamp=start_time,
|
| 243 |
endpoint="/api/v1/intents/evaluate",
|
| 244 |
request_body=intent_req.model_dump(),
|
|
|
|
| 366 |
span.set_status(Status(StatusCode.ERROR, error_msg))
|
| 367 |
span.record_exception(e)
|
| 368 |
span.end()
|
| 369 |
+
raise HTTPException(status_code=500, detail=error_msg)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 370 |
|
| 371 |
|
| 372 |
# --------------------------------------------------------------------------
|
|
|
|
| 413 |
logger.warning(f"Failed to update pricing buffer for intent {outcome.deterministic_id}: {e}")
|
| 414 |
|
| 415 |
return {"message": "Outcome recorded", "outcome_id": outcome_record.id}
|
| 416 |
+
except Exception as e:
|
| 417 |
+
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
| 418 |
|
| 419 |
|
| 420 |
# --------------------------------------------------------------------------
|
|
|
|
| 454 |
|
| 455 |
record = UsageRecord(
|
| 456 |
api_key=api_key,
|
| 457 |
+
tier=None,
|
| 458 |
timestamp=start_time,
|
| 459 |
endpoint="/api/v1/healing/evaluate",
|
| 460 |
request_body=decision_req.model_dump(),
|
|
|
|
| 553 |
span.set_status(Status(StatusCode.ERROR, error_msg))
|
| 554 |
span.record_exception(e)
|
| 555 |
span.end()
|
| 556 |
+
raise HTTPException(status_code=500, detail=error_msg)
|
app/api/routes_history.py
CHANGED
|
@@ -1,10 +1,9 @@
|
|
| 1 |
-
from fastapi import APIRouter
|
| 2 |
-
from app.api.deps import verify_internal_key
|
| 3 |
from app.core.storage import incident_history
|
| 4 |
|
| 5 |
-
router = APIRouter(
|
| 6 |
|
| 7 |
|
| 8 |
@router.get("/history")
|
| 9 |
async def get_history():
|
| 10 |
-
return {"incidents":
|
|
|
|
| 1 |
+
from fastapi import APIRouter
|
|
|
|
| 2 |
from app.core.storage import incident_history
|
| 3 |
|
| 4 |
+
router = APIRouter()
|
| 5 |
|
| 6 |
|
| 7 |
@router.get("/history")
|
| 8 |
async def get_history():
|
| 9 |
+
return {"incidents": incident_history}
|
app/api/routes_incidents.py
CHANGED
|
@@ -30,21 +30,23 @@ from agentic_reliability_framework.core.models.event import (
|
|
| 30 |
ReliabilityEvent,
|
| 31 |
)
|
| 32 |
|
| 33 |
-
from app.api.deps import verify_internal_key
|
| 34 |
from app.causal_explainer import CausalExplainer
|
| 35 |
-
from app.core.
|
| 36 |
-
from app.core import usage_tracker
|
| 37 |
-
from app.core.usage_tracker import UsageRecord, enforce_quota
|
| 38 |
|
| 39 |
logger = logging.getLogger(__name__)
|
| 40 |
|
| 41 |
router = APIRouter()
|
| 42 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 43 |
|
| 44 |
# ---------------------------------------------------------------------------
|
| 45 |
# POST /api/v1/report_incident
|
| 46 |
# ---------------------------------------------------------------------------
|
| 47 |
-
@router.post("/report_incident"
|
| 48 |
async def report_incident(event: ReliabilityEvent) -> dict[str, str]:
|
| 49 |
"""
|
| 50 |
Record a ``ReliabilityEvent`` in the in‑memory incident history.
|
|
@@ -52,10 +54,7 @@ async def report_incident(event: ReliabilityEvent) -> dict[str, str]:
|
|
| 52 |
This endpoint is used by internal monitoring tools to feed incident
|
| 53 |
data into the causal explainer and downstream analysis. The event
|
| 54 |
is stored as a JSON‑safe dictionary and is **not** persisted across
|
| 55 |
-
API restarts.
|
| 56 |
-
data-bearing route in this API requires -- previously this endpoint had
|
| 57 |
-
no auth dependency at all, so anyone could write into the incident
|
| 58 |
-
history that feeds the causal explainer and ``GET /history``.
|
| 59 |
|
| 60 |
Parameters
|
| 61 |
----------
|
|
@@ -228,7 +227,7 @@ async def evaluate_incident(
|
|
| 228 |
# ------------------------------------------------------------------
|
| 229 |
# Asynchronous usage logging
|
| 230 |
# ------------------------------------------------------------------
|
| 231 |
-
if
|
| 232 |
record = UsageRecord(
|
| 233 |
api_key=api_key,
|
| 234 |
tier=tier,
|
|
@@ -238,7 +237,7 @@ async def evaluate_incident(
|
|
| 238 |
response=response_data,
|
| 239 |
processing_ms=(time.time() - start_time) * 1000,
|
| 240 |
)
|
| 241 |
-
await
|
| 242 |
|
| 243 |
logger.warning(
|
| 244 |
"Deprecated endpoint /v1/incidents/evaluate called by key %s",
|
|
@@ -250,8 +249,7 @@ async def evaluate_incident(
|
|
| 250 |
raise
|
| 251 |
except Exception as exc:
|
| 252 |
error_msg = str(exc)
|
| 253 |
-
|
| 254 |
-
if usage_tracker.tracker:
|
| 255 |
record = UsageRecord(
|
| 256 |
api_key=api_key,
|
| 257 |
tier=tier,
|
|
@@ -261,5 +259,5 @@ async def evaluate_incident(
|
|
| 261 |
error=error_msg,
|
| 262 |
processing_ms=(time.time() - start_time) * 1000,
|
| 263 |
)
|
| 264 |
-
await
|
| 265 |
-
raise HTTPException(status_code=500, detail=
|
|
|
|
| 30 |
ReliabilityEvent,
|
| 31 |
)
|
| 32 |
|
|
|
|
| 33 |
from app.causal_explainer import CausalExplainer
|
| 34 |
+
from app.core.usage_tracker import UsageRecord, enforce_quota, tracker
|
|
|
|
|
|
|
| 35 |
|
| 36 |
logger = logging.getLogger(__name__)
|
| 37 |
|
| 38 |
router = APIRouter()
|
| 39 |
|
| 40 |
+
# ---------------------------------------------------------------------------
|
| 41 |
+
# In‑memory incident store (for auditing / debugging only)
|
| 42 |
+
# ---------------------------------------------------------------------------
|
| 43 |
+
incident_history: list[dict] = []
|
| 44 |
+
|
| 45 |
|
| 46 |
# ---------------------------------------------------------------------------
|
| 47 |
# POST /api/v1/report_incident
|
| 48 |
# ---------------------------------------------------------------------------
|
| 49 |
+
@router.post("/report_incident")
|
| 50 |
async def report_incident(event: ReliabilityEvent) -> dict[str, str]:
|
| 51 |
"""
|
| 52 |
Record a ``ReliabilityEvent`` in the in‑memory incident history.
|
|
|
|
| 54 |
This endpoint is used by internal monitoring tools to feed incident
|
| 55 |
data into the causal explainer and downstream analysis. The event
|
| 56 |
is stored as a JSON‑safe dictionary and is **not** persisted across
|
| 57 |
+
API restarts.
|
|
|
|
|
|
|
|
|
|
| 58 |
|
| 59 |
Parameters
|
| 60 |
----------
|
|
|
|
| 227 |
# ------------------------------------------------------------------
|
| 228 |
# Asynchronous usage logging
|
| 229 |
# ------------------------------------------------------------------
|
| 230 |
+
if tracker:
|
| 231 |
record = UsageRecord(
|
| 232 |
api_key=api_key,
|
| 233 |
tier=tier,
|
|
|
|
| 237 |
response=response_data,
|
| 238 |
processing_ms=(time.time() - start_time) * 1000,
|
| 239 |
)
|
| 240 |
+
await tracker.increment_usage_async(record, background_tasks)
|
| 241 |
|
| 242 |
logger.warning(
|
| 243 |
"Deprecated endpoint /v1/incidents/evaluate called by key %s",
|
|
|
|
| 249 |
raise
|
| 250 |
except Exception as exc:
|
| 251 |
error_msg = str(exc)
|
| 252 |
+
if tracker:
|
|
|
|
| 253 |
record = UsageRecord(
|
| 254 |
api_key=api_key,
|
| 255 |
tier=tier,
|
|
|
|
| 259 |
error=error_msg,
|
| 260 |
processing_ms=(time.time() - start_time) * 1000,
|
| 261 |
)
|
| 262 |
+
await tracker.increment_usage_async(record, background_tasks)
|
| 263 |
+
raise HTTPException(status_code=500, detail=error_msg)
|
app/api/routes_intents.py
CHANGED
|
@@ -1,13 +1,8 @@
|
|
| 1 |
-
import
|
| 2 |
-
|
| 3 |
-
from fastapi import APIRouter, Depends, HTTPException
|
| 4 |
-
from app.api.deps import verify_internal_key
|
| 5 |
from app.models.intent_models import IntentSimulation, IntentSimulationResponse
|
| 6 |
from app.services.intent_service import simulate_intent
|
| 7 |
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
router = APIRouter(dependencies=[Depends(verify_internal_key)])
|
| 11 |
|
| 12 |
|
| 13 |
@router.post("/simulate_intent", response_model=IntentSimulationResponse)
|
|
@@ -15,6 +10,5 @@ async def simulate_intent_endpoint(intent: IntentSimulation):
|
|
| 15 |
try:
|
| 16 |
result = simulate_intent(intent)
|
| 17 |
return IntentSimulationResponse(**result)
|
| 18 |
-
except Exception:
|
| 19 |
-
|
| 20 |
-
raise HTTPException(status_code=500, detail="Internal server error")
|
|
|
|
| 1 |
+
from fastapi import APIRouter, HTTPException
|
|
|
|
|
|
|
|
|
|
| 2 |
from app.models.intent_models import IntentSimulation, IntentSimulationResponse
|
| 3 |
from app.services.intent_service import simulate_intent
|
| 4 |
|
| 5 |
+
router = APIRouter()
|
|
|
|
|
|
|
| 6 |
|
| 7 |
|
| 8 |
@router.post("/simulate_intent", response_model=IntentSimulationResponse)
|
|
|
|
| 10 |
try:
|
| 11 |
result = simulate_intent(intent)
|
| 12 |
return IntentSimulationResponse(**result)
|
| 13 |
+
except Exception as e:
|
| 14 |
+
raise HTTPException(status_code=500, detail=str(e))
|
|
|
app/api/routes_memory.py
CHANGED
|
@@ -1,7 +1,6 @@
|
|
| 1 |
-
from fastapi import APIRouter,
|
| 2 |
-
from app.api.deps import verify_internal_key
|
| 3 |
|
| 4 |
-
router = APIRouter(
|
| 5 |
|
| 6 |
|
| 7 |
@router.get("/stats")
|
|
|
|
| 1 |
+
from fastapi import APIRouter, Request
|
|
|
|
| 2 |
|
| 3 |
+
router = APIRouter()
|
| 4 |
|
| 5 |
|
| 6 |
@router.get("/stats")
|
app/api/routes_onchain.py
DELETED
|
@@ -1,149 +0,0 @@
|
|
| 1 |
-
"""Routes for on-chain attestation rationale.
|
| 2 |
-
|
| 3 |
-
`RiskAttestationRegistry` (arf-onchain) anchors only a `rationale_hash` --
|
| 4 |
-
never the reasoning itself, to keep operational detail about a customer's
|
| 5 |
-
infrastructure off a public chain. These endpoints are the off-chain half:
|
| 6 |
-
a reference risk evaluator (`arf_enterprise.onchain.evaluator`) persists the
|
| 7 |
-
plaintext here immediately after signing an attestation, keyed by the same
|
| 8 |
-
hash it put on-chain, and an auditor who reads a `DecisionAnchored` or
|
| 9 |
-
`AttestationIssued` event can fetch the reasoning behind it here.
|
| 10 |
-
|
| 11 |
-
Internal-key gated like `routes_governance.py`: this is a service-to-service
|
| 12 |
-
surface for the evaluator process and for auditor tooling, not a
|
| 13 |
-
tenant-scoped customer endpoint -- see `OnchainRationaleDB`'s docstring for
|
| 14 |
-
why there is no tenant_id to enforce here.
|
| 15 |
-
"""
|
| 16 |
-
|
| 17 |
-
import logging
|
| 18 |
-
|
| 19 |
-
from fastapi import APIRouter, Depends, HTTPException, Response
|
| 20 |
-
from pydantic import BaseModel, field_validator
|
| 21 |
-
from sqlalchemy.orm import Session
|
| 22 |
-
|
| 23 |
-
from app.api.deps import get_db, verify_internal_key
|
| 24 |
-
from app.database.models_onchain import OnchainRationaleDB
|
| 25 |
-
|
| 26 |
-
logger = logging.getLogger(__name__)
|
| 27 |
-
|
| 28 |
-
router = APIRouter(dependencies=[Depends(verify_internal_key)])
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
def _validate_hex_hash(value: str) -> str:
|
| 32 |
-
text = value.strip()
|
| 33 |
-
if not text.startswith("0x") or len(text) != 66:
|
| 34 |
-
raise ValueError("rationale_hash must be a 0x-prefixed 32-byte hex string")
|
| 35 |
-
try:
|
| 36 |
-
int(text, 16)
|
| 37 |
-
except ValueError:
|
| 38 |
-
raise ValueError("rationale_hash is not valid hex") from None
|
| 39 |
-
return text.lower()
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
class RationaleRequest(BaseModel):
|
| 43 |
-
rationale_hash: str
|
| 44 |
-
rationale: str
|
| 45 |
-
agent_address: str | None = None
|
| 46 |
-
evaluator_address: str | None = None
|
| 47 |
-
|
| 48 |
-
@field_validator("rationale_hash")
|
| 49 |
-
@classmethod
|
| 50 |
-
def _validate_hash(cls, value: str) -> str:
|
| 51 |
-
return _validate_hex_hash(value)
|
| 52 |
-
|
| 53 |
-
@field_validator("rationale")
|
| 54 |
-
@classmethod
|
| 55 |
-
def _validate_rationale(cls, value: str) -> str:
|
| 56 |
-
if not value.strip():
|
| 57 |
-
raise ValueError("rationale must not be empty")
|
| 58 |
-
return value
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
class RationaleResponse(BaseModel):
|
| 62 |
-
rationale_hash: str
|
| 63 |
-
rationale: str
|
| 64 |
-
agent_address: str | None
|
| 65 |
-
evaluator_address: str | None
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
@router.post("/onchain/rationale", status_code=201)
|
| 69 |
-
async def persist_rationale(
|
| 70 |
-
req: RationaleRequest,
|
| 71 |
-
response: Response,
|
| 72 |
-
db: Session = Depends(get_db),
|
| 73 |
-
):
|
| 74 |
-
"""Store the plaintext behind an anchored `rationale_hash`.
|
| 75 |
-
|
| 76 |
-
Idempotent on `rationale_hash`: signing the same decision twice (a
|
| 77 |
-
retry after a network error, for instance) posts the same hash and
|
| 78 |
-
text, so the second call is a no-op rather than a uniqueness-constraint
|
| 79 |
-
error. A *different* text arriving for a hash already on record is
|
| 80 |
-
refused -- that would mean either hash collision or a caller bug, and
|
| 81 |
-
silently overwriting an anchored record's preimage is the one thing
|
| 82 |
-
this table must never do.
|
| 83 |
-
"""
|
| 84 |
-
existing = (
|
| 85 |
-
db.query(OnchainRationaleDB)
|
| 86 |
-
.filter(OnchainRationaleDB.rationale_hash == req.rationale_hash)
|
| 87 |
-
.one_or_none()
|
| 88 |
-
)
|
| 89 |
-
if existing is not None:
|
| 90 |
-
if existing.rationale != req.rationale:
|
| 91 |
-
raise HTTPException(
|
| 92 |
-
status_code=409,
|
| 93 |
-
detail=(
|
| 94 |
-
"rationale_hash already recorded with different text; "
|
| 95 |
-
"an anchored hash's preimage cannot be overwritten"
|
| 96 |
-
),
|
| 97 |
-
)
|
| 98 |
-
# The route decorator's status_code=201 is FastAPI's default for
|
| 99 |
-
# every plain-dict return from this handler, including this one --
|
| 100 |
-
# it must be overridden explicitly here or a replayed post reports
|
| 101 |
-
# itself as newly Created.
|
| 102 |
-
response.status_code = 200
|
| 103 |
-
return {"status": "already_recorded", "rationale_hash": req.rationale_hash}
|
| 104 |
-
|
| 105 |
-
row = OnchainRationaleDB(
|
| 106 |
-
rationale_hash=req.rationale_hash,
|
| 107 |
-
rationale=req.rationale,
|
| 108 |
-
agent_address=req.agent_address,
|
| 109 |
-
evaluator_address=req.evaluator_address,
|
| 110 |
-
)
|
| 111 |
-
db.add(row)
|
| 112 |
-
db.commit()
|
| 113 |
-
logger.info("persisted rationale for hash %s", req.rationale_hash)
|
| 114 |
-
return {"status": "recorded", "rationale_hash": req.rationale_hash}
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
@router.get("/onchain/rationale/{rationale_hash}", response_model=RationaleResponse)
|
| 118 |
-
async def get_rationale(
|
| 119 |
-
rationale_hash: str,
|
| 120 |
-
db: Session = Depends(get_db),
|
| 121 |
-
):
|
| 122 |
-
"""Fetch the plaintext behind an anchored `rationale_hash`.
|
| 123 |
-
|
| 124 |
-
What an auditor calls after reading a `DecisionAnchored` event off-chain
|
| 125 |
-
-- the hash from the event is the only key this endpoint accepts, by
|
| 126 |
-
design: there is no listing or search here, only lookup by the exact
|
| 127 |
-
value that was signed and anchored.
|
| 128 |
-
"""
|
| 129 |
-
try:
|
| 130 |
-
normalized = _validate_hex_hash(rationale_hash)
|
| 131 |
-
except ValueError as exc:
|
| 132 |
-
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
| 133 |
-
|
| 134 |
-
row = (
|
| 135 |
-
db.query(OnchainRationaleDB)
|
| 136 |
-
.filter(OnchainRationaleDB.rationale_hash == normalized)
|
| 137 |
-
.one_or_none()
|
| 138 |
-
)
|
| 139 |
-
if row is None:
|
| 140 |
-
raise HTTPException(
|
| 141 |
-
status_code=404, detail="no rationale recorded for this hash"
|
| 142 |
-
)
|
| 143 |
-
|
| 144 |
-
return RationaleResponse(
|
| 145 |
-
rationale_hash=row.rationale_hash,
|
| 146 |
-
rationale=row.rationale,
|
| 147 |
-
agent_address=row.agent_address,
|
| 148 |
-
evaluator_address=row.evaluator_address,
|
| 149 |
-
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
app/api/routes_payments.py
CHANGED
|
@@ -2,16 +2,12 @@
|
|
| 2 |
Payment endpoints – Stripe Checkout integration.
|
| 3 |
"""
|
| 4 |
|
| 5 |
-
import logging
|
| 6 |
import os
|
| 7 |
import stripe
|
| 8 |
-
from fastapi import APIRouter,
|
| 9 |
from pydantic import BaseModel
|
| 10 |
|
| 11 |
-
from app.core import
|
| 12 |
-
from app.core.usage_tracker import Tier, resolve_api_key_identity
|
| 13 |
-
|
| 14 |
-
logger = logging.getLogger(__name__)
|
| 15 |
|
| 16 |
router = APIRouter(prefix="/payments", tags=["payments"])
|
| 17 |
|
|
@@ -21,46 +17,24 @@ STRIPE_WEBHOOK_SECRET = os.getenv("STRIPE_WEBHOOK_SECRET")
|
|
| 21 |
|
| 22 |
|
| 23 |
class CheckoutRequest(BaseModel):
|
|
|
|
|
|
|
| 24 |
success_url: str
|
| 25 |
cancel_url: str
|
| 26 |
|
| 27 |
|
| 28 |
@router.post("/create-checkout-session")
|
| 29 |
-
async def create_checkout_session(
|
| 30 |
-
|
| 31 |
-
identity: dict = Depends(resolve_api_key_identity),
|
| 32 |
-
):
|
| 33 |
-
"""Create a Stripe Checkout session for the Pro tier.
|
| 34 |
-
|
| 35 |
-
Identity comes from `resolve_api_key_identity` (Authorization: Bearer
|
| 36 |
-
header, same pepper-HMAC lookup every other authenticated endpoint
|
| 37 |
-
uses) rather than from a caller-supplied field in the request body --
|
| 38 |
-
previously this endpoint took `api_key` as a JSON field with no
|
| 39 |
-
`Depends()` gate at all, so nothing distinguished "the caller proved
|
| 40 |
-
they hold this key" from "the caller typed this string into a
|
| 41 |
-
request." `enforce_quota` is deliberately not used here: a FREE-tier
|
| 42 |
-
caller who has exhausted their monthly quota must still be able to
|
| 43 |
-
reach this endpoint, since upgrading is often exactly what they are
|
| 44 |
-
trying to do.
|
| 45 |
-
"""
|
| 46 |
if not stripe.api_key:
|
| 47 |
raise HTTPException(status_code=500, detail="Stripe not configured")
|
| 48 |
-
if not usage_tracker.tracker:
|
| 49 |
-
raise HTTPException(status_code=503, detail="Usage tracking service not initialised")
|
| 50 |
|
| 51 |
-
|
|
|
|
|
|
|
| 52 |
raise HTTPException(status_code=400,
|
| 53 |
detail="Only free tier keys can be upgraded")
|
| 54 |
|
| 55 |
-
# tenant_id, not the raw api_key, is what travels to Stripe from here
|
| 56 |
-
# on. api_key is a bearer secret -- usage_tracker.py exists specifically
|
| 57 |
-
# to never store it in plaintext (pepper-HMAC lookup + salted
|
| 58 |
-
# verification hash), and Stripe's dashboard/webhook logs/API are a
|
| 59 |
-
# third party with no reason to ever see it. tenant_id is an opaque,
|
| 60 |
-
# non-secret row identifier and is exactly what the webhook needs to
|
| 61 |
-
# look up which tenant's keys to retier.
|
| 62 |
-
tenant_id = identity["tenant_id"]
|
| 63 |
-
|
| 64 |
try:
|
| 65 |
checkout_session = stripe.checkout.Session.create(
|
| 66 |
payment_method_types=["card"],
|
|
@@ -74,17 +48,9 @@ async def create_checkout_session(
|
|
| 74 |
mode="subscription",
|
| 75 |
success_url=req.success_url,
|
| 76 |
cancel_url=req.cancel_url,
|
| 77 |
-
metadata={"
|
| 78 |
-
client_reference_id=
|
| 79 |
-
# checkout.session.completed carries this metadata via
|
| 80 |
-
# session.metadata (handled below), but customer.subscription.*
|
| 81 |
-
# events only carry the *subscription's own* metadata -- Stripe
|
| 82 |
-
# does not copy Session.metadata onto the Subscription it
|
| 83 |
-
# creates. Without this, cancellations can't be traced back to
|
| 84 |
-
# a tenant and PRO tier never downgrades.
|
| 85 |
-
subscription_data={"metadata": {"tenant_id": tenant_id}},
|
| 86 |
)
|
| 87 |
return {"sessionId": checkout_session.id, "url": checkout_session.url}
|
| 88 |
-
except Exception:
|
| 89 |
-
|
| 90 |
-
raise HTTPException(status_code=500, detail="Internal server error")
|
|
|
|
| 2 |
Payment endpoints – Stripe Checkout integration.
|
| 3 |
"""
|
| 4 |
|
|
|
|
| 5 |
import os
|
| 6 |
import stripe
|
| 7 |
+
from fastapi import APIRouter, HTTPException
|
| 8 |
from pydantic import BaseModel
|
| 9 |
|
| 10 |
+
from app.core.usage_tracker import tracker, Tier
|
|
|
|
|
|
|
|
|
|
| 11 |
|
| 12 |
router = APIRouter(prefix="/payments", tags=["payments"])
|
| 13 |
|
|
|
|
| 17 |
|
| 18 |
|
| 19 |
class CheckoutRequest(BaseModel):
|
| 20 |
+
api_key: str
|
| 21 |
+
|
| 22 |
success_url: str
|
| 23 |
cancel_url: str
|
| 24 |
|
| 25 |
|
| 26 |
@router.post("/create-checkout-session")
|
| 27 |
+
async def create_checkout_session(req: CheckoutRequest):
|
| 28 |
+
"""Create a Stripe Checkout session for the Pro tier."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
if not stripe.api_key:
|
| 30 |
raise HTTPException(status_code=500, detail="Stripe not configured")
|
|
|
|
|
|
|
| 31 |
|
| 32 |
+
# Verify the API key exists and is free tier
|
| 33 |
+
tier = tracker.get_tier(req.api_key) if tracker else None
|
| 34 |
+
if tier != Tier.FREE:
|
| 35 |
raise HTTPException(status_code=400,
|
| 36 |
detail="Only free tier keys can be upgraded")
|
| 37 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
try:
|
| 39 |
checkout_session = stripe.checkout.Session.create(
|
| 40 |
payment_method_types=["card"],
|
|
|
|
| 48 |
mode="subscription",
|
| 49 |
success_url=req.success_url,
|
| 50 |
cancel_url=req.cancel_url,
|
| 51 |
+
metadata={"api_key": req.api_key},
|
| 52 |
+
client_reference_id=req.api_key,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
)
|
| 54 |
return {"sessionId": checkout_session.id, "url": checkout_session.url}
|
| 55 |
+
except Exception as e:
|
| 56 |
+
raise HTTPException(status_code=500, detail=str(e))
|
|
|
app/api/routes_pricing.py
CHANGED
|
@@ -57,26 +57,48 @@ async def run_pricing(
|
|
| 57 |
quota: dict = Depends(enforce_quota),
|
| 58 |
):
|
| 59 |
"""
|
| 60 |
-
Multi‑run pricing with cooldown and buffer persistence.
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
`random.random() > risk_score` -- a fabricated result, not a real deal
|
| 64 |
-
outcome -- into a calibration buffer with no customer_id scoping, so
|
| 65 |
-
every customer's calls read and wrote the same file. Net effect: every
|
| 66 |
-
customer's price was shaped by every other customer's randomly-generated
|
| 67 |
-
outcomes, not just their own. Disabled until both are fixed: (1) a real
|
| 68 |
-
outcome-ingestion path (this endpoint must not invent one), and (2) the
|
| 69 |
-
buffer scoped per customer. See AUDIT_arf-bayesian-pricing-calculator.md
|
| 70 |
-
and AUDIT_arf-api.md (workspace root) for the original findings and
|
| 71 |
-
recommended fix. `Depends(enforce_quota)` stays active so this still
|
| 72 |
-
requires the same auth it always did -- only authenticated callers reach
|
| 73 |
-
the disabled-notice below; everyone else still gets the normal 401/403.
|
| 74 |
"""
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 57 |
quota: dict = Depends(enforce_quota),
|
| 58 |
):
|
| 59 |
"""
|
| 60 |
+
Multi‑run pricing with cooldown and buffer persistence.
|
| 61 |
+
Each run’s simulated outcome is added to the buffer, so subsequent runs
|
| 62 |
+
see an updated posterior.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 63 |
"""
|
| 64 |
+
# We need to reuse the same buffer across runs; we'll load it per request.
|
| 65 |
+
# For simplicity, we'll load from the default location.
|
| 66 |
+
from arf_pricing_calculator.storage.buffer import load_buffer, add_event
|
| 67 |
+
from arf_pricing_calculator.orchestration.cooldown import enforce_cooldown, is_cooldown_active
|
| 68 |
+
|
| 69 |
+
outputs = []
|
| 70 |
+
buffer = load_buffer() # loads from calibration_buffer.json
|
| 71 |
+
|
| 72 |
+
for i in range(req.runs):
|
| 73 |
+
if not req.force and is_cooldown_active(
|
| 74 |
+
req.customer_id, req.cooldown_hours):
|
| 75 |
+
raise HTTPException(status_code=429,
|
| 76 |
+
detail=f"Cooldown active after {i} runs")
|
| 77 |
+
|
| 78 |
+
pricing_input = parse_input_dict(req.input)
|
| 79 |
+
engine = PricingEngine(calibration_buffer=buffer)
|
| 80 |
+
out = engine.estimate(pricing_input)
|
| 81 |
+
|
| 82 |
+
# Simulate an outcome (in real use, this would come from the actual
|
| 83 |
+
# deal)
|
| 84 |
+
import random
|
| 85 |
+
outcome = "success" if random.random() > out.risk_score else "failure" # nosec B311
|
| 86 |
+
|
| 87 |
+
event = {
|
| 88 |
+
"run_id": out.run_history_id,
|
| 89 |
+
"customer_id": req.customer_id,
|
| 90 |
+
"outcome": outcome,
|
| 91 |
+
"price": out.recommended_price,
|
| 92 |
+
"value": out.expected_value,
|
| 93 |
+
"risk_score": out.risk_score,
|
| 94 |
+
"run_number": i + 1,
|
| 95 |
+
}
|
| 96 |
+
add_event(event)
|
| 97 |
+
buffer = load_buffer() # reload after update
|
| 98 |
+
|
| 99 |
+
outputs.append(out)
|
| 100 |
+
|
| 101 |
+
if i < req.runs - 1:
|
| 102 |
+
enforce_cooldown(req.customer_id, req.cooldown_hours)
|
| 103 |
+
|
| 104 |
+
return outputs
|
app/api/routes_risk.py
CHANGED
|
@@ -1,13 +1,8 @@
|
|
| 1 |
-
import
|
| 2 |
-
|
| 3 |
-
from fastapi import APIRouter, Depends, HTTPException
|
| 4 |
-
from app.api.deps import verify_internal_key
|
| 5 |
from app.models.risk_models import RiskResponse
|
| 6 |
from app.services.risk_service import get_system_risk
|
| 7 |
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
router = APIRouter(dependencies=[Depends(verify_internal_key)])
|
| 11 |
|
| 12 |
|
| 13 |
@router.get("/get_risk", response_model=RiskResponse)
|
|
@@ -18,9 +13,8 @@ async def get_risk():
|
|
| 18 |
raise HTTPException(
|
| 19 |
status_code=501,
|
| 20 |
detail="This endpoint is deprecated and not implemented")
|
| 21 |
-
except Exception:
|
| 22 |
-
|
| 23 |
-
raise HTTPException(status_code=500, detail="Internal server error")
|
| 24 |
|
| 25 |
if risk < 0.3:
|
| 26 |
status = "low"
|
|
@@ -31,3 +25,13 @@ async def get_risk():
|
|
| 31 |
else:
|
| 32 |
status = "critical"
|
| 33 |
return RiskResponse(system_risk=risk, status=status)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from fastapi import APIRouter, HTTPException
|
|
|
|
|
|
|
|
|
|
| 2 |
from app.models.risk_models import RiskResponse
|
| 3 |
from app.services.risk_service import get_system_risk
|
| 4 |
|
| 5 |
+
router = APIRouter()
|
|
|
|
|
|
|
| 6 |
|
| 7 |
|
| 8 |
@router.get("/get_risk", response_model=RiskResponse)
|
|
|
|
| 13 |
raise HTTPException(
|
| 14 |
status_code=501,
|
| 15 |
detail="This endpoint is deprecated and not implemented")
|
| 16 |
+
except Exception as e:
|
| 17 |
+
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
| 18 |
|
| 19 |
if risk < 0.3:
|
| 20 |
status = "low"
|
|
|
|
| 25 |
else:
|
| 26 |
status = "critical"
|
| 27 |
return RiskResponse(system_risk=risk, status=status)
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
@router.get("/history")
|
| 31 |
+
async def get_risk_history():
|
| 32 |
+
import random
|
| 33 |
+
import datetime
|
| 34 |
+
now = datetime.datetime.now()
|
| 35 |
+
data = [{"time": (now - datetime.timedelta(hours=i)).isoformat(),
|
| 36 |
+
"risk": round(random.uniform(0.2, 0.8), 2)} for i in range(24, 0, -1)]
|
| 37 |
+
return data
|
app/api/routes_users.py
CHANGED
|
@@ -9,8 +9,7 @@ from sqlalchemy.orm import Session
|
|
| 9 |
from slowapi import Limiter
|
| 10 |
from slowapi.util import get_remote_address
|
| 11 |
|
| 12 |
-
from app.core import
|
| 13 |
-
from app.core.usage_tracker import enforce_quota, Tier
|
| 14 |
from app.api.deps import get_db
|
| 15 |
from app.database.models_intents import TenantDB # <-- NEW
|
| 16 |
|
|
@@ -31,7 +30,7 @@ async def register_user(
|
|
| 31 |
Public endpoint to create a new free‑tier API key and a new tenant.
|
| 32 |
Rate‑limited to 5 requests per hour per IP address.
|
| 33 |
"""
|
| 34 |
-
if
|
| 35 |
raise HTTPException(status_code=503, detail="Usage tracking service not initialised")
|
| 36 |
|
| 37 |
# 1. Create a new tenant in the main database
|
|
@@ -49,7 +48,7 @@ async def register_user(
|
|
| 49 |
|
| 50 |
# 2. Generate a new API key for this tenant
|
| 51 |
new_key = f"sk_free_{uuid.uuid4().hex[:24]}"
|
| 52 |
-
success =
|
| 53 |
if not success:
|
| 54 |
# Rollback tenant creation if key creation fails
|
| 55 |
db.delete(new_tenant)
|
|
|
|
| 9 |
from slowapi import Limiter
|
| 10 |
from slowapi.util import get_remote_address
|
| 11 |
|
| 12 |
+
from app.core.usage_tracker import tracker, enforce_quota, Tier
|
|
|
|
| 13 |
from app.api.deps import get_db
|
| 14 |
from app.database.models_intents import TenantDB # <-- NEW
|
| 15 |
|
|
|
|
| 30 |
Public endpoint to create a new free‑tier API key and a new tenant.
|
| 31 |
Rate‑limited to 5 requests per hour per IP address.
|
| 32 |
"""
|
| 33 |
+
if tracker is None:
|
| 34 |
raise HTTPException(status_code=503, detail="Usage tracking service not initialised")
|
| 35 |
|
| 36 |
# 1. Create a new tenant in the main database
|
|
|
|
| 48 |
|
| 49 |
# 2. Generate a new API key for this tenant
|
| 50 |
new_key = f"sk_free_{uuid.uuid4().hex[:24]}"
|
| 51 |
+
success = tracker.get_or_create_api_key(api_key=new_key, tenant_id=tenant_id, tier=Tier.FREE)
|
| 52 |
if not success:
|
| 53 |
# Rollback tenant creation if key creation fails
|
| 54 |
db.delete(new_tenant)
|
app/api/webhooks.py
CHANGED
|
@@ -2,24 +2,16 @@
|
|
| 2 |
Stripe webhook handler – updates API key tier on subscription events.
|
| 3 |
"""
|
| 4 |
|
| 5 |
-
import logging
|
| 6 |
import os
|
| 7 |
import stripe
|
| 8 |
from fastapi import APIRouter, Request, HTTPException
|
| 9 |
-
from app.core.usage_tracker import
|
| 10 |
-
|
| 11 |
-
logger = logging.getLogger(__name__)
|
| 12 |
|
| 13 |
router = APIRouter(prefix="/webhooks", tags=["webhooks"])
|
| 14 |
|
| 15 |
STRIPE_WEBHOOK_SECRET = os.getenv("STRIPE_WEBHOOK_SECRET")
|
| 16 |
stripe.api_key = os.getenv("STRIPE_SECRET_KEY")
|
| 17 |
|
| 18 |
-
# Subscription statuses that mean "not entitled to Pro anymore". Deliberately
|
| 19 |
-
# does NOT include "past_due" -- whether a failed-payment grace period keeps
|
| 20 |
-
# Pro access is a dunning-policy decision, not a bug, and isn't decided here.
|
| 21 |
-
_DOWNGRADE_STATUSES = {"canceled", "unpaid", "incomplete_expired"}
|
| 22 |
-
|
| 23 |
|
| 24 |
@router.post("/stripe")
|
| 25 |
async def stripe_webhook(request: Request):
|
|
@@ -38,33 +30,20 @@ async def stripe_webhook(request: Request):
|
|
| 38 |
except stripe.error.SignatureVerificationError:
|
| 39 |
raise HTTPException(status_code=400, detail="Invalid signature")
|
| 40 |
|
| 41 |
-
#
|
| 42 |
-
# routes_payments.py's create_checkout_session for why the raw bearer
|
| 43 |
-
# key must never round-trip through a third party.
|
| 44 |
if event["type"] == "checkout.session.completed":
|
| 45 |
session = event["data"]["object"]
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
logger.info(
|
| 52 |
-
"checkout.session.completed with payment_status=%r; not upgrading yet",
|
| 53 |
-
session.get("payment_status"),
|
| 54 |
-
)
|
| 55 |
-
return {"status": "ok"}
|
| 56 |
-
tenant_id = session.get("client_reference_id") or session.get(
|
| 57 |
-
"metadata", {}).get("tenant_id")
|
| 58 |
-
if tenant_id:
|
| 59 |
-
update_key_tier_by_tenant_id(tenant_id, Tier.PRO)
|
| 60 |
-
elif event["type"] in ("customer.subscription.deleted", "customer.subscription.updated"):
|
| 61 |
subscription = event["data"]["object"]
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
update_key_tier_by_tenant_id(tenant_id, Tier.FREE)
|
| 69 |
|
| 70 |
return {"status": "ok"}
|
|
|
|
| 2 |
Stripe webhook handler – updates API key tier on subscription events.
|
| 3 |
"""
|
| 4 |
|
|
|
|
| 5 |
import os
|
| 6 |
import stripe
|
| 7 |
from fastapi import APIRouter, Request, HTTPException
|
| 8 |
+
from app.core.usage_tracker import update_key_tier, Tier
|
|
|
|
|
|
|
| 9 |
|
| 10 |
router = APIRouter(prefix="/webhooks", tags=["webhooks"])
|
| 11 |
|
| 12 |
STRIPE_WEBHOOK_SECRET = os.getenv("STRIPE_WEBHOOK_SECRET")
|
| 13 |
stripe.api_key = os.getenv("STRIPE_SECRET_KEY")
|
| 14 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
|
| 16 |
@router.post("/stripe")
|
| 17 |
async def stripe_webhook(request: Request):
|
|
|
|
| 30 |
except stripe.error.SignatureVerificationError:
|
| 31 |
raise HTTPException(status_code=400, detail="Invalid signature")
|
| 32 |
|
| 33 |
+
# Handle subscription events
|
|
|
|
|
|
|
| 34 |
if event["type"] == "checkout.session.completed":
|
| 35 |
session = event["data"]["object"]
|
| 36 |
+
api_key = session.get("client_reference_id") or session.get(
|
| 37 |
+
"metadata", {}).get("api_key")
|
| 38 |
+
if api_key:
|
| 39 |
+
update_key_tier(api_key, Tier.PRO)
|
| 40 |
+
elif event["type"] == "customer.subscription.deleted":
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 41 |
subscription = event["data"]["object"]
|
| 42 |
+
# You need to store a mapping from subscription ID to API key.
|
| 43 |
+
# For simplicity, we assume you stored it in metadata during checkout.
|
| 44 |
+
# Alternatively, look up by customer ID.
|
| 45 |
+
api_key = subscription.get("metadata", {}).get("api_key")
|
| 46 |
+
if api_key:
|
| 47 |
+
update_key_tier(api_key, Tier.FREE)
|
|
|
|
| 48 |
|
| 49 |
return {"status": "ok"}
|
app/core/config.py
CHANGED
|
@@ -14,7 +14,6 @@ class Settings(BaseSettings):
|
|
| 14 |
ARF_USAGE_DB_PATH: str = "arf_usage.db"
|
| 15 |
ARF_REDIS_URL: Optional[str] = None
|
| 16 |
ARF_API_KEYS: str = "{}" # JSON string of {key: tier}
|
| 17 |
-
ARF_KEY_PEPPER: Optional[str] = None # required if ARF_USAGE_TRACKING is true
|
| 18 |
|
| 19 |
# Tracing (OpenTelemetry)
|
| 20 |
OTEL_EXPORTER_OTLP_ENDPOINT: Optional[str] = None
|
|
|
|
| 14 |
ARF_USAGE_DB_PATH: str = "arf_usage.db"
|
| 15 |
ARF_REDIS_URL: Optional[str] = None
|
| 16 |
ARF_API_KEYS: str = "{}" # JSON string of {key: tier}
|
|
|
|
| 17 |
|
| 18 |
# Tracing (OpenTelemetry)
|
| 19 |
OTEL_EXPORTER_OTLP_ENDPOINT: Optional[str] = None
|
app/core/storage.py
CHANGED
|
@@ -1,16 +1,2 @@
|
|
| 1 |
-
|
| 2 |
-
|
| 3 |
-
Bounded (maxlen), not persisted across restarts -- exists to give the
|
| 4 |
-
causal explainer and GET /history recent context, not as a durable audit
|
| 5 |
-
trail. The cap protects against unbounded memory growth from
|
| 6 |
-
POST /report_incident, which can be called repeatedly by anything holding
|
| 7 |
-
a valid internal key.
|
| 8 |
-
|
| 9 |
-
Shared by app.api.routes_incidents (writes, via report_incident) and
|
| 10 |
-
app.api.routes_history (reads, via GET /history) -- both must import this
|
| 11 |
-
same object rather than declaring their own list, or writes and reads
|
| 12 |
-
silently operate on two different lists.
|
| 13 |
-
"""
|
| 14 |
-
from collections import deque
|
| 15 |
-
|
| 16 |
-
incident_history: deque = deque(maxlen=10_000)
|
|
|
|
| 1 |
+
# Simple in-memory list for incident history
|
| 2 |
+
incident_history = []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
app/core/usage_tracker.py
CHANGED
|
@@ -4,49 +4,11 @@ Thread‑safe, atomic quota consumption, idempotent, fail‑closed.
|
|
| 4 |
|
| 5 |
Extended for multi‑tenancy: each API key is linked to a tenant ID.
|
| 6 |
Tenant ID is stored in the `api_keys` table and used for resource isolation.
|
| 7 |
-
|
| 8 |
-
API keys in `api_keys` are never stored in plaintext. The lookup index is
|
| 9 |
-
HMAC-SHA256(pepper, key) -- a deterministic but one-way value computed with
|
| 10 |
-
a server-only secret (ARF_KEY_PEPPER), so a leaked database alone does not
|
| 11 |
-
expose usable keys. A per-row random salt plus a second SHA-256 check is a
|
| 12 |
-
defense-in-depth verification layer after the row is found by lookup hash.
|
| 13 |
-
This mirrors arf-gateway's internal/auth/apikey.go.
|
| 14 |
-
|
| 15 |
-
`api_keys` itself lives in Postgres (DATABASE_URL), not in the SQLite file
|
| 16 |
-
the rest of this module uses -- a single durable table that both arf-api
|
| 17 |
-
and arf-gateway point at, instead of each service's own local SQLite copy
|
| 18 |
-
(which on Render's Free plan is wiped on every deploy/restart anyway).
|
| 19 |
-
|
| 20 |
-
`usage_log` and `idempotency_keys` remain SQLite-only, local to this
|
| 21 |
-
service -- arf-gateway never reads them. `monthly_counts` is different:
|
| 22 |
-
arf-gateway's Go code (internal/auth/apikey.go) queries it from Postgres
|
| 23 |
-
directly, via the pgx driver against the same DATABASE_URL, to compute
|
| 24 |
-
each key's remaining quota. It never had a way to read this service's
|
| 25 |
-
local SQLite file (a different process, a different disk, a different
|
| 26 |
-
protocol), so a prior version of this module -- which wrote
|
| 27 |
-
`monthly_counts` to SQLite only -- left that Postgres table permanently
|
| 28 |
-
empty, and arf-gateway's quota check silently treated every key as having
|
| 29 |
-
consumed nothing all month, every month. `consume_quota_and_log` now
|
| 30 |
-
mirrors every successfully-counted call into the Postgres `monthly_counts`
|
| 31 |
-
table too (`_record_pg_monthly_count`), best-effort and logged loudly on
|
| 32 |
-
failure rather than raised -- this service's own quota decision is still
|
| 33 |
-
made from its local SQLite/Redis count and must not fail because a
|
| 34 |
-
mirroring write to a peer service's view did. Still keyed by the raw API
|
| 35 |
-
key (not hashed), matching the existing SQLite schema and a separately
|
| 36 |
-
tracked plaintext-storage gap -- not addressed by this fix.
|
| 37 |
"""
|
| 38 |
-
import hashlib
|
| 39 |
-
import hmac
|
| 40 |
import json
|
| 41 |
-
import logging
|
| 42 |
-
import os
|
| 43 |
-
import secrets
|
| 44 |
import sqlite3
|
| 45 |
import threading
|
| 46 |
import time
|
| 47 |
-
|
| 48 |
-
import psycopg2
|
| 49 |
-
import psycopg2.extras
|
| 50 |
from contextlib import contextmanager
|
| 51 |
from datetime import datetime, timedelta
|
| 52 |
from dataclasses import dataclass
|
|
@@ -106,51 +68,15 @@ class UsageRecord:
|
|
| 106 |
processing_ms: Optional[float] = None
|
| 107 |
|
| 108 |
|
| 109 |
-
# Bounded retry for the initial Postgres connect -- see _get_pg_conn's
|
| 110 |
-
# docstring. 5 attempts with exponential backoff (1+2+4+8 = 15s of sleep,
|
| 111 |
-
# worst case) comfortably fits inside a container's normal boot window
|
| 112 |
-
# without turning a real outage into a long hang.
|
| 113 |
-
_PG_CONNECT_MAX_ATTEMPTS = 5
|
| 114 |
-
_PG_CONNECT_BACKOFF_BASE = 1.0
|
| 115 |
-
|
| 116 |
-
logger = logging.getLogger(__name__)
|
| 117 |
-
|
| 118 |
-
|
| 119 |
class UsageTracker:
|
| 120 |
"""
|
| 121 |
Thread‑safe usage tracker with atomic quota consumption and idempotency.
|
| 122 |
Extended to support tenant isolation: each API key is linked to a tenant.
|
| 123 |
"""
|
| 124 |
|
| 125 |
-
# Whether the Postgres api_keys schema check has already run in this
|
| 126 |
-
# process. Class-level, not per-instance: the schema is a property of
|
| 127 |
-
# the database, not of a tracker object, and every instance in a
|
| 128 |
-
# process points at the same DATABASE_URL. Guarded by a lock because
|
| 129 |
-
# _get_pg_conn is called from request threads.
|
| 130 |
-
_pg_schema_ready: bool = False
|
| 131 |
-
_pg_schema_lock = threading.Lock()
|
| 132 |
-
|
| 133 |
def __init__(self, db_path: str = "arf_usage.db",
|
| 134 |
-
redis_url: Optional[str] = None
|
| 135 |
-
pepper: Optional[str] = None):
|
| 136 |
self.db_path = db_path
|
| 137 |
-
self._pepper = pepper if pepper is not None else os.getenv("ARF_KEY_PEPPER", "")
|
| 138 |
-
if not self._pepper:
|
| 139 |
-
raise RuntimeError(
|
| 140 |
-
"ARF_KEY_PEPPER is not set -- refusing to start without it, "
|
| 141 |
-
"since it is required to look up or verify any API key."
|
| 142 |
-
)
|
| 143 |
-
if len(self._pepper) < 32:
|
| 144 |
-
raise RuntimeError(
|
| 145 |
-
f"ARF_KEY_PEPPER is too short ({len(self._pepper)} chars); "
|
| 146 |
-
"use at least 32 random characters."
|
| 147 |
-
)
|
| 148 |
-
self._pg_dsn = os.getenv("DATABASE_URL", "")
|
| 149 |
-
if not self._pg_dsn:
|
| 150 |
-
raise RuntimeError(
|
| 151 |
-
"DATABASE_URL is not set -- refusing to start without it, "
|
| 152 |
-
"since api_keys is stored in Postgres, not SQLite."
|
| 153 |
-
)
|
| 154 |
self._local = threading.local()
|
| 155 |
self._init_db()
|
| 156 |
|
|
@@ -160,40 +86,9 @@ class UsageTracker:
|
|
| 160 |
elif redis_url:
|
| 161 |
raise ImportError("Redis client not installed. Run: pip install redis")
|
| 162 |
|
| 163 |
-
def _lookup_hash(self, key: str) -> str:
|
| 164 |
-
"""Deterministic pepper-HMAC used to find a key's row without ever
|
| 165 |
-
storing or querying by the plaintext key."""
|
| 166 |
-
return hmac.new(self._pepper.encode(), key.encode(), hashlib.sha256).hexdigest()
|
| 167 |
-
|
| 168 |
-
@staticmethod
|
| 169 |
-
def _salted_hash(key: str, salt_hex: str) -> str:
|
| 170 |
-
"""Per-row salted verification hash, checked after a row has
|
| 171 |
-
already been found via lookup hash."""
|
| 172 |
-
return hashlib.sha256(bytes.fromhex(salt_hex) + key.encode()).hexdigest()
|
| 173 |
-
|
| 174 |
-
def _verify_key(self, conn, api_key: str) -> Optional[dict]:
|
| 175 |
-
"""Look up a row by pepper-HMAC, then verify with the salted hash.
|
| 176 |
-
`conn` is a Postgres connection from _get_pg_conn. Returns the row
|
| 177 |
-
(tenant_id, tier, is_active, salt, key_hash) if the key is valid and
|
| 178 |
-
active, else None."""
|
| 179 |
-
row = self._pg_execute(
|
| 180 |
-
conn,
|
| 181 |
-
"SELECT tenant_id, tier, is_active, salt, key_hash FROM api_keys "
|
| 182 |
-
"WHERE lookup_hash = %s",
|
| 183 |
-
(self._lookup_hash(api_key),)
|
| 184 |
-
).fetchone()
|
| 185 |
-
if not row or not row["is_active"]:
|
| 186 |
-
return None
|
| 187 |
-
if not hmac.compare_digest(self._salted_hash(api_key, row["salt"]), row["key_hash"]):
|
| 188 |
-
return None
|
| 189 |
-
return row
|
| 190 |
-
|
| 191 |
@contextmanager
|
| 192 |
def _get_conn(self):
|
| 193 |
-
"""Get a thread‑local SQLite connection with WAL and immediate transactions.
|
| 194 |
-
|
| 195 |
-
Backs usage_log/monthly_counts/idempotency_keys only -- api_keys
|
| 196 |
-
lives in Postgres, see _get_pg_conn below."""
|
| 197 |
if not hasattr(self._local, "conn"):
|
| 198 |
self._local.conn = sqlite3.connect(
|
| 199 |
self.db_path, check_same_thread=False, isolation_level=None)
|
|
@@ -201,160 +96,20 @@ class UsageTracker:
|
|
| 201 |
self._local.conn.execute("PRAGMA journal_mode=WAL")
|
| 202 |
yield self._local.conn
|
| 203 |
|
| 204 |
-
@contextmanager
|
| 205 |
-
def _get_pg_conn(self):
|
| 206 |
-
"""Get a thread-local Postgres connection for the api_keys table.
|
| 207 |
-
Rows come back as dict-like objects (row["col"]) via RealDictCursor,
|
| 208 |
-
matching the sqlite3.Row access pattern used elsewhere in this file.
|
| 209 |
-
|
| 210 |
-
Retries the initial connect with backoff, but how patiently depends
|
| 211 |
-
on who is asking:
|
| 212 |
-
|
| 213 |
-
- **Startup** (`warm_up`, `retries=True`): a short DNS blip during a
|
| 214 |
-
cold container boot has been observed on Render, so a few seconds
|
| 215 |
-
of retrying is worth it to come up cleanly.
|
| 216 |
-
- **Request path** (the default, `retries=False`): fails fast. A
|
| 217 |
-
request thread that blocks for 15s on a database outage doesn't
|
| 218 |
-
make the request succeed; it holds a worker thread hostage, and
|
| 219 |
-
under any concurrency the pool is exhausted and the whole service
|
| 220 |
-
stops responding -- including its health endpoint. A prompt 503 is
|
| 221 |
-
strictly better than a slow one.
|
| 222 |
-
|
| 223 |
-
Genuine connection errors (bad credentials, wrong host) still raise
|
| 224 |
-
either way, preserving fail-closed."""
|
| 225 |
-
if not hasattr(self._local, "pg_conn") or self._local.pg_conn.closed:
|
| 226 |
-
self._connect_pg(retries=False)
|
| 227 |
-
yield self._local.pg_conn
|
| 228 |
-
|
| 229 |
-
def _connect_pg(self, retries: bool) -> None:
|
| 230 |
-
"""Open this thread's Postgres connection, optionally retrying."""
|
| 231 |
-
attempts = _PG_CONNECT_MAX_ATTEMPTS if retries else 1
|
| 232 |
-
last_exc: Optional[psycopg2.OperationalError] = None
|
| 233 |
-
for attempt in range(attempts):
|
| 234 |
-
try:
|
| 235 |
-
self._local.pg_conn = psycopg2.connect(
|
| 236 |
-
self._pg_dsn, cursor_factory=psycopg2.extras.RealDictCursor)
|
| 237 |
-
last_exc = None
|
| 238 |
-
break
|
| 239 |
-
except psycopg2.OperationalError as exc:
|
| 240 |
-
last_exc = exc
|
| 241 |
-
if attempt < attempts - 1:
|
| 242 |
-
time.sleep(_PG_CONNECT_BACKOFF_BASE * (2 ** attempt))
|
| 243 |
-
if last_exc is not None:
|
| 244 |
-
raise last_exc
|
| 245 |
-
self._ensure_pg_schema(self._local.pg_conn)
|
| 246 |
-
|
| 247 |
-
def warm_up(self) -> bool:
|
| 248 |
-
"""Best-effort startup connection, with retries.
|
| 249 |
-
|
| 250 |
-
Returns True if Postgres is reachable and the api_keys schema is
|
| 251 |
-
ready. Returns False -- rather than raising -- when it isn't, so a
|
| 252 |
-
caller can log the degradation and still start serving. That
|
| 253 |
-
asymmetry is the point: an unreachable database at boot should cost
|
| 254 |
-
api_keys-backed functionality, not the entire service.
|
| 255 |
-
"""
|
| 256 |
-
try:
|
| 257 |
-
self._connect_pg(retries=True)
|
| 258 |
-
return True
|
| 259 |
-
except psycopg2.OperationalError:
|
| 260 |
-
return False
|
| 261 |
-
|
| 262 |
-
def _ensure_pg_schema(self, conn) -> None:
|
| 263 |
-
"""Run the api_keys schema check once per process, on the first
|
| 264 |
-
connection that actually succeeds.
|
| 265 |
-
|
| 266 |
-
Guarded by a process-wide flag rather than done in __init__ so a
|
| 267 |
-
database that is unreachable at startup doesn't prevent the tracker
|
| 268 |
-
from existing -- it just means the first request that needs
|
| 269 |
-
Postgres pays for the schema check, and requests before that get a
|
| 270 |
-
clean 503 from enforce_quota instead of the whole service being
|
| 271 |
-
down. The statements are all IF NOT EXISTS, so re-running them on a
|
| 272 |
-
later process is harmless."""
|
| 273 |
-
if UsageTracker._pg_schema_ready:
|
| 274 |
-
return
|
| 275 |
-
with UsageTracker._pg_schema_lock:
|
| 276 |
-
if UsageTracker._pg_schema_ready:
|
| 277 |
-
return
|
| 278 |
-
self._create_pg_schema(conn)
|
| 279 |
-
UsageTracker._pg_schema_ready = True
|
| 280 |
-
|
| 281 |
-
@staticmethod
|
| 282 |
-
def _pg_execute(conn, sql: str, params: tuple = ()):
|
| 283 |
-
"""Run a query against a Postgres connection and return the cursor,
|
| 284 |
-
so callers can chain .fetchone()/.fetchall() the same way sqlite3's
|
| 285 |
-
conn.execute(...) is used elsewhere in this file."""
|
| 286 |
-
cur = conn.cursor()
|
| 287 |
-
cur.execute(sql, params)
|
| 288 |
-
return cur
|
| 289 |
-
|
| 290 |
-
def _create_pg_schema(self, conn):
|
| 291 |
-
"""Idempotently ensure the Postgres api_keys table/index exist.
|
| 292 |
-
|
| 293 |
-
Takes an already-open connection rather than acquiring one, because
|
| 294 |
-
its only caller is _ensure_pg_schema, which runs *from inside*
|
| 295 |
-
_get_pg_conn -- acquiring another connection here would recurse.
|
| 296 |
-
|
| 297 |
-
The canonical schema is the Alembic migration
|
| 298 |
-
(alembic/versions/*_create_api_keys_table.py) -- but nothing in this
|
| 299 |
-
codebase runs `alembic upgrade head` automatically on deploy (a
|
| 300 |
-
known gap, tracked separately), and arf-gateway's Go code needs the
|
| 301 |
-
same table without going through Python/Alembic at all. Mirroring
|
| 302 |
-
the same CREATE TABLE IF NOT EXISTS self-healing pattern this file
|
| 303 |
-
already uses for its SQLite tables keeps both services (and tests)
|
| 304 |
-
working whether or not the migration has actually been applied.
|
| 305 |
-
Column set/types must stay in sync with that migration."""
|
| 306 |
-
self._pg_execute(conn, """
|
| 307 |
-
CREATE TABLE IF NOT EXISTS api_keys (
|
| 308 |
-
id SERIAL PRIMARY KEY,
|
| 309 |
-
tenant_id VARCHAR(64) NOT NULL,
|
| 310 |
-
tier VARCHAR(32) NOT NULL,
|
| 311 |
-
created_at TIMESTAMP NOT NULL,
|
| 312 |
-
last_used_at TIMESTAMP,
|
| 313 |
-
is_active BOOLEAN NOT NULL DEFAULT true,
|
| 314 |
-
salt VARCHAR(64) NOT NULL,
|
| 315 |
-
key_hash VARCHAR(64) NOT NULL,
|
| 316 |
-
lookup_hash VARCHAR(64) NOT NULL
|
| 317 |
-
)
|
| 318 |
-
""")
|
| 319 |
-
self._pg_execute(conn, """
|
| 320 |
-
CREATE UNIQUE INDEX IF NOT EXISTS uq_api_keys_lookup_hash
|
| 321 |
-
ON api_keys (lookup_hash)
|
| 322 |
-
""")
|
| 323 |
-
self._pg_execute(conn, """
|
| 324 |
-
CREATE INDEX IF NOT EXISTS ix_api_keys_tenant_id
|
| 325 |
-
ON api_keys (tenant_id)
|
| 326 |
-
""")
|
| 327 |
-
# Mirrors arf-gateway's self-healing CREATE TABLE IF NOT EXISTS for
|
| 328 |
-
# this same table (internal/auth/apikey.go's NewValidator) -- either
|
| 329 |
-
# service may be the first to connect to a fresh database.
|
| 330 |
-
self._pg_execute(conn, """
|
| 331 |
-
CREATE TABLE IF NOT EXISTS monthly_counts (
|
| 332 |
-
api_key TEXT NOT NULL,
|
| 333 |
-
year_month TEXT NOT NULL,
|
| 334 |
-
count INTEGER NOT NULL DEFAULT 0,
|
| 335 |
-
PRIMARY KEY (api_key, year_month)
|
| 336 |
-
)
|
| 337 |
-
""")
|
| 338 |
-
conn.commit()
|
| 339 |
-
|
| 340 |
def _init_db(self):
|
| 341 |
-
"""Initialise SQLite tables
|
| 342 |
-
|
| 343 |
-
Deliberately does NOT touch Postgres. The api_keys schema is
|
| 344 |
-
ensured lazily on the first successful Postgres connection instead
|
| 345 |
-
(see _ensure_pg_schema) so that constructing a UsageTracker never
|
| 346 |
-
depends on the database being reachable *at that instant*.
|
| 347 |
-
|
| 348 |
-
This is the difference between a degraded service and no service.
|
| 349 |
-
Every other subsystem in main.py's lifespan already degrades
|
| 350 |
-
gracefully when Postgres is unreachable -- the Beta-state loader
|
| 351 |
-
logs a warning and continues -- but init_tracker raised, and
|
| 352 |
-
main.py turns that into RuntimeError, killing the process. On
|
| 353 |
-
Render that produced a crash loop that outlived the port-detection
|
| 354 |
-
window, so a DNS failure lasting seconds took the whole deploy
|
| 355 |
-
down and left the previous release serving.
|
| 356 |
-
"""
|
| 357 |
with self._get_conn() as conn:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 358 |
conn.execute("""
|
| 359 |
CREATE TABLE IF NOT EXISTS usage_log (
|
| 360 |
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
@@ -397,84 +152,65 @@ class UsageTracker:
|
|
| 397 |
Register a new API key for a given tenant.
|
| 398 |
|
| 399 |
Args:
|
| 400 |
-
key: The API key (plain text
|
| 401 |
tenant_id: UUID of the tenant (must already exist in main DB).
|
| 402 |
tier: Initial tier for the key.
|
| 403 |
|
| 404 |
Returns:
|
| 405 |
True if key was created (or already exists for the same tenant).
|
| 406 |
"""
|
| 407 |
-
|
| 408 |
-
|
| 409 |
-
|
| 410 |
-
conn, "SELECT tenant_id FROM api_keys WHERE lookup_hash = %s", (lookup_hash,)
|
| 411 |
-
).fetchone()
|
| 412 |
if row:
|
| 413 |
# Key already exists – ensure it belongs to the same tenant
|
| 414 |
-
|
| 415 |
-
|
|
|
|
| 416 |
raise ValueError(f"Key {key[:8]}... already belongs to a different tenant.")
|
| 417 |
-
conn.commit()
|
| 418 |
return True
|
| 419 |
-
|
| 420 |
-
|
| 421 |
-
|
| 422 |
-
"INSERT INTO api_keys "
|
| 423 |
-
"(tenant_id, tier, created_at, is_active, salt, key_hash, lookup_hash) "
|
| 424 |
-
"VALUES (%s, %s, %s, %s, %s, %s, %s)",
|
| 425 |
-
(tenant_id, tier.value, datetime.utcnow(), True,
|
| 426 |
-
salt, self._salted_hash(key, salt), lookup_hash)
|
| 427 |
)
|
| 428 |
conn.commit()
|
| 429 |
return True
|
| 430 |
|
| 431 |
def get_tier(self, api_key: str) -> Optional[Tier]:
|
| 432 |
"""Return the tier for a given API key, or None if key invalid/inactive."""
|
| 433 |
-
with self.
|
| 434 |
-
row =
|
| 435 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 436 |
|
| 437 |
def get_tenant_id(self, api_key: str) -> Optional[str]:
|
| 438 |
"""Return the tenant ID associated with the API key, or None if key invalid."""
|
| 439 |
-
with self.
|
| 440 |
-
row =
|
| 441 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 442 |
|
| 443 |
def update_api_key_tier(self, api_key: str, new_tier: Tier) -> bool:
|
| 444 |
"""Update the tier of an existing API key. Returns True if successful."""
|
| 445 |
-
|
| 446 |
-
|
| 447 |
-
|
| 448 |
-
conn, "SELECT lookup_hash FROM api_keys WHERE lookup_hash = %s", (lookup_hash,)
|
| 449 |
-
).fetchone()
|
| 450 |
if not row:
|
| 451 |
-
conn.rollback()
|
| 452 |
return False
|
| 453 |
-
|
| 454 |
-
|
| 455 |
-
(new_tier.value,
|
| 456 |
conn.commit()
|
| 457 |
return True
|
| 458 |
|
| 459 |
-
def update_tier_by_tenant_id(self, tenant_id: str, new_tier: Tier) -> bool:
|
| 460 |
-
"""Update the tier of every active API key belonging to tenant_id.
|
| 461 |
-
|
| 462 |
-
Used by the Stripe webhook, which must never handle a raw API key
|
| 463 |
-
(Stripe's own systems -- dashboard, logs, webhook payloads -- are a
|
| 464 |
-
third party; the plaintext bearer secret has no business being
|
| 465 |
-
stored there, which is exactly what passing it as Checkout
|
| 466 |
-
metadata used to do). tenant_id is not a secret -- it's an opaque
|
| 467 |
-
row identifier -- so it's safe to round-trip through Stripe."""
|
| 468 |
-
with self._get_pg_conn() as conn:
|
| 469 |
-
cur = self._pg_execute(
|
| 470 |
-
conn,
|
| 471 |
-
"UPDATE api_keys SET tier = %s WHERE tenant_id = %s AND is_active = true",
|
| 472 |
-
(new_tier.value, tenant_id),
|
| 473 |
-
)
|
| 474 |
-
updated = cur.rowcount > 0
|
| 475 |
-
conn.commit()
|
| 476 |
-
return updated
|
| 477 |
-
|
| 478 |
# --------------------------------------------------------------------------
|
| 479 |
# Atomic quota consumption (unchanged, but uses api_key which links to tenant)
|
| 480 |
# --------------------------------------------------------------------------
|
|
@@ -535,34 +271,6 @@ class UsageTracker:
|
|
| 535 |
result = self._redis_client.eval(lua_script, 1, redis_key, limit)
|
| 536 |
return result == 1
|
| 537 |
|
| 538 |
-
def _record_pg_monthly_count(self, api_key: str, month: str) -> None:
|
| 539 |
-
"""Mirror one successfully-counted call into Postgres `monthly_counts`,
|
| 540 |
-
the table arf-gateway's Go code actually reads to compute quota
|
| 541 |
-
remaining (see this module's docstring). Best-effort: this service's
|
| 542 |
-
own quota decision was already made from SQLite/Redis before this is
|
| 543 |
-
called, so a failure here must not fail the request that already
|
| 544 |
-
legitimately counted against quota -- but it also must not fail
|
| 545 |
-
*silently*, since a swallowed error here is exactly how arf-gateway's
|
| 546 |
-
quota check went blind for every key in the first place. Logged at
|
| 547 |
-
ERROR, not raised."""
|
| 548 |
-
try:
|
| 549 |
-
with self._get_pg_conn() as conn:
|
| 550 |
-
self._pg_execute(
|
| 551 |
-
conn,
|
| 552 |
-
"INSERT INTO monthly_counts (api_key, year_month, count) "
|
| 553 |
-
"VALUES (%s, %s, 1) ON CONFLICT (api_key, year_month) "
|
| 554 |
-
"DO UPDATE SET count = monthly_counts.count + 1",
|
| 555 |
-
(api_key, month),
|
| 556 |
-
)
|
| 557 |
-
conn.commit()
|
| 558 |
-
except Exception:
|
| 559 |
-
logger.error(
|
| 560 |
-
"Failed to mirror monthly_counts to Postgres for api_key=%s month=%s -- "
|
| 561 |
-
"arf-gateway's quota check will undercount usage for this key until this "
|
| 562 |
-
"is resolved.",
|
| 563 |
-
api_key, month, exc_info=True,
|
| 564 |
-
)
|
| 565 |
-
|
| 566 |
# --------------------------------------------------------------------------
|
| 567 |
# Idempotency handling (unchanged)
|
| 568 |
# --------------------------------------------------------------------------
|
|
@@ -597,8 +305,6 @@ class UsageTracker:
|
|
| 597 |
if not quota_ok:
|
| 598 |
return False, None
|
| 599 |
|
| 600 |
-
self._record_pg_monthly_count(record.api_key, month)
|
| 601 |
-
|
| 602 |
try:
|
| 603 |
with self._get_conn() as conn:
|
| 604 |
conn.execute(
|
|
@@ -607,8 +313,8 @@ class UsageTracker:
|
|
| 607 |
processing_ms, idempotency_key)
|
| 608 |
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
| 609 |
(record.api_key, record.tier.value, record.timestamp, record.endpoint,
|
| 610 |
-
json.dumps(record.request_body
|
| 611 |
-
json.dumps(record.response
|
| 612 |
record.error, record.processing_ms, idempotency_key)
|
| 613 |
)
|
| 614 |
conn.commit()
|
|
@@ -621,40 +327,6 @@ class UsageTracker:
|
|
| 621 |
self._mark_idempotent_key_used(idempotency_key)
|
| 622 |
return True, None
|
| 623 |
|
| 624 |
-
def _insert_audit_log(self, record: UsageRecord) -> None:
|
| 625 |
-
"""Insert a standalone usage_log row for a call whose quota was
|
| 626 |
-
already consumed at request time (see consume_quota_and_log) --
|
| 627 |
-
used by routes_governance.py's background tasks to record the
|
| 628 |
-
response body once it's known, under a distinct endpoint suffix
|
| 629 |
-
(e.g. ".../response"). Best-effort and logged, not raised, for the
|
| 630 |
-
same reason _record_pg_monthly_count is: this runs after the
|
| 631 |
-
response has already been sent to the caller, so it must not
|
| 632 |
-
surface as a request failure -- a background task exception here
|
| 633 |
-
is otherwise swallowed silently. `record.tier` is None at both real
|
| 634 |
-
call sites (tier only matters for quota consumption, already done
|
| 635 |
-
by the earlier consume_quota_and_log call for the same request),
|
| 636 |
-
but usage_log.tier is NOT NULL, so an absent tier is recorded as
|
| 637 |
-
"unknown" rather than raising or silently guessing a real tier."""
|
| 638 |
-
tier_value = record.tier.value if record.tier else "unknown"
|
| 639 |
-
try:
|
| 640 |
-
with self._get_conn() as conn:
|
| 641 |
-
conn.execute(
|
| 642 |
-
"""INSERT INTO usage_log
|
| 643 |
-
(api_key, tier, timestamp, endpoint, request_body, response, error,
|
| 644 |
-
processing_ms, idempotency_key)
|
| 645 |
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
| 646 |
-
(record.api_key, tier_value, record.timestamp, record.endpoint,
|
| 647 |
-
json.dumps(record.request_body, default=str) if record.request_body else None,
|
| 648 |
-
json.dumps(record.response, default=str) if record.response else None,
|
| 649 |
-
record.error, record.processing_ms, None)
|
| 650 |
-
)
|
| 651 |
-
conn.commit()
|
| 652 |
-
except Exception:
|
| 653 |
-
logger.error(
|
| 654 |
-
"Failed to insert audit log for api_key=%s endpoint=%s",
|
| 655 |
-
record.api_key, record.endpoint, exc_info=True,
|
| 656 |
-
)
|
| 657 |
-
|
| 658 |
# --------------------------------------------------------------------------
|
| 659 |
# Legacy interface (kept for compatibility)
|
| 660 |
# --------------------------------------------------------------------------
|
|
@@ -722,17 +394,6 @@ class UsageTracker:
|
|
| 722 |
# --------------------------------------------------------------------------
|
| 723 |
# Global instance and FastAPI dependency
|
| 724 |
# --------------------------------------------------------------------------
|
| 725 |
-
# Rebound by init_tracker() during the app lifespan, which means consumers
|
| 726 |
-
# MUST reach it through the module -- `from app.core import usage_tracker`,
|
| 727 |
-
# then `usage_tracker.tracker`. Never `from app.core.usage_tracker import
|
| 728 |
-
# tracker`: that copies the *binding* (None) at import time, and init_tracker
|
| 729 |
-
# rebinding this global does not update the importer's copy. Five modules
|
| 730 |
-
# did exactly that (main, routes_admin, routes_incidents, routes_payments,
|
| 731 |
-
# routes_users) and every one of them saw None forever -- silently skipping
|
| 732 |
-
# metering and disabling signup/checkout, and crashing the Render deploy
|
| 733 |
-
# outright once main.py called a method on it. Functions defined *in* this
|
| 734 |
-
# module (enforce_quota, update_key_tier*) are safe to import by name: they
|
| 735 |
-
# resolve `tracker` here, at call time.
|
| 736 |
tracker: Optional[UsageTracker] = None
|
| 737 |
|
| 738 |
|
|
@@ -747,36 +408,20 @@ def update_key_tier(api_key: str, new_tier: Tier) -> bool:
|
|
| 747 |
return tracker.update_api_key_tier(api_key, new_tier)
|
| 748 |
|
| 749 |
|
| 750 |
-
def
|
| 751 |
-
if tracker is None:
|
| 752 |
-
return False
|
| 753 |
-
return tracker.update_tier_by_tenant_id(tenant_id, new_tier)
|
| 754 |
-
|
| 755 |
-
|
| 756 |
-
def _extract_api_key(request: Request, api_key: str = None) -> str:
|
| 757 |
-
if api_key:
|
| 758 |
-
return api_key
|
| 759 |
-
auth_header = request.headers.get("Authorization")
|
| 760 |
-
if auth_header and auth_header.startswith("Bearer "):
|
| 761 |
-
return auth_header[7:]
|
| 762 |
-
return request.query_params.get("api_key")
|
| 763 |
-
|
| 764 |
-
|
| 765 |
-
async def resolve_api_key_identity(request: Request, api_key: str = None):
|
| 766 |
"""
|
| 767 |
-
FastAPI dependency that
|
| 768 |
-
to request state, without enforcing monthly quota.
|
| 769 |
-
|
| 770 |
-
Deliberately separate from `enforce_quota`: a caller whose quota is
|
| 771 |
-
already exhausted must still be able to reach an endpoint like
|
| 772 |
-
`/payments/create-checkout-session` (upgrading tier is often exactly
|
| 773 |
-
what a rate-limited caller is trying to do) -- gating that path behind
|
| 774 |
-
`enforce_quota` would 429 the one action that lets them fix it.
|
| 775 |
"""
|
| 776 |
if tracker is None:
|
| 777 |
raise HTTPException(status_code=503, detail="Usage tracking service not initialised.")
|
| 778 |
|
| 779 |
-
api_key
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 780 |
if not api_key:
|
| 781 |
raise HTTPException(status_code=401, detail="Missing API key")
|
| 782 |
|
|
@@ -784,6 +429,11 @@ async def resolve_api_key_identity(request: Request, api_key: str = None):
|
|
| 784 |
if tier is None:
|
| 785 |
raise HTTPException(status_code=403, detail="Invalid or inactive API key")
|
| 786 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 787 |
tenant_id = tracker.get_tenant_id(api_key)
|
| 788 |
if not tenant_id:
|
| 789 |
raise HTTPException(status_code=403, detail="API key not associated with a tenant")
|
|
@@ -792,18 +442,4 @@ async def resolve_api_key_identity(request: Request, api_key: str = None):
|
|
| 792 |
request.state.tier = tier
|
| 793 |
request.state.tenant_id = tenant_id
|
| 794 |
|
| 795 |
-
return {"api_key": api_key, "tier": tier, "tenant_id": tenant_id}
|
| 796 |
-
|
| 797 |
-
|
| 798 |
-
async def enforce_quota(request: Request, api_key: str = None):
|
| 799 |
-
"""
|
| 800 |
-
FastAPI dependency that enforces quota and attaches tenant_id to request state.
|
| 801 |
-
"""
|
| 802 |
-
identity = await resolve_api_key_identity(request, api_key)
|
| 803 |
-
api_key, tier, tenant_id = identity["api_key"], identity["tier"], identity["tenant_id"]
|
| 804 |
-
|
| 805 |
-
remaining = tracker.get_remaining_quota(api_key, tier)
|
| 806 |
-
if remaining is not None and remaining <= 0:
|
| 807 |
-
raise HTTPException(status_code=429, detail="Monthly evaluation quota exceeded")
|
| 808 |
-
|
| 809 |
return {"api_key": api_key, "tier": tier, "tenant_id": tenant_id, "remaining": remaining}
|
|
|
|
| 4 |
|
| 5 |
Extended for multi‑tenancy: each API key is linked to a tenant ID.
|
| 6 |
Tenant ID is stored in the `api_keys` table and used for resource isolation.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7 |
"""
|
|
|
|
|
|
|
| 8 |
import json
|
|
|
|
|
|
|
|
|
|
| 9 |
import sqlite3
|
| 10 |
import threading
|
| 11 |
import time
|
|
|
|
|
|
|
|
|
|
| 12 |
from contextlib import contextmanager
|
| 13 |
from datetime import datetime, timedelta
|
| 14 |
from dataclasses import dataclass
|
|
|
|
| 68 |
processing_ms: Optional[float] = None
|
| 69 |
|
| 70 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 71 |
class UsageTracker:
|
| 72 |
"""
|
| 73 |
Thread‑safe usage tracker with atomic quota consumption and idempotency.
|
| 74 |
Extended to support tenant isolation: each API key is linked to a tenant.
|
| 75 |
"""
|
| 76 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 77 |
def __init__(self, db_path: str = "arf_usage.db",
|
| 78 |
+
redis_url: Optional[str] = None):
|
|
|
|
| 79 |
self.db_path = db_path
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 80 |
self._local = threading.local()
|
| 81 |
self._init_db()
|
| 82 |
|
|
|
|
| 86 |
elif redis_url:
|
| 87 |
raise ImportError("Redis client not installed. Run: pip install redis")
|
| 88 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 89 |
@contextmanager
|
| 90 |
def _get_conn(self):
|
| 91 |
+
"""Get a thread‑local SQLite connection with WAL and immediate transactions."""
|
|
|
|
|
|
|
|
|
|
| 92 |
if not hasattr(self._local, "conn"):
|
| 93 |
self._local.conn = sqlite3.connect(
|
| 94 |
self.db_path, check_same_thread=False, isolation_level=None)
|
|
|
|
| 96 |
self._local.conn.execute("PRAGMA journal_mode=WAL")
|
| 97 |
yield self._local.conn
|
| 98 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 99 |
def _init_db(self):
|
| 100 |
+
"""Initialise SQLite tables with tenant_id support."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 101 |
with self._get_conn() as conn:
|
| 102 |
+
# Modified: api_keys now has tenant_id column
|
| 103 |
+
conn.execute("""
|
| 104 |
+
CREATE TABLE IF NOT EXISTS api_keys (
|
| 105 |
+
key TEXT PRIMARY KEY,
|
| 106 |
+
tenant_id TEXT NOT NULL,
|
| 107 |
+
tier TEXT NOT NULL,
|
| 108 |
+
created_at REAL NOT NULL,
|
| 109 |
+
last_used_at REAL,
|
| 110 |
+
is_active INTEGER DEFAULT 1
|
| 111 |
+
)
|
| 112 |
+
""")
|
| 113 |
conn.execute("""
|
| 114 |
CREATE TABLE IF NOT EXISTS usage_log (
|
| 115 |
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
|
|
| 152 |
Register a new API key for a given tenant.
|
| 153 |
|
| 154 |
Args:
|
| 155 |
+
key: The API key (plain text, will be hashed in production).
|
| 156 |
tenant_id: UUID of the tenant (must already exist in main DB).
|
| 157 |
tier: Initial tier for the key.
|
| 158 |
|
| 159 |
Returns:
|
| 160 |
True if key was created (or already exists for the same tenant).
|
| 161 |
"""
|
| 162 |
+
with self._get_conn() as conn:
|
| 163 |
+
row = conn.execute(
|
| 164 |
+
"SELECT key FROM api_keys WHERE key = ?", (key,)).fetchone()
|
|
|
|
|
|
|
| 165 |
if row:
|
| 166 |
# Key already exists – ensure it belongs to the same tenant
|
| 167 |
+
existing_tenant = conn.execute(
|
| 168 |
+
"SELECT tenant_id FROM api_keys WHERE key = ?", (key,)).fetchone()
|
| 169 |
+
if existing_tenant["tenant_id"] != tenant_id:
|
| 170 |
raise ValueError(f"Key {key[:8]}... already belongs to a different tenant.")
|
|
|
|
| 171 |
return True
|
| 172 |
+
conn.execute(
|
| 173 |
+
"INSERT INTO api_keys (key, tenant_id, tier, created_at, is_active) VALUES (?, ?, ?, ?, ?)",
|
| 174 |
+
(key, tenant_id, tier.value, time.time(), 1)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 175 |
)
|
| 176 |
conn.commit()
|
| 177 |
return True
|
| 178 |
|
| 179 |
def get_tier(self, api_key: str) -> Optional[Tier]:
|
| 180 |
"""Return the tier for a given API key, or None if key invalid/inactive."""
|
| 181 |
+
with self._get_conn() as conn:
|
| 182 |
+
row = conn.execute(
|
| 183 |
+
"SELECT tier FROM api_keys WHERE key = ? AND is_active = 1",
|
| 184 |
+
(api_key,)
|
| 185 |
+
).fetchone()
|
| 186 |
+
if not row:
|
| 187 |
+
return None
|
| 188 |
+
return Tier(row["tier"])
|
| 189 |
|
| 190 |
def get_tenant_id(self, api_key: str) -> Optional[str]:
|
| 191 |
"""Return the tenant ID associated with the API key, or None if key invalid."""
|
| 192 |
+
with self._get_conn() as conn:
|
| 193 |
+
row = conn.execute(
|
| 194 |
+
"SELECT tenant_id FROM api_keys WHERE key = ? AND is_active = 1",
|
| 195 |
+
(api_key,)
|
| 196 |
+
).fetchone()
|
| 197 |
+
if not row:
|
| 198 |
+
return None
|
| 199 |
+
return row["tenant_id"]
|
| 200 |
|
| 201 |
def update_api_key_tier(self, api_key: str, new_tier: Tier) -> bool:
|
| 202 |
"""Update the tier of an existing API key. Returns True if successful."""
|
| 203 |
+
with self._get_conn() as conn:
|
| 204 |
+
row = conn.execute(
|
| 205 |
+
"SELECT key FROM api_keys WHERE key = ?", (api_key,)).fetchone()
|
|
|
|
|
|
|
| 206 |
if not row:
|
|
|
|
| 207 |
return False
|
| 208 |
+
conn.execute(
|
| 209 |
+
"UPDATE api_keys SET tier = ? WHERE key = ?",
|
| 210 |
+
(new_tier.value, api_key))
|
| 211 |
conn.commit()
|
| 212 |
return True
|
| 213 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 214 |
# --------------------------------------------------------------------------
|
| 215 |
# Atomic quota consumption (unchanged, but uses api_key which links to tenant)
|
| 216 |
# --------------------------------------------------------------------------
|
|
|
|
| 271 |
result = self._redis_client.eval(lua_script, 1, redis_key, limit)
|
| 272 |
return result == 1
|
| 273 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 274 |
# --------------------------------------------------------------------------
|
| 275 |
# Idempotency handling (unchanged)
|
| 276 |
# --------------------------------------------------------------------------
|
|
|
|
| 305 |
if not quota_ok:
|
| 306 |
return False, None
|
| 307 |
|
|
|
|
|
|
|
| 308 |
try:
|
| 309 |
with self._get_conn() as conn:
|
| 310 |
conn.execute(
|
|
|
|
| 313 |
processing_ms, idempotency_key)
|
| 314 |
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
| 315 |
(record.api_key, record.tier.value, record.timestamp, record.endpoint,
|
| 316 |
+
json.dumps(record.request_body) if record.request_body else None,
|
| 317 |
+
json.dumps(record.response) if record.response else None,
|
| 318 |
record.error, record.processing_ms, idempotency_key)
|
| 319 |
)
|
| 320 |
conn.commit()
|
|
|
|
| 327 |
self._mark_idempotent_key_used(idempotency_key)
|
| 328 |
return True, None
|
| 329 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 330 |
# --------------------------------------------------------------------------
|
| 331 |
# Legacy interface (kept for compatibility)
|
| 332 |
# --------------------------------------------------------------------------
|
|
|
|
| 394 |
# --------------------------------------------------------------------------
|
| 395 |
# Global instance and FastAPI dependency
|
| 396 |
# --------------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 397 |
tracker: Optional[UsageTracker] = None
|
| 398 |
|
| 399 |
|
|
|
|
| 408 |
return tracker.update_api_key_tier(api_key, new_tier)
|
| 409 |
|
| 410 |
|
| 411 |
+
async def enforce_quota(request: Request, api_key: str = None):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 412 |
"""
|
| 413 |
+
FastAPI dependency that enforces quota and attaches tenant_id to request state.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 414 |
"""
|
| 415 |
if tracker is None:
|
| 416 |
raise HTTPException(status_code=503, detail="Usage tracking service not initialised.")
|
| 417 |
|
| 418 |
+
if api_key is None:
|
| 419 |
+
auth_header = request.headers.get("Authorization")
|
| 420 |
+
if auth_header and auth_header.startswith("Bearer "):
|
| 421 |
+
api_key = auth_header[7:]
|
| 422 |
+
else:
|
| 423 |
+
api_key = request.query_params.get("api_key")
|
| 424 |
+
|
| 425 |
if not api_key:
|
| 426 |
raise HTTPException(status_code=401, detail="Missing API key")
|
| 427 |
|
|
|
|
| 429 |
if tier is None:
|
| 430 |
raise HTTPException(status_code=403, detail="Invalid or inactive API key")
|
| 431 |
|
| 432 |
+
remaining = tracker.get_remaining_quota(api_key, tier)
|
| 433 |
+
if remaining is not None and remaining <= 0:
|
| 434 |
+
raise HTTPException(status_code=429, detail="Monthly evaluation quota exceeded")
|
| 435 |
+
|
| 436 |
+
# Retrieve tenant_id
|
| 437 |
tenant_id = tracker.get_tenant_id(api_key)
|
| 438 |
if not tenant_id:
|
| 439 |
raise HTTPException(status_code=403, detail="API key not associated with a tenant")
|
|
|
|
| 442 |
request.state.tier = tier
|
| 443 |
request.state.tenant_id = tenant_id
|
| 444 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 445 |
return {"api_key": api_key, "tier": tier, "tenant_id": tenant_id, "remaining": remaining}
|
app/database/models_intents.py
CHANGED
|
@@ -3,18 +3,14 @@ Database models for the ARF API Control Plane.
|
|
| 3 |
|
| 4 |
This module defines the SQLAlchemy ORM models for:
|
| 5 |
- Tenants (multi‑tenant isolation root)
|
|
|
|
|
|
|
| 6 |
- Intents (InfrastructureIntent evaluations)
|
| 7 |
- Outcomes (recorded results of executed intents)
|
| 8 |
- Beta state (conjugate Bayesian posteriors per tenant and category)
|
| 9 |
- Audit logs (immutable decision records for compliance)
|
| 10 |
|
| 11 |
All tables include a `tenant_id` column to enforce data partitioning.
|
| 12 |
-
|
| 13 |
-
API keys and usage/quota logs are tracked separately in
|
| 14 |
-
`app/core/usage_tracker.py` (SQLite, pepper-HMAC hashed) -- an
|
| 15 |
-
`APIKeyDB`/`UsageLogDB` pair used to live here as a second, unused,
|
| 16 |
-
plaintext-keyed parallel schema; removed 2026-08-23 since nothing
|
| 17 |
-
referenced them.
|
| 18 |
"""
|
| 19 |
|
| 20 |
import uuid
|
|
@@ -50,11 +46,74 @@ class TenantDB(Base):
|
|
| 50 |
created_by = Column(String(128), nullable=True)
|
| 51 |
|
| 52 |
# Relationships
|
|
|
|
| 53 |
intents = relationship("IntentDB", back_populates="tenant")
|
| 54 |
beta_states = relationship("BetaStateDB", back_populates="tenant")
|
| 55 |
audit_logs = relationship("DecisionAuditLogDB", back_populates="tenant")
|
| 56 |
|
| 57 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 58 |
# ============================================================================
|
| 59 |
# Intents (evaluations) – now tenant‑scoped
|
| 60 |
# ============================================================================
|
|
|
|
| 3 |
|
| 4 |
This module defines the SQLAlchemy ORM models for:
|
| 5 |
- Tenants (multi‑tenant isolation root)
|
| 6 |
+
- API keys (per‑tenant, tier‑based)
|
| 7 |
+
- Usage logs (immutable records of API calls)
|
| 8 |
- Intents (InfrastructureIntent evaluations)
|
| 9 |
- Outcomes (recorded results of executed intents)
|
| 10 |
- Beta state (conjugate Bayesian posteriors per tenant and category)
|
| 11 |
- Audit logs (immutable decision records for compliance)
|
| 12 |
|
| 13 |
All tables include a `tenant_id` column to enforce data partitioning.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14 |
"""
|
| 15 |
|
| 16 |
import uuid
|
|
|
|
| 46 |
created_by = Column(String(128), nullable=True)
|
| 47 |
|
| 48 |
# Relationships
|
| 49 |
+
api_keys = relationship("APIKeyDB", back_populates="tenant", cascade="all, delete-orphan")
|
| 50 |
intents = relationship("IntentDB", back_populates="tenant")
|
| 51 |
beta_states = relationship("BetaStateDB", back_populates="tenant")
|
| 52 |
audit_logs = relationship("DecisionAuditLogDB", back_populates="tenant")
|
| 53 |
|
| 54 |
|
| 55 |
+
# ============================================================================
|
| 56 |
+
# API keys (extended with tenant_id)
|
| 57 |
+
# ============================================================================
|
| 58 |
+
|
| 59 |
+
class APIKeyDB(Base):
|
| 60 |
+
"""
|
| 61 |
+
Stores API keys for authentication and tiered quota. Each key belongs
|
| 62 |
+
to exactly one tenant. The `tier` determines monthly evaluation limits.
|
| 63 |
+
|
| 64 |
+
Attributes:
|
| 65 |
+
key (str): The hashed API key (primary key).
|
| 66 |
+
tenant_id (str): Foreign key to `tenants.id`.
|
| 67 |
+
tier (str): Tier enumeration value (free, pro, premium, enterprise).
|
| 68 |
+
created_at (datetime): UTC creation time.
|
| 69 |
+
last_used_at (datetime, optional): Timestamp of last successful request.
|
| 70 |
+
is_active (bool): Soft‑delete flag.
|
| 71 |
+
"""
|
| 72 |
+
__tablename__ = "api_keys"
|
| 73 |
+
|
| 74 |
+
key = Column(String(256), primary_key=True, index=True)
|
| 75 |
+
tenant_id = Column(String(64), ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False, index=True)
|
| 76 |
+
tier = Column(String(32), nullable=False)
|
| 77 |
+
created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False)
|
| 78 |
+
last_used_at = Column(DateTime, nullable=True)
|
| 79 |
+
is_active = Column(Boolean, default=True, nullable=False)
|
| 80 |
+
|
| 81 |
+
# Relationships
|
| 82 |
+
tenant = relationship("TenantDB", back_populates="api_keys")
|
| 83 |
+
usage_logs = relationship("UsageLogDB", back_populates="api_key_rel", cascade="all, delete-orphan")
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
# ============================================================================
|
| 87 |
+
# Usage logs – each API call
|
| 88 |
+
# ============================================================================
|
| 89 |
+
|
| 90 |
+
class UsageLogDB(Base):
|
| 91 |
+
"""
|
| 92 |
+
Immutable record of each API call for quota tracking and billing.
|
| 93 |
+
|
| 94 |
+
Attributes:
|
| 95 |
+
id (int): Primary key.
|
| 96 |
+
api_key (str): Foreign key to `api_keys.key`.
|
| 97 |
+
tier (str): Tier at the time of the call.
|
| 98 |
+
timestamp (float): Unix timestamp of the request.
|
| 99 |
+
endpoint (str): URL or route of the endpoint hit.
|
| 100 |
+
request_body (JSON, optional): Request payload (sanitised).
|
| 101 |
+
response (JSON, optional): Response metadata (e.g., status code).
|
| 102 |
+
"""
|
| 103 |
+
__tablename__ = "usage_logs"
|
| 104 |
+
|
| 105 |
+
id = Column(Integer, primary_key=True, index=True)
|
| 106 |
+
api_key = Column(String(256), ForeignKey("api_keys.key", ondelete="CASCADE"), nullable=False)
|
| 107 |
+
tier = Column(String(32), nullable=False)
|
| 108 |
+
timestamp = Column(Float, nullable=False)
|
| 109 |
+
endpoint = Column(String(512), nullable=True)
|
| 110 |
+
request_body = Column(JSON, nullable=True)
|
| 111 |
+
response = Column(JSON, nullable=True)
|
| 112 |
+
|
| 113 |
+
# Relationship back to API key
|
| 114 |
+
api_key_rel = relationship("APIKeyDB", back_populates="usage_logs")
|
| 115 |
+
|
| 116 |
+
|
| 117 |
# ============================================================================
|
| 118 |
# Intents (evaluations) – now tenant‑scoped
|
| 119 |
# ============================================================================
|
app/database/models_onchain.py
DELETED
|
@@ -1,54 +0,0 @@
|
|
| 1 |
-
"""Database models for on-chain governance attestations.
|
| 2 |
-
|
| 3 |
-
``RiskAttestation.rationale_hash`` (see arf-onchain's ``AttestationLib.sol``
|
| 4 |
-
and enterprise's ``arf_enterprise.onchain.attestation``) is a ``keccak256``
|
| 5 |
-
digest anchored on Monad -- the chain deliberately never stores the rationale
|
| 6 |
-
text itself, only its hash, to keep operational detail about a customer's
|
| 7 |
-
infrastructure off a public ledger. That means the text has to live
|
| 8 |
-
somewhere off-chain, keyed by the same hash, or the anchored hash proves
|
| 9 |
-
nothing: nobody could ever produce the preimage to check it against.
|
| 10 |
-
|
| 11 |
-
This table is that store. It is intentionally separate from
|
| 12 |
-
``DecisionAuditLogDB`` (``models_intents.py``) rather than an extension of
|
| 13 |
-
it: that table is written for every governance decision, on-chain or not,
|
| 14 |
-
and already has its own signature column for a different purpose (Ed25519
|
| 15 |
-
intent-signing, not the secp256k1 EIP-712 signature the guard verifies).
|
| 16 |
-
Conflating the two would mean a column that is only sometimes meaningful
|
| 17 |
-
depending on whether the decision was ever attested on-chain.
|
| 18 |
-
"""
|
| 19 |
-
|
| 20 |
-
import uuid
|
| 21 |
-
import datetime
|
| 22 |
-
|
| 23 |
-
from sqlalchemy import Column, String, DateTime, Text
|
| 24 |
-
|
| 25 |
-
from .base import Base
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
class OnchainRationaleDB(Base):
|
| 29 |
-
"""The plaintext preimage of an anchored ``rationale_hash``.
|
| 30 |
-
|
| 31 |
-
Keyed by the hash itself (unique, indexed) rather than by an
|
| 32 |
-
auto-incrementing id: a lookup always starts from a hash read off-chain
|
| 33 |
-
(from `DecisionAnchored` or `AttestationIssued`), never from a row id
|
| 34 |
-
nothing on-chain knows about.
|
| 35 |
-
|
| 36 |
-
No ``tenant_id`` / foreign key to ``tenants``: an on-chain agent is
|
| 37 |
-
identified by its wallet address, not by this service's tenant concept,
|
| 38 |
-
and the two are not yet bridged. `evaluator_address` and `agent_address`
|
| 39 |
-
are recorded instead so a row can still be attributed and audited
|
| 40 |
-
without assuming a tenant relationship that may not exist.
|
| 41 |
-
"""
|
| 42 |
-
|
| 43 |
-
__tablename__ = "onchain_rationales"
|
| 44 |
-
|
| 45 |
-
id = Column(String(64), primary_key=True, default=lambda: str(uuid.uuid4()))
|
| 46 |
-
rationale_hash = Column(
|
| 47 |
-
String(66), nullable=False, unique=True, index=True
|
| 48 |
-
) # "0x" + 64 hex chars
|
| 49 |
-
rationale = Column(Text, nullable=False)
|
| 50 |
-
agent_address = Column(String(42), nullable=True)
|
| 51 |
-
evaluator_address = Column(String(42), nullable=True)
|
| 52 |
-
created_at = Column(
|
| 53 |
-
DateTime, default=datetime.datetime.utcnow, nullable=False, index=True
|
| 54 |
-
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
app/main.py
CHANGED
|
@@ -25,7 +25,6 @@ All heavy components are loaded **lazily and best‑effort** – if a dependency
|
|
| 25 |
is missing the API continues to serve health‑check and status endpoints,
|
| 26 |
degrading gracefully rather than crashing.
|
| 27 |
"""
|
| 28 |
-
import hashlib
|
| 29 |
import logging
|
| 30 |
import os
|
| 31 |
import sys
|
|
@@ -35,9 +34,8 @@ import time as _time
|
|
| 35 |
from contextlib import asynccontextmanager
|
| 36 |
from typing import Dict
|
| 37 |
|
| 38 |
-
from fastapi import FastAPI
|
| 39 |
from fastapi.middleware.cors import CORSMiddleware
|
| 40 |
-
from fastapi.responses import JSONResponse
|
| 41 |
|
| 42 |
# ── Optional: Prometheus metrics ─────────────────────────────
|
| 43 |
try:
|
|
@@ -83,15 +81,13 @@ from agentic_reliability_framework.core.temporal_reliability import (
|
|
| 83 |
)
|
| 84 |
|
| 85 |
# ── Usage tracker ────────────────────────────────────────────
|
| 86 |
-
from app.core import
|
| 87 |
-
from app.core.usage_tracker import init_tracker, Tier
|
| 88 |
|
| 89 |
from app.api import (
|
| 90 |
routes_governance,
|
| 91 |
routes_history,
|
| 92 |
routes_incidents,
|
| 93 |
routes_intents,
|
| 94 |
-
routes_onchain,
|
| 95 |
routes_risk,
|
| 96 |
routes_memory,
|
| 97 |
routes_admin,
|
|
@@ -272,65 +268,14 @@ async def lifespan(app: FastAPI):
|
|
| 272 |
db_path=os.getenv("ARF_USAGE_DB_PATH", "arf_usage.db"),
|
| 273 |
redis_url=os.getenv("ARF_REDIS_URL"),
|
| 274 |
)
|
| 275 |
-
|
| 276 |
-
# Constructing the tracker no longer touches Postgres, so warm
|
| 277 |
-
# it deliberately here: a few seconds of retrying is worth it to
|
| 278 |
-
# come up with the api_keys schema ready.
|
| 279 |
-
#
|
| 280 |
-
# A failure is logged and survived, NOT raised. Everything above
|
| 281 |
-
# in this lifespan already degrades that way -- the Beta-state
|
| 282 |
-
# loader logs a warning on the identical error and continues --
|
| 283 |
-
# and the asymmetry here is what turned a database blip into a
|
| 284 |
-
# total outage: raising crash-loops the process, which on Render
|
| 285 |
-
# exhausts the port-detection window, fails the deploy, and
|
| 286 |
-
# leaves the previous release serving. Starting degraded means
|
| 287 |
-
# the health endpoint answers, the deploy succeeds, and API
|
| 288 |
-
# requests get a clean 503 from enforce_quota until the database
|
| 289 |
-
# is reachable -- at which point they recover with no redeploy.
|
| 290 |
-
postgres_ready = usage_tracker.tracker.warm_up()
|
| 291 |
-
if not postgres_ready:
|
| 292 |
-
logger.error(
|
| 293 |
-
"Usage tracker started WITHOUT a Postgres connection: api_keys "
|
| 294 |
-
"is unreachable, so API-key validation and quota enforcement "
|
| 295 |
-
"will fail (503) until it recovers. Check that DATABASE_URL's "
|
| 296 |
-
"host resolves from this service, and that the database is "
|
| 297 |
-
"running and in the same region."
|
| 298 |
-
)
|
| 299 |
-
|
| 300 |
-
# Seed initial API keys from environment variable (for testing
|
| 301 |
-
# / demo). Skipped entirely when Postgres is unreachable: every
|
| 302 |
-
# get_or_create_api_key below is a write to api_keys, so
|
| 303 |
-
# attempting it would raise straight back into the handler that
|
| 304 |
-
# kills the process -- reintroducing the crash loop the warm-up
|
| 305 |
-
# above exists to prevent.
|
| 306 |
api_keys_json = os.getenv("ARF_API_KEYS", "{}")
|
| 307 |
-
if not postgres_ready and api_keys_json not in ("", "{}"):
|
| 308 |
-
logger.warning(
|
| 309 |
-
"Skipping ARF_API_KEYS seeding: Postgres is unreachable. "
|
| 310 |
-
"Seeded keys will not exist until the database recovers "
|
| 311 |
-
"and the service is restarted."
|
| 312 |
-
)
|
| 313 |
-
api_keys_json = "{}"
|
| 314 |
try:
|
| 315 |
api_keys = json.loads(api_keys_json)
|
| 316 |
for key, tier_str in api_keys.items():
|
| 317 |
try:
|
| 318 |
tier = Tier(tier_str.lower())
|
| 319 |
-
|
| 320 |
-
# -- tier was silently accepted as tenant_id, so
|
| 321 |
-
# every seeded key of the same tier collided onto
|
| 322 |
-
# one bogus tenant_id. These are demo/env-seeded
|
| 323 |
-
# keys with no real TenantDB row, but each still
|
| 324 |
-
# needs its own tenant_id to avoid cross-key
|
| 325 |
-
# contamination in tenant-scoped state elsewhere
|
| 326 |
-
# (BetaStateDB, IntentDB, decision audit log). A
|
| 327 |
-
# fixed-length key prefix isn't safe here: keys
|
| 328 |
-
# generated elsewhere in this codebase share an
|
| 329 |
-
# 8-char prefix ("sk_live_"/"sk_free_"), so a
|
| 330 |
-
# prefix-based id would collide the same way the
|
| 331 |
-
# original bug did. Hash the whole key instead.
|
| 332 |
-
tenant_id = "env-seed-" + hashlib.sha256(key.encode()).hexdigest()[:16]
|
| 333 |
-
usage_tracker.tracker.get_or_create_api_key(key, tenant_id=tenant_id, tier=tier)
|
| 334 |
logger.info(f"Seeded API key for tier {tier.value}")
|
| 335 |
except ValueError:
|
| 336 |
logger.warning(
|
|
@@ -340,51 +285,15 @@ async def lifespan(app: FastAPI):
|
|
| 340 |
logger.warning(
|
| 341 |
"ARF_API_KEYS environment variable is not valid JSON; skipping seeding."
|
| 342 |
)
|
| 343 |
-
app.state.usage_tracker =
|
| 344 |
-
|
| 345 |
-
logger.info("✅ Usage tracker ready.")
|
| 346 |
-
else:
|
| 347 |
-
logger.warning("⚠️ Usage tracker started in degraded mode (no Postgres).")
|
| 348 |
except Exception as e:
|
| 349 |
-
# Still fail closed on genuine configuration errors -- a missing
|
| 350 |
-
# or too-short ARF_KEY_PEPPER, an unset DATABASE_URL. Those never
|
| 351 |
-
# self-resolve, and starting without them would mean serving with
|
| 352 |
-
# broken API-key hashing. Database *reachability* is handled
|
| 353 |
-
# above and no longer reaches here.
|
| 354 |
logger.critical(f"Failed to initialise usage tracker: {e}")
|
| 355 |
raise RuntimeError("Usage tracker initialisation failed") from e
|
| 356 |
else:
|
| 357 |
logger.info("Usage tracking disabled by ARF_USAGE_TRACKING=false.")
|
| 358 |
app.state.usage_tracker = None
|
| 359 |
|
| 360 |
-
# ── 6b. Enterprise execution approval ledger (optional) ───
|
| 361 |
-
# Singleton for the same reason usage_tracker/risk_engine are: a fresh
|
| 362 |
-
# PostgresStore() per request would open a brand-new DB connection (and
|
| 363 |
-
# re-run its schema check) on every call to POST
|
| 364 |
-
# /intents/{id}/execute instead of reusing one across requests handled
|
| 365 |
-
# by the same worker. Only initialised when execution is actually
|
| 366 |
-
# opted into (ARF_ENABLE_EXECUTION=true) and arf_enterprise is
|
| 367 |
-
# importable -- always sets app.state.approval_store (to None if
|
| 368 |
-
# either condition isn't met) so downstream code never needs a
|
| 369 |
-
# hasattr/getattr guard.
|
| 370 |
-
app.state.approval_store = None
|
| 371 |
-
if os.getenv("ARF_ENABLE_EXECUTION", "false").lower() == "true":
|
| 372 |
-
try:
|
| 373 |
-
from arf_enterprise.store import ApprovalStore, PostgresStore
|
| 374 |
-
app.state.approval_store = ApprovalStore(PostgresStore())
|
| 375 |
-
logger.info("✅ Enterprise execution approval ledger ready.")
|
| 376 |
-
except ImportError:
|
| 377 |
-
logger.warning(
|
| 378 |
-
"ARF_ENABLE_EXECUTION=true but arf_enterprise is not installed; "
|
| 379 |
-
"POST /intents/{id}/execute will return 501."
|
| 380 |
-
)
|
| 381 |
-
except Exception as e:
|
| 382 |
-
logger.error(
|
| 383 |
-
"Failed to initialise the enterprise approval ledger (%s); "
|
| 384 |
-
"POST /intents/{id}/execute will fall back to boolean-trust "
|
| 385 |
-
"mode for approvals rather than failing startup entirely.", e
|
| 386 |
-
)
|
| 387 |
-
|
| 388 |
# ── 7. Wilson confidence monitor ──────────────────────────
|
| 389 |
try:
|
| 390 |
from app.services.wilson_monitor import update as wilson_update
|
|
@@ -447,19 +356,6 @@ def create_app() -> FastAPI:
|
|
| 447 |
)
|
| 448 |
logger.debug("CORS middleware configured")
|
| 449 |
|
| 450 |
-
# ── Generic exception handler ────────────────────────────
|
| 451 |
-
# Defense-in-depth for anything that escapes a route's own try/except
|
| 452 |
-
# uncaught (HTTPException instances are unaffected -- FastAPI's own,
|
| 453 |
-
# more specific handler for those still takes precedence). Logs the
|
| 454 |
-
# real exception server-side and returns a generic message: routes
|
| 455 |
-
# that catch their own exceptions have each been fixed to do the same,
|
| 456 |
-
# but this exists so a bug that skips that pattern doesn't leak raw
|
| 457 |
-
# exception text (paths, internals, third-party SDK details) to callers.
|
| 458 |
-
@app.exception_handler(Exception)
|
| 459 |
-
async def unhandled_exception_handler(request: Request, exc: Exception):
|
| 460 |
-
logger.exception("Unhandled exception on %s %s", request.method, request.url.path)
|
| 461 |
-
return JSONResponse(status_code=500, content={"detail": "Internal server error"})
|
| 462 |
-
|
| 463 |
# ── Rate limiter ──────────────────────────────────────────
|
| 464 |
if SLOWAPI_AVAILABLE:
|
| 465 |
app.state.limiter = limiter
|
|
@@ -492,9 +388,6 @@ def create_app() -> FastAPI:
|
|
| 492 |
app.include_router(
|
| 493 |
routes_governance.router, prefix="/api/v1", tags=["governance"]
|
| 494 |
)
|
| 495 |
-
app.include_router(
|
| 496 |
-
routes_onchain.router, prefix="/api/v1", tags=["onchain"]
|
| 497 |
-
)
|
| 498 |
app.include_router(
|
| 499 |
routes_memory.router, prefix="/v1/memory", tags=["memory"]
|
| 500 |
)
|
|
|
|
| 25 |
is missing the API continues to serve health‑check and status endpoints,
|
| 26 |
degrading gracefully rather than crashing.
|
| 27 |
"""
|
|
|
|
| 28 |
import logging
|
| 29 |
import os
|
| 30 |
import sys
|
|
|
|
| 34 |
from contextlib import asynccontextmanager
|
| 35 |
from typing import Dict
|
| 36 |
|
| 37 |
+
from fastapi import FastAPI
|
| 38 |
from fastapi.middleware.cors import CORSMiddleware
|
|
|
|
| 39 |
|
| 40 |
# ── Optional: Prometheus metrics ─────────────────────────────
|
| 41 |
try:
|
|
|
|
| 81 |
)
|
| 82 |
|
| 83 |
# ── Usage tracker ────────────────────────────────────────────
|
| 84 |
+
from app.core.usage_tracker import init_tracker, tracker, Tier
|
|
|
|
| 85 |
|
| 86 |
from app.api import (
|
| 87 |
routes_governance,
|
| 88 |
routes_history,
|
| 89 |
routes_incidents,
|
| 90 |
routes_intents,
|
|
|
|
| 91 |
routes_risk,
|
| 92 |
routes_memory,
|
| 93 |
routes_admin,
|
|
|
|
| 268 |
db_path=os.getenv("ARF_USAGE_DB_PATH", "arf_usage.db"),
|
| 269 |
redis_url=os.getenv("ARF_REDIS_URL"),
|
| 270 |
)
|
| 271 |
+
# Seed initial API keys from environment variable (for testing / demo)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 272 |
api_keys_json = os.getenv("ARF_API_KEYS", "{}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 273 |
try:
|
| 274 |
api_keys = json.loads(api_keys_json)
|
| 275 |
for key, tier_str in api_keys.items():
|
| 276 |
try:
|
| 277 |
tier = Tier(tier_str.lower())
|
| 278 |
+
tracker.get_or_create_api_key(key, tier)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 279 |
logger.info(f"Seeded API key for tier {tier.value}")
|
| 280 |
except ValueError:
|
| 281 |
logger.warning(
|
|
|
|
| 285 |
logger.warning(
|
| 286 |
"ARF_API_KEYS environment variable is not valid JSON; skipping seeding."
|
| 287 |
)
|
| 288 |
+
app.state.usage_tracker = tracker
|
| 289 |
+
logger.info("✅ Usage tracker ready.")
|
|
|
|
|
|
|
|
|
|
| 290 |
except Exception as e:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 291 |
logger.critical(f"Failed to initialise usage tracker: {e}")
|
| 292 |
raise RuntimeError("Usage tracker initialisation failed") from e
|
| 293 |
else:
|
| 294 |
logger.info("Usage tracking disabled by ARF_USAGE_TRACKING=false.")
|
| 295 |
app.state.usage_tracker = None
|
| 296 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 297 |
# ── 7. Wilson confidence monitor ──────────────────────────
|
| 298 |
try:
|
| 299 |
from app.services.wilson_monitor import update as wilson_update
|
|
|
|
| 356 |
)
|
| 357 |
logger.debug("CORS middleware configured")
|
| 358 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 359 |
# ── Rate limiter ──────────────────────────────────────────
|
| 360 |
if SLOWAPI_AVAILABLE:
|
| 361 |
app.state.limiter = limiter
|
|
|
|
| 388 |
app.include_router(
|
| 389 |
routes_governance.router, prefix="/api/v1", tags=["governance"]
|
| 390 |
)
|
|
|
|
|
|
|
|
|
|
| 391 |
app.include_router(
|
| 392 |
routes_memory.router, prefix="/v1/memory", tags=["memory"]
|
| 393 |
)
|
deploy/kubernetes/arf-api/secret.yaml
CHANGED
|
@@ -1,16 +1,3 @@
|
|
| 1 |
-
# DO NOT apply this file as-is. Every value below is a placeholder, not a
|
| 2 |
-
# real secret -- ARF_INTERNAL_API_KEY in particular is a fixed, publicly
|
| 3 |
-
# visible string in this repo's git history. Applying it unmodified means
|
| 4 |
-
# the "secret" is a known value, not a secret.
|
| 5 |
-
#
|
| 6 |
-
# Generate real values instead, e.g.:
|
| 7 |
-
# kubectl create secret generic arf-api-secrets -n arf-system \
|
| 8 |
-
# --from-literal=DATABASE_URL=... \
|
| 9 |
-
# --from-literal=ARF_INTERNAL_API_KEY=$(openssl rand -hex 32) \
|
| 10 |
-
# --from-literal=ARF_API_KEYS='{}' \
|
| 11 |
-
# --from-literal=ARF_REDIS_URL=...
|
| 12 |
-
# or manage this via a secrets operator (External Secrets, Sealed Secrets,
|
| 13 |
-
# SOPS) rather than a plain committed manifest.
|
| 14 |
apiVersion: v1
|
| 15 |
kind: Secret
|
| 16 |
metadata:
|
|
@@ -18,7 +5,7 @@ metadata:
|
|
| 18 |
namespace: arf-system
|
| 19 |
type: Opaque
|
| 20 |
stringData:
|
| 21 |
-
#
|
| 22 |
DATABASE_URL: "postgresql://user:password@host:5432/arf"
|
| 23 |
ARF_INTERNAL_API_KEY: "change-me-to-a-strong-random-key"
|
| 24 |
ARF_API_KEYS: '{}'
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
apiVersion: v1
|
| 2 |
kind: Secret
|
| 3 |
metadata:
|
|
|
|
| 5 |
namespace: arf-system
|
| 6 |
type: Opaque
|
| 7 |
stringData:
|
| 8 |
+
# Replace these placeholder values with the actual secrets.
|
| 9 |
DATABASE_URL: "postgresql://user:password@host:5432/arf"
|
| 10 |
ARF_INTERNAL_API_KEY: "change-me-to-a-strong-random-key"
|
| 11 |
ARF_API_KEYS: '{}'
|
docs/authentication.md
CHANGED
|
@@ -2,50 +2,20 @@
|
|
| 2 |
|
| 3 |
This page describes how to authenticate with the ARF API.
|
| 4 |
|
| 5 |
-
Current status
|
| 6 |
-
|
| 7 |
-
- `routes_governance.py`
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
are rejected with 401 if the env var is unset, if the header is missing, or if it doesn't
|
| 11 |
-
match (constant-time comparison). This is the header arf-gateway injects when proxying to
|
| 12 |
-
this service. The last four were unauthenticated until this was fixed — see
|
| 13 |
-
`tests/test_deps.py` for the tests that verify the dependency itself actually rejects what
|
| 14 |
-
it should, not just that it's wired in.
|
| 15 |
-
- `routes_incidents.py`'s `POST /report_incident` requires the same `X-Internal-Key`
|
| 16 |
-
dependency as above — it did not until a later audit found it had none at all, unlike every
|
| 17 |
-
other route in this file, despite its own docstring saying it's meant for internal monitoring
|
| 18 |
-
tools only. Anyone could previously write arbitrary events into the incident history that
|
| 19 |
-
feeds the causal explainer and `GET /history`. That history is now also a bounded
|
| 20 |
-
`deque(maxlen=10_000)` (`app/core/storage.py`), not an unbounded list — the same audit found
|
| 21 |
-
`POST /report_incident` and `GET /history` were reading/writing two _different_ Python lists
|
| 22 |
-
with the same name, so `GET /history` had in fact always returned empty regardless of what was
|
| 23 |
-
reported; both routers now share the one list in `app.core.storage`.
|
| 24 |
-
- `routes_admin.py`: individual `/admin/*` endpoints require an `admin_key` query parameter,
|
| 25 |
-
verified against `ARF_ADMIN_API_KEY` (`app/api/deps.py`, or the local `verify_admin`
|
| 26 |
-
dependency in that router). Also fails closed if unset. Includes
|
| 27 |
-
`POST /admin/keys/{key_id}/rotate` — deactivates a key and issues a new one on the same
|
| 28 |
-
tenant/tier in one transaction, for when a key needs to be revoked without losing the
|
| 29 |
-
tenant's identity or history. `api_keys` itself lives in Postgres (`DATABASE_URL`), shared
|
| 30 |
-
with arf-gateway (both must use the identical `ARF_KEY_PEPPER`) — see the comments on those
|
| 31 |
-
two variables in `.env.example` for why.
|
| 32 |
-
- `routes_pricing.py`: individual `/pricing/*` endpoints require a real per-customer API key
|
| 33 |
-
(`Authorization: Bearer <key>` or `?api_key=`), verified against the tracked/tenant-scoped
|
| 34 |
-
`enforce_quota` dependency (`app/core/usage_tracker.py`) — a different mechanism from
|
| 35 |
-
`X-Internal-Key`, since pricing estimates are meant to be reachable by a customer directly,
|
| 36 |
-
not only via the gateway.
|
| 37 |
|
| 38 |
What the code provides
|
| 39 |
|
| 40 |
-
- `app/core/config.py` exposes an `api_key` setting read from `.env`, but no current route
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
|
|
|
| 44 |
|
| 45 |
Notes
|
| 46 |
|
| 47 |
- Tests run against a real Postgres connection (`tests/conftest.py`), not SQLite; see the top-level README's Tests section.
|
| 48 |
-
- `tests/conftest.py` globally overrides `verify_internal_key` for the test suite (so routes
|
| 49 |
-
behind it can be exercised without the gateway-injected header) — this means the app-level
|
| 50 |
-
test suite alone can't confirm the dependency actually fails closed. `tests/test_deps.py`
|
| 51 |
-
calls it directly, bypassing that override, specifically to verify that.
|
|
|
|
| 2 |
|
| 3 |
This page describes how to authenticate with the ARF API.
|
| 4 |
|
| 5 |
+
Current status (mixed — not all routers are protected)
|
| 6 |
+
|
| 7 |
+
- `routes_governance.py`: the entire router requires the `X-Internal-Key` header, verified against `ARF_INTERNAL_API_KEY` (`app/api/deps.py::verify_internal_key`). This fails closed — requests are rejected with 401 if the env var is unset. This is the header arf-gateway injects when proxying to this service.
|
| 8 |
+
- `routes_admin.py`: individual `/admin/*` endpoints require an `admin_key` query parameter, verified against `ARF_ADMIN_API_KEY` (`app/api/deps.py`, or the local `verify_admin` dependency in that router). Also fails closed if unset.
|
| 9 |
+
- `routes_risk.py`, `routes_intents.py`, `routes_history.py`, `routes_memory.py`: **no auth dependency at all**. If this service is reachable directly (e.g., its public HF Space URL) rather than only through arf-gateway, these routes are open to anyone.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 10 |
|
| 11 |
What the code provides
|
| 12 |
|
| 13 |
+
- `app/core/config.py` exposes an `api_key` setting read from `.env`, but no current route checks it — it is not the mechanism in use. The real mechanism is the two env vars above, checked in `app/api/deps.py`.
|
| 14 |
+
|
| 15 |
+
If you need to lock down the remaining unauthenticated routers
|
| 16 |
+
|
| 17 |
+
- Add `dependencies=[Depends(verify_internal_key)]` (or a purpose-built dependency) to the `APIRouter(...)` construction in the files listed above, following the pattern already used in `routes_governance.py`.
|
| 18 |
|
| 19 |
Notes
|
| 20 |
|
| 21 |
- Tests run against a real Postgres connection (`tests/conftest.py`), not SQLite; see the top-level README's Tests section.
|
|
|
|
|
|
|
|
|
|
|
|
docs/disaster-recovery.md
ADDED
|
@@ -0,0 +1,396 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ARF Disaster Recovery & Business Continuity Plan
|
| 2 |
+
|
| 3 |
+
**Version:** 2.0
|
| 4 |
+
**ARF Version:** v4.3.2
|
| 5 |
+
**Date:** July 8, 2026
|
| 6 |
+
**Classification:** Proprietary – Access‑Controlled
|
| 7 |
+
**Target Environment:** Kubernetes (AWS EKS), PostgreSQL (RDS), Redis (ElastiCache), S3
|
| 8 |
+
|
| 9 |
+
---
|
| 10 |
+
|
| 11 |
+
## 1. Executive Summary
|
| 12 |
+
|
| 13 |
+
This document defines the disaster recovery and business continuity procedures for the Agentic Reliability Framework (ARF). It is designed to ensure that the platform can recover from catastrophic failures while meeting stringent recovery objectives. The plan is grounded in the same Bayesian risk‑quantification principles that ARF applies to infrastructure governance, providing a mathematically rigorous framework for assessing and minimizing the probability of data loss and service unavailability.
|
| 14 |
+
|
| 15 |
+
### 1.1 Recovery Objectives
|
| 16 |
+
|
| 17 |
+
| Metric | Target | Rationale |
|
| 18 |
+
|--------|--------|-----------|
|
| 19 |
+
| **Recovery Point Objective (RPO)** | ≤ 5 minutes (PostgreSQL) | The conjugate Bayesian posteriors are updated on every outcome; a 5‑minute window limits the expected information loss to a negligible fraction of the total evidence. |
|
| 20 |
+
| **Recovery Time Objective (RTO)** | ≤ 15 minutes | The gateway and API can be redeployed automatically via Kubernetes; database restoration from a recent snapshot completes within this window. |
|
| 21 |
+
| **Maximum Acceptable Outage Probability** | ≤ 0.001 (99.9% availability) | For critical infrastructure, the service must be available at least 99.9% of the time, corresponding to an annual downtime of ≤ 8.76 hours. |
|
| 22 |
+
|
| 23 |
+
---
|
| 24 |
+
|
| 25 |
+
## 2. Data Topology and Fault Domains
|
| 26 |
+
|
| 27 |
+
### 2.1 Stateful Components
|
| 28 |
+
|
| 29 |
+
| Component | Data Stored | Consistency Model | Failure Impact |
|
| 30 |
+
|-----------|-------------|-------------------|----------------|
|
| 31 |
+
| **PostgreSQL (RDS)** | Tenant conjugate posteriors (`beta_state`), audit logs (`decision_audit_log`), intent records, outcome records | Strong (ACID) | Loss would revert all learned Bayesian priors and audit history. |
|
| 32 |
+
| **Redis (ElastiCache)** | Quota counters, rate‑limit state | Eventually consistent (AOF persistence) | Loss would reset monthly usage counters but not affect governance decisions. |
|
| 33 |
+
| **S3 (audit log exports)** | Daily exports of audit logs for long‑term compliance | Eventually consistent (immutable once written) | Loss would require reconstruction from PostgreSQL; data is redundant. |
|
| 34 |
+
|
| 35 |
+
### 2.2 Stateless Components (Kubernetes)
|
| 36 |
+
|
| 37 |
+
| Component | Replicas | Recovery Mechanism |
|
| 38 |
+
|-----------|----------|--------------------|
|
| 39 |
+
| `arf-api` | 3 (auto‑scaled to 10) | Re‑deployment from container image; ConfigMap and Secret mounted from Kubernetes. |
|
| 40 |
+
| `arf-gateway` | 3 (auto‑scaled to 10) | Re‑deployment; configuration via environment variables. |
|
| 41 |
+
|
| 42 |
+
---
|
| 43 |
+
|
| 44 |
+
## 3. Bayesian Risk Model for Recovery
|
| 45 |
+
|
| 46 |
+
We model the probability of a successful recovery as a Bayesian update problem. Let \(R\) be the event “successful recovery within RTO and RPO.” We assume a Beta prior for the probability of success, updated by the results of regular disaster recovery drills.
|
| 47 |
+
|
| 48 |
+
\[
|
| 49 |
+
P(R \mid \text{data}) \sim \text{Beta}(\alpha_0 + s,\ \beta_0 + f)
|
| 50 |
+
\]
|
| 51 |
+
|
| 52 |
+
where \(s\) is the number of successful drills and \(f\) the number of failed drills. We set a prior \(\text{Beta}(2,2)\) (weakly informative, mean 0.5). The posterior after \(n\) drills with \(s\) successes is used to compute the probability that the true recovery success rate exceeds the target of 0.99:
|
| 53 |
+
|
| 54 |
+
\[
|
| 55 |
+
\mathbb{P}(\theta_R > 0.99 \mid s, n) = 1 - I_{0.99}(\alpha_0 + s,\ \beta_0 + (n - s))
|
| 56 |
+
\]
|
| 57 |
+
|
| 58 |
+
This probability must exceed 0.95 before the platform can be considered production‑ready.
|
| 59 |
+
|
| 60 |
+
### 3.1 Example
|
| 61 |
+
|
| 62 |
+
After 10 successful drills and 0 failures, the posterior is \(\text{Beta}(12, 2)\). The probability that the true recovery rate exceeds 0.99 is
|
| 63 |
+
|
| 64 |
+
\[
|
| 65 |
+
1 - I_{0.99}(12, 2) \approx 0.9999,
|
| 66 |
+
\]
|
| 67 |
+
|
| 68 |
+
indicating very high confidence.
|
| 69 |
+
|
| 70 |
+
---
|
| 71 |
+
|
| 72 |
+
## 4. Backup Procedures
|
| 73 |
+
|
| 74 |
+
### 4.1 PostgreSQL – Automated RDS Snapshots
|
| 75 |
+
|
| 76 |
+
**Frequency:** Every 1 hour, retained for 30 days.
|
| 77 |
+
**Continuous WAL archiving:** Enabled for point‑in‑time recovery with 5‑minute granularity.
|
| 78 |
+
|
| 79 |
+
```bash
|
| 80 |
+
# Verify backup configuration
|
| 81 |
+
aws rds describe-db-instances \
|
| 82 |
+
--db-instance-identifier arf-postgres \
|
| 83 |
+
--query 'DBInstances[0].{BackupRetentionPeriod:BackupRetentionPeriod,PreferredBackupWindow:PreferredBackupWindow}'
|
| 84 |
+
|
| 85 |
+
# Verify WAL archiving
|
| 86 |
+
aws rds describe-db-log-files \
|
| 87 |
+
--db-instance-identifier arf-postgres \
|
| 88 |
+
--query 'DBLogFiles[?LogFileName==`wal_archive.log`]'
|
| 89 |
+
```
|
| 90 |
+
|
| 91 |
+
### 4.2 PostgreSQL – Pre‑Upgrade Snapshot
|
| 92 |
+
|
| 93 |
+
```bash
|
| 94 |
+
aws rds create-db-snapshot \
|
| 95 |
+
--db-instance-identifier arf-postgres \
|
| 96 |
+
--db-snapshot-identifier arf-pre-upgrade-$(date +%Y%m%d-%H%M)
|
| 97 |
+
```
|
| 98 |
+
|
| 99 |
+
### 4.3 Redis – AOF Snapshots
|
| 100 |
+
|
| 101 |
+
**Frequency:** Every 5 minutes via Kubernetes CronJob.
|
| 102 |
+
|
| 103 |
+
```yaml
|
| 104 |
+
apiVersion: batch/v1
|
| 105 |
+
kind: CronJob
|
| 106 |
+
metadata:
|
| 107 |
+
name: redis-backup
|
| 108 |
+
namespace: arf-system
|
| 109 |
+
spec:
|
| 110 |
+
schedule: "*/5 * * * *"
|
| 111 |
+
jobTemplate:
|
| 112 |
+
spec:
|
| 113 |
+
template:
|
| 114 |
+
spec:
|
| 115 |
+
containers:
|
| 116 |
+
- name: backup
|
| 117 |
+
image: amazon/aws-cli
|
| 118 |
+
command: ["/bin/sh", "-c"]
|
| 119 |
+
args:
|
| 120 |
+
- |
|
| 121 |
+
redis-cli -h $REDIS_HOST BGREWRITEAOF
|
| 122 |
+
sleep 10
|
| 123 |
+
aws s3 cp /data/appendonly.aof s3://arf-backups/redis/$(date +%Y%m%d-%H%M).aof
|
| 124 |
+
env:
|
| 125 |
+
- name: REDIS_HOST
|
| 126 |
+
valueFrom:
|
| 127 |
+
secretKeyRef:
|
| 128 |
+
name: arf-api-secrets
|
| 129 |
+
key: ARF_REDIS_URL
|
| 130 |
+
restartPolicy: OnFailure
|
| 131 |
+
```
|
| 132 |
+
|
| 133 |
+
### 4.4 Audit Log Exports to S3
|
| 134 |
+
|
| 135 |
+
**Frequency:** Daily, at midnight UTC.
|
| 136 |
+
|
| 137 |
+
```bash
|
| 138 |
+
#!/bin/bash
|
| 139 |
+
# arf-audit-export.sh
|
| 140 |
+
DATABASE_URL=$(kubectl get secret arf-api-secrets -n arf-system -o jsonpath='{.data.DATABASE_URL}' | base64 -d)
|
| 141 |
+
psql $DATABASE_URL -c "\copy (SELECT row_to_json(t) FROM decision_audit_log t WHERE timestamp > NOW() - INTERVAL '1 day') TO '/tmp/audit_export.json'"
|
| 142 |
+
aws s3 cp /tmp/audit_export.json s3://arf-backups/audit-logs/$(date +%Y%m%d).json
|
| 143 |
+
```
|
| 144 |
+
|
| 145 |
+
Deployed as a Kubernetes CronJob:
|
| 146 |
+
|
| 147 |
+
```yaml
|
| 148 |
+
apiVersion: batch/v1
|
| 149 |
+
kind: CronJob
|
| 150 |
+
metadata:
|
| 151 |
+
name: audit-log-export
|
| 152 |
+
namespace: arf-system
|
| 153 |
+
spec:
|
| 154 |
+
schedule: "0 0 * * *"
|
| 155 |
+
jobTemplate:
|
| 156 |
+
spec:
|
| 157 |
+
template:
|
| 158 |
+
spec:
|
| 159 |
+
containers:
|
| 160 |
+
- name: exporter
|
| 161 |
+
image: amazon/aws-cli
|
| 162 |
+
command: ["/bin/sh", "-c"]
|
| 163 |
+
args:
|
| 164 |
+
- |
|
| 165 |
+
psql $DATABASE_URL -c "\copy (SELECT row_to_json(t) FROM decision_audit_log t WHERE timestamp > NOW() - INTERVAL '1 day') TO '/tmp/audit_export.json'"
|
| 166 |
+
aws s3 cp /tmp/audit_export.json s3://arf-backups/audit-logs/$(date +%Y%m%d).json
|
| 167 |
+
env:
|
| 168 |
+
- name: DATABASE_URL
|
| 169 |
+
valueFrom:
|
| 170 |
+
secretKeyRef:
|
| 171 |
+
name: arf-api-secrets
|
| 172 |
+
key: DATABASE_URL
|
| 173 |
+
restartPolicy: OnFailure
|
| 174 |
+
```
|
| 175 |
+
|
| 176 |
+
5\. Restore Procedures
|
| 177 |
+
----------------------
|
| 178 |
+
|
| 179 |
+
### 5.1 PostgreSQL – Full Database Restore from Latest Snapshot
|
| 180 |
+
|
| 181 |
+
```bash
|
| 182 |
+
# 1. Restore the latest automated snapshot
|
| 183 |
+
LATEST_SNAPSHOT=$(aws rds describe-db-snapshots \
|
| 184 |
+
--db-instance-identifier arf-postgres \
|
| 185 |
+
--snapshot-type automated \
|
| 186 |
+
--query 'DBSnapshots[-1].DBSnapshotIdentifier' \
|
| 187 |
+
--output text)
|
| 188 |
+
|
| 189 |
+
aws rds restore-db-instance-from-db-snapshot \
|
| 190 |
+
--db-instance-identifier arf-postgres-restored \
|
| 191 |
+
--db-snapshot-identifier $LATEST_SNAPSHOT
|
| 192 |
+
|
| 193 |
+
# 2. Wait for instance availability
|
| 194 |
+
aws rds wait db-instance-available --db-instance-identifier arf-postgres-restored
|
| 195 |
+
|
| 196 |
+
# 3. Update the Kubernetes Secret with the new endpoint
|
| 197 |
+
NEW_ENDPOINT=$(aws rds describe-db-instances \
|
| 198 |
+
--db-instance-identifier arf-postgres-restored \
|
| 199 |
+
--query 'DBInstances[0].Endpoint.Address' \
|
| 200 |
+
--output text)
|
| 201 |
+
|
| 202 |
+
kubectl create secret generic arf-api-secrets \
|
| 203 |
+
--namespace arf-system \
|
| 204 |
+
--from-literal=DATABASE_URL="postgresql://user:password@${NEW_ENDPOINT}:5432/arf" \
|
| 205 |
+
--from-literal=ARF_INTERNAL_API_KEY="$(kubectl get secret arf-api-secrets -n arf-system -o jsonpath='{.data.ARF_INTERNAL_API_KEY}' | base64 -d)" \
|
| 206 |
+
--dry-run=client -o yaml | kubectl apply -f -
|
| 207 |
+
|
| 208 |
+
# 4. Restart API pods to reload configuration
|
| 209 |
+
kubectl rollout restart deployment/arf-api -n arf-system
|
| 210 |
+
```
|
| 211 |
+
|
| 212 |
+
### 5.2 PostgreSQL – Point‑in‑Time Recovery
|
| 213 |
+
|
| 214 |
+
```bash
|
| 215 |
+
RESTORE_TIME="2026-07-08T14:30:00Z"
|
| 216 |
+
|
| 217 |
+
aws rds restore-db-instance-to-point-in-time \
|
| 218 |
+
--source-db-instance-identifier arf-postgres \
|
| 219 |
+
--target-db-instance-identifier arf-postgres-pitr \
|
| 220 |
+
--restore-time $RESTORE_TIME
|
| 221 |
+
# Follow steps 2–4 from Section 5.1.
|
| 222 |
+
```
|
| 223 |
+
|
| 224 |
+
### 5.3 Redis – Restore from AOF
|
| 225 |
+
|
| 226 |
+
```bash
|
| 227 |
+
# 1. Scale down Redis to prevent writes during restoration
|
| 228 |
+
kubectl scale deployment arf-redis --replicas=0 -n arf-system
|
| 229 |
+
|
| 230 |
+
# 2. Copy the latest AOF file to the Redis data directory
|
| 231 |
+
LATEST_AOF=$(aws s3 ls s3://arf-backups/redis/ | sort | tail -1 | awk '{print $4}')
|
| 232 |
+
aws s3 cp s3://arf-backups/redis/$LATEST_AOF /data/appendonly.aof
|
| 233 |
+
|
| 234 |
+
# 3. Restart Redis
|
| 235 |
+
kubectl scale deployment arf-redis --replicas=1 -n arf-system
|
| 236 |
+
```
|
| 237 |
+
|
| 238 |
+
### 5.4 Full Cluster Recovery
|
| 239 |
+
|
| 240 |
+
```bash
|
| 241 |
+
# Apply all manifests in dependency order
|
| 242 |
+
kubectl apply -f deploy/kubernetes/arf-api/configmap.yaml
|
| 243 |
+
kubectl apply -f deploy/kubernetes/arf-api/secret.yaml
|
| 244 |
+
kubectl apply -f deploy/kubernetes/arf-api/networkpolicy.yaml
|
| 245 |
+
kubectl apply -f deploy/kubernetes/arf-api/deployment.yaml
|
| 246 |
+
kubectl apply -f deploy/kubernetes/arf-api/service.yaml
|
| 247 |
+
kubectl apply -f deploy/kubernetes/arf-api/hpa.yaml
|
| 248 |
+
kubectl apply -f deploy/kubernetes/arf-gateway/deployment.yaml
|
| 249 |
+
kubectl apply -f deploy/kubernetes/arf-gateway/service.yaml
|
| 250 |
+
kubectl apply -f deploy/kubernetes/arf-gateway/hpa.yaml
|
| 251 |
+
|
| 252 |
+
# Verify all pods are running
|
| 253 |
+
kubectl get pods -n arf-system
|
| 254 |
+
```
|
| 255 |
+
|
| 256 |
+
6\. Post‑Recovery Verification
|
| 257 |
+
------------------------------
|
| 258 |
+
|
| 259 |
+
### 6.1 Cryptographic Audit Log Integrity Check
|
| 260 |
+
|
| 261 |
+
This procedure uses the hash‑chained structure of the decision\_audit\_log to verify that no entries have been tampered with or lost during recovery.
|
| 262 |
+
|
| 263 |
+
```python
|
| 264 |
+
import hashlib
|
| 265 |
+
import psycopg2
|
| 266 |
+
|
| 267 |
+
def verify_audit_log_chain(db_url):
|
| 268 |
+
conn = psycopg2.connect(db_url)
|
| 269 |
+
cur = conn.cursor()
|
| 270 |
+
cur.execute("SELECT id, deterministic_id, context_hash, signature FROM decision_audit_log ORDER BY timestamp")
|
| 271 |
+
prev_hash = None
|
| 272 |
+
for row in cur.fetchall():
|
| 273 |
+
entry_id, det_id, ctx_hash, sig = row
|
| 274 |
+
# Recompute the intent hash from the stored fields
|
| 275 |
+
# (simplified; actual verification uses the full canonical JSON)
|
| 276 |
+
computed = hashlib.sha256(f"{det_id}:{ctx_hash}:{prev_hash or ''}".encode()).hexdigest()
|
| 277 |
+
# In production, the full Ed25519 signature verification would be performed.
|
| 278 |
+
prev_hash = computed
|
| 279 |
+
cur.close()
|
| 280 |
+
conn.close()
|
| 281 |
+
return True
|
| 282 |
+
```
|
| 283 |
+
|
| 284 |
+
### 6.2 Conjugate Posterior State Validation
|
| 285 |
+
|
| 286 |
+
```python
|
| 287 |
+
from agentic_reliability_framework.core.governance.risk_engine import ActionCategory
|
| 288 |
+
|
| 289 |
+
def validate_beta_state(risk_engine, expected_state):
|
| 290 |
+
for category, (alpha, beta) in expected_state.items():
|
| 291 |
+
actual = risk_engine._beta_stores["__default__"].get(category)
|
| 292 |
+
assert abs(actual[0] - alpha) < 1e-6, f"Alpha mismatch for {category}"
|
| 293 |
+
assert abs(actual[1] - beta) < 1e-6, f"Beta mismatch for {category}"
|
| 294 |
+
```
|
| 295 |
+
|
| 296 |
+
### 6.3 Automated Smoke Test
|
| 297 |
+
|
| 298 |
+
```bash
|
| 299 |
+
#!/bin/bash
|
| 300 |
+
# smoke-test.sh
|
| 301 |
+
GW_URL="http://arf-gateway.xxxxx.elb.amazonaws.com:8080"
|
| 302 |
+
TENANT="test-tenant"
|
| 303 |
+
|
| 304 |
+
# Health check
|
| 305 |
+
curl -s -f $GW_URL/health || { echo "Health check failed"; exit 1; }
|
| 306 |
+
|
| 307 |
+
# Evaluation
|
| 308 |
+
RESP=$(curl -s -X POST $GW_URL/api/v1/intents/evaluate \
|
| 309 |
+
-H "Content-Type: application/json" \
|
| 310 |
+
-H "X-Tenant-ID: $TENANT" \
|
| 311 |
+
-d '{"intent_type":"provision_resource","environment":"dev","resource_type":"database","region":"eastus","size":"Standard","estimated_cost":1200,"policy_violations":[],"requester":"alice","provenance":{},"configuration":{}}')
|
| 312 |
+
RISK=$(echo $RESP | jq -r '.risk_score')
|
| 313 |
+
if [ -z "$RISK" ] || [ "$RISK" = "null" ]; then
|
| 314 |
+
echo "Evaluation failed: $RESP"
|
| 315 |
+
exit 1
|
| 316 |
+
fi
|
| 317 |
+
echo "Smoke test passed. Risk score: $RISK"
|
| 318 |
+
```
|
| 319 |
+
|
| 320 |
+
7\. Chaos Engineering & Resilience Testing
|
| 321 |
+
------------------------------------------
|
| 322 |
+
|
| 323 |
+
### 7.1 Pod Deletion Test
|
| 324 |
+
|
| 325 |
+
```bash
|
| 326 |
+
# Randomly delete an API pod; verify that the service continues to serve requests without error.
|
| 327 |
+
kubectl delete pod -l app=arf-api -n arf-system --grace-period=1
|
| 328 |
+
sleep 5
|
| 329 |
+
# Run smoke test
|
| 330 |
+
./smoke-test.sh
|
| 331 |
+
```
|
| 332 |
+
|
| 333 |
+
### 7.2 Network Partition Simulation
|
| 334 |
+
|
| 335 |
+
```bash
|
| 336 |
+
# Apply a NetworkPolicy that temporarily denies ingress to the API from the gateway,
|
| 337 |
+
# then verify that the gateway returns 503.
|
| 338 |
+
kubectl apply -f - <<EOF
|
| 339 |
+
apiVersion: networking.k8s.io/v1
|
| 340 |
+
kind: NetworkPolicy
|
| 341 |
+
metadata:
|
| 342 |
+
name: arf-api-deny-all
|
| 343 |
+
namespace: arf-system
|
| 344 |
+
spec:
|
| 345 |
+
podSelector:
|
| 346 |
+
matchLabels:
|
| 347 |
+
app: arf-api
|
| 348 |
+
policyTypes:
|
| 349 |
+
- Ingress
|
| 350 |
+
EOF
|
| 351 |
+
|
| 352 |
+
sleep 5
|
| 353 |
+
# Gateway should return 503 Service Unavailable
|
| 354 |
+
curl -s -o /dev/null -w "%{http_code}" $GW_URL/health | grep 503
|
| 355 |
+
|
| 356 |
+
# Revert
|
| 357 |
+
kubectl delete networkpolicy arf-api-deny-all -n arf-system
|
| 358 |
+
```
|
| 359 |
+
|
| 360 |
+
### 7.3 Database Connection Failure
|
| 361 |
+
|
| 362 |
+
```bash
|
| 363 |
+
# Simulate a database outage by temporarily misconfiguring the Secret.
|
| 364 |
+
kubectl create secret generic arf-api-secrets \
|
| 365 |
+
--namespace arf-system \
|
| 366 |
+
--from-literal=DATABASE_URL="postgresql://nonexistent:5432/arf" \
|
| 367 |
+
--dry-run=client -o yaml | kubectl apply -f -
|
| 368 |
+
kubectl rollout restart deployment/arf-api -n arf-system
|
| 369 |
+
|
| 370 |
+
# Verify that the readiness probe fails (pods should not become ready).
|
| 371 |
+
kubectl wait --for=condition=ready pod -l app=arf-api -n arf-system --timeout=60s || echo "Expected: pods not ready"
|
| 372 |
+
# Restore the correct secret and restart.
|
| 373 |
+
```
|
| 374 |
+
|
| 375 |
+
8\. Continuous Improvement and Bayesian Updates
|
| 376 |
+
-----------------------------------------------
|
| 377 |
+
|
| 378 |
+
After each disaster recovery drill, we update the Beta posterior for the recovery success probability using the procedure in Section 3. The results are reviewed quarterly:
|
| 379 |
+
|
| 380 |
+
Drill DateSuccess (s)Failure (f)Posterior αPosterior βP(θ > 0.99)2026‑07‑0810320.6875(target)1001220.9999
|
| 381 |
+
|
| 382 |
+
The posterior probability is used to decide whether the platform can be promoted from pilot to production.
|
| 383 |
+
|
| 384 |
+
9\. Alignment with Regulatory Frameworks
|
| 385 |
+
----------------------------------------
|
| 386 |
+
|
| 387 |
+
FrameworkRequirementARF DR CapabilityNIST AI RMF Manage‑4Post‑deployment monitoring and incident responseAutomated backups, disaster recovery tests, continuous recalibrationEU AI Act Art. 12Record‑keeping and data integrityHash‑chained audit logs verified after recoverySOC 2 A1.1Availability commitmentsRTO ≤ 15 minutes, RPO ≤ 5 minutesISO/IEC 42001 §8.2Operational resilienceChaos engineering tests, rolling updates, multi‑AZ deployment
|
| 388 |
+
|
| 389 |
+
10\. Document Maintenance
|
| 390 |
+
-------------------------
|
| 391 |
+
|
| 392 |
+
This document is reviewed and updated quarterly, or after any major infrastructure change. The revision history is maintained in the repository.
|
| 393 |
+
|
| 394 |
+
VersionDateAuthorChanges1.02026‑07‑08ARF EngineeringInitial version2.02026‑07‑08ARF EngineeringExtended with Bayesian risk model, chaos engineering, regulatory alignment
|
| 395 |
+
|
| 396 |
+
_This document is proprietary and access‑controlled. Distribution is limited to qualified pilots and enterprise customers under written agreement._
|
docs/pilot-readiness.md
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
### 12.2 Key Manifests
|
| 3 |
+
|
| 4 |
+
| Repository | File | Purpose |
|
| 5 |
+
|------------|------|---------|
|
| 6 |
+
| `arf-api` | `deploy/kubernetes/arf-api/deployment.yaml` | API Deployment (3 replicas) |
|
| 7 |
+
| `arf-api` | `deploy/kubernetes/arf-api/service.yaml` | API ClusterIP Service |
|
| 8 |
+
| `arf-api` | `deploy/kubernetes/arf-api/hpa.yaml` | API HPA (3–10) |
|
| 9 |
+
| `arf-api` | `deploy/kubernetes/arf-api/networkpolicy.yaml` | Restrict ingress to gateway |
|
| 10 |
+
| `arf-api` | `deploy/kubernetes/arf-api/configmap.yaml` | Non‑sensitive config |
|
| 11 |
+
| `arf-api` | `deploy/kubernetes/arf-api/secret.yaml` | Sensitive values |
|
| 12 |
+
| `arf-gateway` | `deploy/kubernetes/arf-gateway/deployment.yaml` | Gateway Deployment (3 replicas) |
|
| 13 |
+
| `arf-gateway` | `deploy/kubernetes/arf-gateway/service.yaml` | LoadBalancer Service |
|
| 14 |
+
| `arf-gateway` | `deploy/kubernetes/arf-gateway/hpa.yaml` | Gateway HPA (3–10) |
|
| 15 |
+
|
| 16 |
+
---
|
| 17 |
+
|
| 18 |
+
## 13. Test & Verification Evidence
|
| 19 |
+
|
| 20 |
+
### 13.1 Pressure Test Suite
|
| 21 |
+
|
| 22 |
+
**44 tests, 100% pass rate.** Covers:
|
| 23 |
+
|
| 24 |
+
- Bayesian conjugate updates
|
| 25 |
+
- Policy condition evaluation
|
| 26 |
+
- Governance loop integration
|
| 27 |
+
- HealingIntent serialization
|
| 28 |
+
- Edge cases (zero data, large data, concurrency)
|
| 29 |
+
|
| 30 |
+
### 13.2 Formal Verification Suite
|
| 31 |
+
|
| 32 |
+
**7 property‑based test classes with 60,000+ examples.**
|
| 33 |
+
|
| 34 |
+
| Test | Examples | Result |
|
| 35 |
+
|------|----------|--------|
|
| 36 |
+
| Determinism (10,000 runs) | 10,000 | ✅ 1 unique hash |
|
| 37 |
+
| Criticality monotonicity | 5,000 | ✅ No violations |
|
| 38 |
+
| Stability gate cross‑validation | 10,000 | ✅ All within 1e‑12 |
|
| 39 |
+
| Skill gate monotonicity | 5,000 | ✅ No violations |
|
| 40 |
+
| Context hash determinism | 5,000 | ✅ Order‑independent |
|
| 41 |
+
| CUSUM optimality | 1,000 | ✅ Detection ≤150 steps |
|
| 42 |
+
| Conjugate update correctness | — | ✅ α, β match theory |
|
| 43 |
+
|
| 44 |
+
### 13.3 Performance Benchmarks
|
| 45 |
+
|
| 46 |
+
| Operation | p50 | p99 | Target |
|
| 47 |
+
|-----------|-----|-----|--------|
|
| 48 |
+
| Full governance loop | < 50 ms | < 100 ms | ✅ Met |
|
| 49 |
+
| Risk calculation | < 1 ms | < 5 ms | ✅ Met |
|
| 50 |
+
| HealingIntent serialization | < 5 ms | < 10 ms | ✅ Met |
|
| 51 |
+
|
| 52 |
+
### 13.4 Integration Tests
|
| 53 |
+
|
| 54 |
+
**8 end‑to‑end tests** covering the full HTTP → API → governance → audit pipeline,
|
| 55 |
+
including skill context, criticality, and outcome recording.
|
| 56 |
+
|
| 57 |
+
---
|
| 58 |
+
|
| 59 |
+
## 14. Roadmap & Future
|
| 60 |
+
|
| 61 |
+
### 14.1 v4.3.3 (Q4 2026)
|
| 62 |
+
|
| 63 |
+
- Multi‑agent Lyapunov coupling
|
| 64 |
+
- Emergent behavior detection
|
| 65 |
+
- Gateway Prometheus metrics
|
| 66 |
+
- Brute‑force protection on API keys
|
| 67 |
+
- Dependency vulnerability scanning in CI
|
| 68 |
+
|
| 69 |
+
### 14.2 v4.4 (Q1 2027)
|
| 70 |
+
|
| 71 |
+
- Gaussian Process sandbox dynamics
|
| 72 |
+
- Active GP‑based stability control
|
| 73 |
+
- Multi‑objective policy optimisation
|
| 74 |
+
- Helm charts for all components
|
| 75 |
+
|
| 76 |
+
### 14.3 v5.0 (Q2 2027)
|
| 77 |
+
|
| 78 |
+
- Federated learning of risk models across tenants
|
| 79 |
+
- Integration with major cloud policy frameworks (AWS SCP, Azure Policy)
|
| 80 |
+
- Certified NIST AI RMF profile
|
| 81 |
+
|
| 82 |
+
---
|
| 83 |
+
|
| 84 |
+
## 15. Next Steps
|
| 85 |
+
|
| 86 |
+
1. **Identify a design partner** in a regulated sector (finance, healthcare, telecom, energy).
|
| 87 |
+
2. **Execute a mutual NDA** and share this package.
|
| 88 |
+
3. **Schedule a 2‑hour technical deep‑dive** with the partner’s SRE and security teams.
|
| 89 |
+
4. **Deploy the sandbox** in the partner’s Kubernetes environment (Week 1).
|
| 90 |
+
5. **Begin the 8‑week pilot program** as described in Section 10.
|
| 91 |
+
|
| 92 |
+
---
|
| 93 |
+
|
| 94 |
+
## 16. Appendices
|
| 95 |
+
|
| 96 |
+
### A. Glossary
|
| 97 |
+
|
| 98 |
+
| Term | Definition |
|
| 99 |
+
|------|------------|
|
| 100 |
+
| CVaR | Conditional Value‑at‑Risk – expected loss in the worst 5% of outcomes |
|
| 101 |
+
| CUSUM | Cumulative Sum – sequential change‑detection algorithm |
|
| 102 |
+
| E‑value | Minimum confounding strength needed to nullify a causal effect |
|
| 103 |
+
| HMC | Hamiltonian Monte Carlo – Bayesian sampling method |
|
| 104 |
+
| IPW | Inverse Probability Weighting – causal effect estimator |
|
| 105 |
+
| TLA⁺ | Temporal Logic of Actions – formal specification language |
|
| 106 |
+
|
| 107 |
+
### B. Code Repositories
|
| 108 |
+
|
| 109 |
+
| Repository | Purpose | Access |
|
| 110 |
+
|------------|---------|--------|
|
| 111 |
+
| `agentic_reliability_framework` | Core Bayesian engine, governance loop, policies | Private |
|
| 112 |
+
| `arf-api` | FastAPI control plane, database models, routes | Private |
|
| 113 |
+
| `enterprise` | Rust execution ladder, safety gates | Private |
|
| 114 |
+
| `arf-gateway` | Go reverse proxy with auth, rate limiting, circuit breaker | Private |
|
| 115 |
+
|
| 116 |
+
### C. Contact
|
| 117 |
+
|
| 118 |
+
**Juan Petter**
|
| 119 |
+
Founder & Steward, Agentic Reliability Framework
|
| 120 |
+
Email: juan@arf-ai.com
|
| 121 |
+
Website: https://arf-ai.com
|
| 122 |
+
|
| 123 |
+
---
|
| 124 |
+
|
| 125 |
+
*This document is proprietary and access‑controlled. Distribution is limited to qualified pilots and enterprise customers under written agreement. No part of this document may be reproduced, distributed, or used for AI training without express written permission.*
|
docs/regulatory-mapping.md
ADDED
|
@@ -0,0 +1,212 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ARF Comprehensive Regulatory Alignment & Compliance Handbook
|
| 2 |
+
## NIST AI RMF 1.0, EU AI Act, ISO/IEC 42001, SOC 2, GDPR, and More
|
| 3 |
+
|
| 4 |
+
**Document Version:** 2.0
|
| 5 |
+
**ARF Version:** v4.3.2
|
| 6 |
+
**Date:** July 8, 2026
|
| 7 |
+
**Classification:** Proprietary – Access‑Controlled
|
| 8 |
+
|
| 9 |
+
---
|
| 10 |
+
|
| 11 |
+
## 1. Introduction
|
| 12 |
+
|
| 13 |
+
The Agentic Reliability Framework (ARF) is a governance control plane for AI‑driven infrastructure operations. This document provides an exhaustive, auditable mapping between ARF's capabilities and the requirements of global regulatory frameworks applicable to high‑risk AI systems in critical infrastructure.
|
| 14 |
+
|
| 15 |
+
It is intended for:
|
| 16 |
+
- **Design partners** evaluating ARF for regulated deployments.
|
| 17 |
+
- **Compliance officers** preparing for certification audits.
|
| 18 |
+
- **Independent auditors** verifying the platform's claims.
|
| 19 |
+
- **Regulators** assessing the adequacy of ARF’s governance controls.
|
| 20 |
+
|
| 21 |
+
Every mapping includes a **code reference**, **audit evidence location**, and **verification procedure** so that claims can be tested without relying on vendor assertions.
|
| 22 |
+
|
| 23 |
+
---
|
| 24 |
+
|
| 25 |
+
## 2. Framework Overview
|
| 26 |
+
|
| 27 |
+
ARF addresses the following frameworks:
|
| 28 |
+
|
| 29 |
+
| Framework | Jurisdiction | Status | ARF Alignment |
|
| 30 |
+
|-----------|--------------|--------|---------------|
|
| 31 |
+
| **NIST AI RMF 1.0** | United States | Active (revision underway) | Full – Govern, Map, Measure, Manage |
|
| 32 |
+
| **EU AI Act** | European Union | In force | Full – Articles 9–16, Annex IV |
|
| 33 |
+
| **ISO/IEC 42001:2023** | International | Published | Full – AI management system |
|
| 34 |
+
| **SOC 2 (Trust Services Criteria)** | International (AICPA) | Widely adopted | Security, Availability, Confidentiality |
|
| 35 |
+
| **GDPR** | European Union | In force | Data protection by design, right to explanation |
|
| 36 |
+
| **NIST SP 800‑53** | United States | Active | Security and privacy controls |
|
| 37 |
+
| **OWASP Top 10 for LLM Applications** | International | Best practice | Prompt injection, supply chain |
|
| 38 |
+
|
| 39 |
+
---
|
| 40 |
+
|
| 41 |
+
## 3. Detailed Framework Mappings
|
| 42 |
+
|
| 43 |
+
### 3.1 NIST AI RMF 1.0
|
| 44 |
+
|
| 45 |
+
The NIST AI RMF is organized into four core functions: **Govern**, **Map**, **Measure**, and **Manage**. ARF addresses each function with specific technical controls.
|
| 46 |
+
|
| 47 |
+
#### 3.1.1 Govern
|
| 48 |
+
|
| 49 |
+
| NIST Subcategory | ARF Capability | Code Reference | Audit Evidence | Verification Procedure |
|
| 50 |
+
|------------------|----------------|----------------|----------------|------------------------|
|
| 51 |
+
| **GOV‑1: Policies, processes, and procedures** | Policy algebra (TLA⁺ verified); criticality parameter; pre‑built policy packs | `policies.py` (PolicyAlgebra), `gates.rs` (dynamic thresholds), `policy_engine.py` (healing policies), `policy_packs/` | Policy tree definitions stored in ConfigMap; every decision logs which policies were checked. | 1. Export the active policy tree from the ConfigMap. 2. Run TLC on PolicyAlgebra.tla to verify algebraic laws. 3. Verify that 100% of decisions in the audit log include policy violation lists. |
|
| 52 |
+
| **GOV‑2: Roles, responsibilities, and delegated authority** | Gateway RBAC via API key tiers; internal API key protects governance endpoints; human‑override fields in `HealingIntent`. | `auth/apikey.go` (gateway), `deps.py` (verify_internal_key), `healing_intent.py` (human_overrides, approvals) | Gateway access logs show per‑key requests; audit log entries record approver identity for overridden decisions. | 1. Attempt to call the API directly without X‑Internal‑Key; confirm 401. 2. Call the API with a read‑only tier key; confirm that outcome recording fails. 3. Inspect the audit log for any decision with `status=approved_with_overrides` and verify the `approved_by` field is populated. |
|
| 53 |
+
| **GOV‑4: AI risk management integration with organizational risk** | Bayesian risk scores feed into CVaR loss minimization; stability and drift flags provide continuous risk signals. | `risk_engine.py`, `governance_loop.py` (CVaR, stability, drift) | Every `HealingIntent` carries a risk score, epistemic breakdown, and stability/drift metadata. | 1. Query the audit log for any decision where `risk_score > 0.8` and verify that the action was DENY or ESCALATE. 2. Verify that `lyapunov_stable` and `temporal_drift_detected` are present in every `HealingIntent.metadata`. |
|
| 54 |
+
| **GOV‑5: Continuous improvement** | Conjugate Beta updates, online recalibration, memory‑weight optimization, and causal effect re‑estimation after every outcome. | `risk_engine.py` (update_outcome, _maybe_recalibrate), `memory_weight_optimizer.py`, `causal_effect_estimator.py` | Outcome log shows feedback loop; recalibration events are logged. | 1. Record a sequence of outcomes; verify that the conjugate α and β values change as expected. 2. Verify that after 100 outcomes, the `risk_engine` recalibrates if ECE exceeds 0.1. |
|
| 55 |
+
|
| 56 |
+
#### 3.1.2 Map
|
| 57 |
+
|
| 58 |
+
| NIST Subcategory | ARF Capability | Code Reference | Audit Evidence | Verification Procedure |
|
| 59 |
+
|------------------|----------------|----------------|----------------|------------------------|
|
| 60 |
+
| **MAP‑1: Context and intended use** | `InfrastructureIntent` carries provenance, environment, requester identity; `context_hash` cryptographically binds all inputs. | `intents.py` (InfrastructureIntent), `healing_intent.py` (context_hash), `governance_loop.py` (context extraction) | `HealingIntent.context_hash` is a SHA‑256 of the canonical context; auditor can recompute and verify. | 1. For any decision in the audit log, retrieve the stored context and recompute `SHA‑256(canonical_json(context))`. 2. Assert equality with the stored `context_hash`. |
|
| 61 |
+
| **MAP‑2: AI system categorization** | `RiskEngine` categorizes every intent into action categories (database, network, compute, security) with category‑specific priors. | `risk_engine.py` (categorize_intent, PRIORS) | The `risk_score` explanation string includes the category; audit log records the category. | 1. Submit intents of each type (ProvisionResource, GrantAccess, DeployConfiguration). 2. Verify that the risk explanation string contains the correct category. |
|
| 62 |
+
| **MAP‑3: AI capabilities, limitations, and appropriate use** | Skill registry tracks per‑skill reliability; skill gate blocks unreliable skills; counterfactual explanations describe limitations. | `skill_registry.py`, `gates.rs` (SkillGate), `causal_effect_estimator.py` (generate_counterfactual) | Skill alpha/beta values are logged; counterfactual text is included in every `HealingIntent`. | 1. Register a new skill with no history. 2. Submit an intent with that skill; verify that the SkillGate blocks it (P(θ>0.5) < 0.95). 3. Submit an intent with a well‑established skill (α=20, β=3); verify that the SkillGate passes. |
|
| 63 |
+
| **MAP‑4: Risk mapping to AI system lifecycle** | Time‑decayed risk and temporal drift detection monitor risk evolution over time. | `governance_loop.py` (time‑decayed risk, temporal drift), `temporal_reliability.py` | `HealingIntent.metadata.decayed_risk`, `.temporal_drift_detected` | 1. Run 100 decisions with a fixed risk score; verify that `decayed_risk` converges to the input risk. 2. Inject a sudden risk spike; verify that `temporal_drift_detected` becomes True within the CUSUM threshold. |
|
| 64 |
+
|
| 65 |
+
#### 3.1.3 Measure
|
| 66 |
+
|
| 67 |
+
| NIST Subcategory | ARF Capability | Code Reference | Audit Evidence | Verification Procedure |
|
| 68 |
+
|------------------|----------------|----------------|----------------|------------------------|
|
| 69 |
+
| **MEASURE‑1: Risk measurement methodologies** | Bayesian risk fusion (conjugate + hyperprior + HMC), CVaR, epistemic uncertainty decomposition (CUDL Shapley values). | `risk_engine.py` (calculate_risk), `governance_loop.py` (CVaR, epistemic), `research/cudl/` | `HealingIntent` contains risk score, epistemic breakdown, and Shapley attribution. | 1. Submit an intent with known risk factors; verify that `risk_factors` sum to the `risk_score`. 2. Enable epistemic probing; verify that `epistemic_breakdown` contains hallucination, forecast, and sparsity components. |
|
| 70 |
+
| **MEASURE‑2: Evaluation of trustworthiness characteristics** | Lyapunov stability monitoring, CUSUM drift detection, Expected Calibration Error (ECE) recalibration, E‑value sensitivity analysis. | `stability_controller.py`, `temporal_reliability.py`, `risk_engine.py` (ECE), `causal_effect_estimator.py` (E‑value) | `HealingIntent.metadata.lyapunov_stable`, `.temporal_drift_detected`; E‑values reported with ATE estimates. | 1. Run two consecutive decisions with increasing risk; verify that `lyapunov_stable` becomes False. 2. Compute the E‑value for a known ATE; verify that it matches the formula `RR + sqrt(RR(RR‑1))`. |
|
| 71 |
+
| **MEASURE‑3: Mechanisms for tracking and responding to emergent risks** | Passive Lyapunov check triggers active stability override; CUSUM drift triggers recalibration; skill gate blocks unproven skills. | `governance_loop.py` (active stability response), `gates.rs` (StabilityGate, SkillGate) | Gate failure reasons are logged; stability override events are recorded. | 1. Artificially destabilize the Lyapunov monitor by alternating risk/psi; verify that the active stability response overrides the decision to ESCALATE. 2. Inject a skill with α=3, β=3; verify that the SkillGate fails with a message containing "Bayesian confidence low". |
|
| 72 |
+
|
| 73 |
+
#### 3.1.4 Manage
|
| 74 |
+
|
| 75 |
+
| NIST Subcategory | ARF Capability | Code Reference | Audit Evidence | Verification Procedure |
|
| 76 |
+
|------------------|----------------|----------------|----------------|------------------------|
|
| 77 |
+
| **MANAGE‑1: Risk treatment strategies** | Three‑action decision (approve, deny, escalate) based on posterior expected loss minimization; human‑in‑the‑loop overrides. | `governance_loop.py` (decision rule), `healing_intent.py` (human_overrides) | `HealingIntent.action` and `.status` fields; audit log records the final decision and any overrides. | 1. Submit an intent with policy violations; verify action is DENY. 2. Submit an intent with high epistemic uncertainty; verify action is ESCALATE. 3. Submit an intent with low risk and no violations; verify action is APPROVE. |
|
| 78 |
+
| **MANAGE‑2: Documentation and reporting** | Immutable, hash‑chained audit log; every decision carries full trace (risk score, justification, counterfactual, metadata). | `routes_governance.py` (write_audit_log), `models_intents.py` (DecisionAuditLogDB) | The `decision_audit_log` table is queryable by tenant and timestamp; hashes are cryptographically verifiable. | 1. Query the audit log for a specific tenant and date range; verify that all decisions are present and ordered. 2. Select two consecutive entries; verify that the hash chain is intact. |
|
| 79 |
+
| **MANAGE‑3: Stakeholder communication** | Plain‑language justification; counterfactual explanation; advisory‑only status for OSS edition. | `healing_intent.py` (justification, metadata.counterfactual) | Every decision response includes a human‑readable explanation. | 1. Call the `/intents/evaluate` endpoint; verify that the response includes a `justification` field with a plain‑language explanation. 2. Verify that when a causal model is available, the response includes a `counterfactual` field. |
|
| 80 |
+
| **MANAGE‑4: Post‑deployment monitoring** | Outcome feedback loop updates conjugate posteriors, memory weights, and causal estimates; usage tracker provides quota visibility. | `outcome_service.py`, `risk_engine.py` (update_outcome), `usage_tracker.py` | Outcome log shows feedback events; usage tracker shows remaining quota. | 1. Record an outcome via the `/intents/outcome` endpoint; verify that the risk engine's conjugate parameters have changed. 2. Call the `/auth/info` endpoint via the gateway; verify that `remaining` decreases after each evaluation. |
|
| 81 |
+
|
| 82 |
+
---
|
| 83 |
+
|
| 84 |
+
### 3.2 EU AI Act
|
| 85 |
+
|
| 86 |
+
ARF is designed to govern AI‑driven infrastructure actions, which may be classified as high‑risk under the EU AI Act when they affect critical infrastructure.
|
| 87 |
+
|
| 88 |
+
| Article | Requirement | ARF Capability | Code Reference | Audit Evidence | Verification Procedure |
|
| 89 |
+
|---------|-------------|----------------|----------------|----------------|------------------------|
|
| 90 |
+
| **Art. 9** | Risk management system | CVaR expected loss minimization; StabilityGate blocks when platform is unstable; temporal drift triggers recalibration. | `governance_loop.py` (CVaR, stability), `gates.rs` (StabilityGate), `risk_engine.py` (recalibration) | Every `HealingIntent` contains risk score, CVaR usage flag, stability sample, and drift metadata. | Same as NIST Measure‑1/Measure‑3. |
|
| 91 |
+
| **Art. 10** | Data governance and data quality | `context_hash` cryptographically binds decisions to input data; `provenance` field in intent tracks data origin. | `healing_intent.py` (context_hash), `intents.py` (provenance) | Auditor can recompute `context_hash` from stored context; provenance chain is logged. | Same as NIST Map‑1. |
|
| 92 |
+
| **Art. 11** | Technical documentation | Immutable audit log; `HealingIntent` carries full decision trace including pre‑/post‑memory risk, epistemic breakdown, counterfactual. | `models_intents.py` (DecisionAuditLogDB), `healing_intent.py` (to_enterprise_request) | Audit log table is queryable; each entry contains the complete decision payload. | Same as NIST Manage‑2. |
|
| 93 |
+
| **Art. 12** | Record‑keeping | Every decision logged with timestamp, tenant, risk score, action, justification; logs are hash‑chained and Ed25519‑signed. | `routes_governance.py` (write_audit_log), `crypto.py` (Ed25519 signing) | Hash chain integrity can be verified without trusting the runtime; signatures are base64‑encoded Ed25519. | 1. Export the audit log. 2. For each entry, verify the Ed25519 signature using the stored public key. 3. Verify that `SHA‑256(entry_n || entry_n‑1.hash) == entry_n.hash`. |
|
| 94 |
+
| **Art. 13** | Transparency and provision of information | Plain‑language justification; counterfactual explanation; memory‑based evidence summary. | `healing_intent.py` (justification, metadata.counterfactual) | Every API response includes `justification` and `counterfactual` fields. | Same as NIST Manage‑3. |
|
| 95 |
+
| **Art. 14** | Human oversight | Escalate action; human override fields; approval tracking; gateway RBAC for human reviewers. | `governance_loop.py` (decision rule), `healing_intent.py` (with_human_approval), `auth/apikey.go` | `HealingIntent.approvals` records reviewer identity and timestamp; gateway enforces role‑based access. | Same as NIST Govern‑2. |
|
| 96 |
+
| **Art. 15** | Accuracy, robustness, and cybersecurity | Deterministic RNG (SHA‑256 seeded), cryptographic signatures, constant‑time API key comparison, internal API key protection. | `governance_loop.py` (deterministic RNG), `crypto.py` (signatures), `deps.py` (constant‑time compare) | All probabilistic operations are reproducible; tampering with signed intents is detected. | 1. Run the governance loop twice with the same input; verify identical output. 2. Modify one field of a signed `HealingIntent`; verify that `verify()` returns False. |
|
| 97 |
+
| **Art. 16** | Reporting obligations | Usage tracker and audit log provide quota consumption and decision history; Wilson monitor tracks Rust enforcer agreement. | `usage_tracker.py`, `models_intents.py`, `wilson_monitor.py` | Usage and audit logs are queryable; Wilson confidence interval is updated every 5 minutes. | 1. Query the usage log for a given API key; verify that monthly counts match the quota. 2. Verify that the Wilson monitor emits a Prometheus metric with the current confidence interval. |
|
| 98 |
+
|
| 99 |
+
---
|
| 100 |
+
|
| 101 |
+
### 3.3 ISO/IEC 42001:2023 – AI Management System
|
| 102 |
+
|
| 103 |
+
ISO/IEC 42001 provides a certifiable framework for an AI management system. ARF can serve as the technical enforcement layer for the controls required by this standard.
|
| 104 |
+
|
| 105 |
+
| Clause | Requirement | ARF Capability | Code Reference |
|
| 106 |
+
|--------|-------------|----------------|----------------|
|
| 107 |
+
| 4.1 | Understanding the organization and its context | `InfrastructureIntent` captures operational context; `context_hash` binds it to decisions. | `intents.py`, `healing_intent.py` |
|
| 108 |
+
| 5.1 | Leadership and commitment | Criticality parameter allows leadership to set risk appetite. | `infrastructure_intents.py` (criticality) |
|
| 109 |
+
| 6.1 | Actions to address risks and opportunities | Full Bayesian risk pipeline, CVaR, stability monitoring. | `risk_engine.py`, `governance_loop.py` |
|
| 110 |
+
| 7.5 | Documented information | Immutable, hash‑chained audit log. | `models_intents.py`, `routes_governance.py` |
|
| 111 |
+
| 8.1 | Operational planning and control | Policy engine enforces rules; gates block unsafe actions. | `policies.py`, `gates.rs` |
|
| 112 |
+
| 9.1 | Monitoring, measurement, analysis, and evaluation | Epistemic uncertainty, Shapley decomposition, ECE recalibration, Lyapunov stability, CUSUM drift. | `governance_loop.py`, `stability_controller.py`, `temporal_reliability.py` |
|
| 113 |
+
| 10.1 | Continual improvement | Conjugate updates, memory weight optimization, causal re‑estimation. | `risk_engine.py`, `memory_weight_optimizer.py`, `causal_effect_estimator.py` |
|
| 114 |
+
|
| 115 |
+
---
|
| 116 |
+
|
| 117 |
+
### 3.4 SOC 2 (Trust Services Criteria)
|
| 118 |
+
|
| 119 |
+
SOC 2 evaluates the security, availability, processing integrity, confidentiality, and privacy of a system.
|
| 120 |
+
|
| 121 |
+
| Trust Service Criterion | ARF Capability | Code Reference |
|
| 122 |
+
|--------------------------|----------------|----------------|
|
| 123 |
+
| **Security** (CC6.1, CC6.6) | Internal API key authentication, constant‑time comparison, gateway salted SHA‑256 hashing, RBAC. | `deps.py` (verify_internal_key), `auth/apikey.go` |
|
| 124 |
+
| **Availability** (A1.1, A1.2) | Kubernetes HPA (3–10 replicas), liveness/readiness probes, rolling updates. | `deploy/kubernetes/arf‑api/hpa.yaml`, `deployment.yaml` |
|
| 125 |
+
| **Confidentiality** (C1.1) | NetworkPolicy restricts API to gateway only; CORS restricted to specific origin. | `networkpolicy.yaml`, `main.py` (CORS) |
|
| 126 |
+
| **Processing Integrity** (PI1.2, PI1.3) | Deterministic policy evaluation, TLA⁺ verified algebra, property‑based testing. | `PolicyAlgebra.tla`, `test_policy_properties.py`, `proptest_policy.rs` |
|
| 127 |
+
| **Privacy** (P1.1, P4.1) | Tenant isolation in risk engine; API keys scoped to tenants; usage data retained per retention policy. | `risk_engine.py` (tenant isolation), `usage_tracker.py` (retention) |
|
| 128 |
+
|
| 129 |
+
---
|
| 130 |
+
|
| 131 |
+
### 3.5 GDPR (General Data Protection Regulation)
|
| 132 |
+
|
| 133 |
+
For deployments processing personal data, ARF provides the following controls:
|
| 134 |
+
|
| 135 |
+
| GDPR Article | Requirement | ARF Capability |
|
| 136 |
+
|--------------|-------------|----------------|
|
| 137 |
+
| Art. 5(1)(f) | Integrity and confidentiality | Cryptographic signatures, hash‑chained audit log, constant‑time API key verification. |
|
| 138 |
+
| Art. 25 | Data protection by design | `context_hash` minimizes the need to store raw personal data; only hashes are retained. |
|
| 139 |
+
| Art. 30 | Records of processing activities | Audit log provides a complete record of all decisions, including requester identity and justification. |
|
| 140 |
+
| Art. 35 | Data protection impact assessment | Risk scores, epistemic uncertainty, and counterfactuals provide evidence for DPIAs. |
|
| 141 |
+
| Art. 22 | Automated individual decision‑making | Human‑in‑the‑loop override ensures that no solely automated decision is made without review capability. |
|
| 142 |
+
|
| 143 |
+
---
|
| 144 |
+
|
| 145 |
+
## 4. Cross‑Framework Alignment Matrix
|
| 146 |
+
|
| 147 |
+
| Requirement Category | NIST AI RMF | EU AI Act | ISO 42001 | SOC 2 | GDPR | ARF Feature(s) |
|
| 148 |
+
|----------------------|-------------|-----------|-----------|-------|------|----------------|
|
| 149 |
+
| Risk identification and quantification | Measure‑1, Measure‑2 | Art. 9 | 6.1 | — | Art. 35 | Bayesian risk fusion, CVaR, epistemic uncertainty, E‑value |
|
| 150 |
+
| Policy enforcement and controls | Govern‑1, Manage‑1 | Art. 9, Art. 14 | 8.1 | PI1.2 | Art. 22 | Policy algebra, Rust gates, human override |
|
| 151 |
+
| Data provenance and quality | Map‑1 | Art. 10 | 4.1 | — | Art. 5(1)(f) | `context_hash`, `InfrastructureIntent.provenance` |
|
| 152 |
+
| Documentation and record‑keeping | Manage‑2 | Art. 11, Art. 12 | 7.5 | — | Art. 30 | Immutable audit log, Ed25519 signatures |
|
| 153 |
+
| Transparency and explainability | Map‑3, Manage‑3 | Art. 13 | — | — | Art. 22 | Plain‑language justification, counterfactuals |
|
| 154 |
+
| Continuous monitoring and improvement | Map‑4, Measure‑3, Govern‑5 | Art. 9, Art. 15 | 9.1, 10.1 | — | — | Conjugate updates, stability/drift detection, recalibration |
|
| 155 |
+
| Human oversight | Govern‑2, Manage‑1 | Art. 14 | — | — | Art. 22 | Escalate action, human approval workflow, RBAC |
|
| 156 |
+
| Security and access control | — | Art. 15 | — | CC6.1, CC6.6, C1.1 | Art. 5(1)(f) | Internal API key, gateway auth, NetworkPolicy, CORS |
|
| 157 |
+
| Availability and resilience | — | — | — | A1.1, A1.2 | — | Kubernetes HPA, liveness/readiness probes, rolling updates |
|
| 158 |
+
| Privacy | — | — | — | P1.1, P4.1 | Art. 25 | Tenant isolation, API key scoping, usage retention |
|
| 159 |
+
|
| 160 |
+
---
|
| 161 |
+
|
| 162 |
+
## 5. Evidence Collection & Audit Procedure
|
| 163 |
+
|
| 164 |
+
### 5.1 Automated Evidence Collection
|
| 165 |
+
|
| 166 |
+
ARF provides the following automated evidence sources:
|
| 167 |
+
|
| 168 |
+
1. **Audit log (PostgreSQL):** The `decision_audit_log` table contains every governance decision with timestamp, tenant, risk score, action, justification, and cryptographic hashes.
|
| 169 |
+
2. **Prometheus metrics:** `arf_evaluations_total`, `arf_rust_agreement_total`, `arf_evaluation_duration_seconds` provide real‑time observability.
|
| 170 |
+
3. **OpenTelemetry traces:** Every request is traced with a unique `trace_id`, linking gateway logs to API decisions.
|
| 171 |
+
4. **Usage tracker (SQLite/PostgreSQL):** Provides per‑API‑key quota consumption and audit logs.
|
| 172 |
+
|
| 173 |
+
### 5.2 Independent Auditor Verification Checklist
|
| 174 |
+
|
| 175 |
+
| Step | Procedure | Expected Outcome |
|
| 176 |
+
|------|-----------|-----------------|
|
| 177 |
+
| 1. Deterministic replay | Run `GovernanceLoop.run()` twice with identical inputs. | SHA‑256 of serialized `HealingIntent` is identical. |
|
| 178 |
+
| 2. Hash chain integrity | Query `decision_audit_log` ordered by timestamp; verify that each hash links to the previous entry. | No broken chains. |
|
| 179 |
+
| 3. Signature verification | For any signed intent, verify the Ed25519 signature using the stored public key fingerprint. | Signature verification returns `True`. |
|
| 180 |
+
| 4. Tamper detection | Modify one field of a signed `HealingIntent` and call `verify()`. | `verify()` returns `False`. |
|
| 181 |
+
| 5. Policy algebra | Run TLC on `PolicyAlgebra.tla`. | All invariants hold. |
|
| 182 |
+
| 6. Cross‑language policy equivalence | Generate random policy trees and evaluate in both Python and Rust; compare violation sets. | Identical violation sets. |
|
| 183 |
+
| 7. Bayesian update correctness | Record a sequence of outcomes; manually compute expected α, β. | `BetaStore` matches manual computation. |
|
| 184 |
+
| 8. Skill gate | Submit intents with varying skill evidence; verify gate behavior. | Low‑evidence skills fail; high‑evidence skills pass. |
|
| 185 |
+
| 9. Criticality‑aware gates | Submit intents with criticality=1.0; verify that tolerance is zero and confidence threshold is 1.0. | High‑criticality actions are blocked unless perfect. |
|
| 186 |
+
| 10. Context hash verification | Retrieve stored context for any decision; recompute SHA‑256. | Matches stored `context_hash`. |
|
| 187 |
+
|
| 188 |
+
---
|
| 189 |
+
|
| 190 |
+
## 6. Continuous Compliance Monitoring
|
| 191 |
+
|
| 192 |
+
ARF includes built‑in mechanisms for ongoing compliance verification:
|
| 193 |
+
|
| 194 |
+
- **Wilson confidence interval monitor:** Every 5 minutes, the Wilson updater checks the Rust enforcer agreement and adjusts the canary promotion status. This provides a statistical control chart for policy enforcement consistency.
|
| 195 |
+
- **Expected Calibration Error (ECE):** Monitored per tenant and category; triggers recalibration when exceeding 0.1.
|
| 196 |
+
- **Lyapunov stability window:** The StabilityGate blocks execution when the recent stability record is poor, preventing the platform from operating in a degraded state.
|
| 197 |
+
- **Temporal drift detection:** CUSUM tracks sustained drift in risk estimates; alerts when the model becomes stale.
|
| 198 |
+
|
| 199 |
+
---
|
| 200 |
+
|
| 201 |
+
## 7. References
|
| 202 |
+
|
| 203 |
+
- NIST AI RMF 1.0: `https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.100-1.pdf`
|
| 204 |
+
- EU AI Act: `https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=CELEX:52021PC0206`
|
| 205 |
+
- ISO/IEC 42001:2023: `https://www.iso.org/standard/81230.html`
|
| 206 |
+
- SOC 2 Trust Services Criteria: `https://www.aicpa.org/soc2`
|
| 207 |
+
- GDPR: `https://gdpr-info.eu/`
|
| 208 |
+
- ARF Mathematical Work Journal: `docs/math_journal.md`
|
| 209 |
+
- ARF Policy Algebra TLA⁺ Specification: `agentic_reliability_framework/spec/tla/PolicyAlgebra.tla`
|
| 210 |
+
- ARF Pressure Test Suite: `tests/pressure/test_pressure.py`
|
| 211 |
+
|
| 212 |
+
*This document is proprietary and access‑controlled. Distribution is limited to qualified pilots and enterprise customers under written agreement.*
|
docs/security-assessment.md
ADDED
|
@@ -0,0 +1,186 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ARF Security Self‑Assessment & Penetration Test Report
|
| 2 |
+
|
| 3 |
+
**Version:** 1.0
|
| 4 |
+
**ARF Version:** v4.3.2
|
| 5 |
+
**Date:** July 9, 2026
|
| 6 |
+
**Classification:** Proprietary – Access‑Controlled
|
| 7 |
+
**Scope:** arf-api, arf-gateway, agentic_reliability_framework
|
| 8 |
+
|
| 9 |
+
---
|
| 10 |
+
|
| 11 |
+
## 1. Executive Summary
|
| 12 |
+
|
| 13 |
+
This document presents the results of a structured security review of the ARF platform, conducted by the development team and supplemented by manual penetration testing performed by the steward. It is organized according to the OWASP Top 10 (2021) and includes specific findings, code references, and a risk‑based prioritization of residual gaps. The Bayesian confidence model introduced in Section 7 quantifies our degree of belief that the platform is secure enough for a regulated pilot deployment.
|
| 14 |
+
|
| 15 |
+
### 1.1 Overall Risk Rating
|
| 16 |
+
|
| 17 |
+
| Category | Rating | Explanation |
|
| 18 |
+
|----------|--------|-------------|
|
| 19 |
+
| Confidentiality | **Medium** | Internal API key protects governance endpoints; no encryption at rest for audit logs. |
|
| 20 |
+
| Integrity | **High** | Ed25519 signatures, hash‑chained audit logs, and constant‑time key comparison prevent tampering. |
|
| 21 |
+
| Availability | **High** | Kubernetes HPA, liveness/readiness probes, and rolling updates ensure resilience. |
|
| 22 |
+
| Authentication | **High** | Gateway uses salted SHA‑256 API keys; internal API key prevents direct API access. |
|
| 23 |
+
| Authorization | **Medium** | API key tiers exist but are not granularly enforced at the endpoint level. |
|
| 24 |
+
|
| 25 |
+
---
|
| 26 |
+
|
| 27 |
+
## 2. Scope
|
| 28 |
+
|
| 29 |
+
The security review covered the following components:
|
| 30 |
+
|
| 31 |
+
| Component | Language | Lines of Code | Reviewed |
|
| 32 |
+
|-----------|----------|---------------|----------|
|
| 33 |
+
| `arf-api` (FastAPI) | Python | ~1,700 | Yes |
|
| 34 |
+
| `arf-gateway` (Go) | Go | ~500 | Yes |
|
| 35 |
+
| `agentic_reliability_framework` | Python | ~15,000 | Partial (core governance only) |
|
| 36 |
+
| `enterprise/arf_execution` (Rust) | Rust | ~1,200 | Partial (gates only) |
|
| 37 |
+
|
| 38 |
+
**Out of scope:** Third‑party dependencies not directly audited; infrastructure‑level security (AWS IAM, Kubernetes RBAC); physical security.
|
| 39 |
+
|
| 40 |
+
---
|
| 41 |
+
|
| 42 |
+
## 3. OWASP Top 10 (2021) Assessment
|
| 43 |
+
|
| 44 |
+
### 3.1 Broken Access Control (A01)
|
| 45 |
+
|
| 46 |
+
**Finding:** The API routes previously had no authentication. This has been **remediated** in v4.3.2 by adding the `verify_internal_key` dependency to the governance router (`routes_governance.py`). The gateway proxies requests and injects the `X‑Internal‑Key` header. The API key is compared using a constant‑time algorithm to prevent timing attacks.
|
| 47 |
+
|
| 48 |
+
**Code evidence:** `app/api/deps.py` (verify_internal_key), `app/api/routes_governance.py` (router dependency).
|
| 49 |
+
|
| 50 |
+
**Residual risk:** Low. If the `ARF_INTERNAL_API_KEY` environment variable is not set, the dependency is a no‑op (for local development). This must be enforced in production via Kubernetes Secrets.
|
| 51 |
+
|
| 52 |
+
### 3.2 Cryptographic Failures (A02)
|
| 53 |
+
|
| 54 |
+
**Finding:** No cryptographic failures detected. The gateway uses salted SHA‑256 for API key storage (`auth/apikey.go`). `HealingIntent` supports Ed25519 signatures (`healing_intent.py`). Context hashes use SHA‑256. Audit logs are hash‑chained.
|
| 55 |
+
|
| 56 |
+
**Code evidence:** `internal/auth/apikey.go`, `healing_intent.py` (sign/verify), `governance_loop.py` (context_hash).
|
| 57 |
+
|
| 58 |
+
**Residual risk:** Low. No known weaknesses in the employed algorithms.
|
| 59 |
+
|
| 60 |
+
### 3.3 Injection (A03)
|
| 61 |
+
|
| 62 |
+
**Finding:** The API uses Pydantic models with strict field validation (`BaseIntentRequest`), which mitigates type‑based injection. The gateway and API do not construct SQL queries with user input directly (the risk engine uses parameterized queries via SQLAlchemy; the gateway uses SQLite with placeholders). No injection vulnerabilities were found.
|
| 63 |
+
|
| 64 |
+
**Code evidence:** `models/infrastructure_intents.py` (validators), `usage_tracker.py` (parameterized queries), `auth/apikey.go` (placeholders).
|
| 65 |
+
|
| 66 |
+
**Residual risk:** Low.
|
| 67 |
+
|
| 68 |
+
### 3.4 Insecure Design (A04)
|
| 69 |
+
|
| 70 |
+
**Finding:** No design‑level flaws identified. The separation of advisory and execution layers, the immutable HealingIntent contract, and the deterministic governance loop are strong design patterns that reduce the attack surface.
|
| 71 |
+
|
| 72 |
+
**Residual risk:** Low.
|
| 73 |
+
|
| 74 |
+
### 3.5 Security Misconfiguration (A05)
|
| 75 |
+
|
| 76 |
+
**Finding:** CORS is restricted to a specific frontend origin in `main.py`. The Kubernetes NetworkPolicy restricts API access to the gateway pod only. However, the gateway’s rate‑limiter (token bucket) is configured with a global rate of 100 req/min and burst of 20 – this may be too permissive for some tiers.
|
| 77 |
+
|
| 78 |
+
**Code evidence:** `app/main.py` (CORS), `deploy/kubernetes/arf-api/networkpolicy.yaml`, `internal/middleware/ratelimit.go`.
|
| 79 |
+
|
| 80 |
+
**Residual risk:** Medium. Rate limiting should be tier‑specific.
|
| 81 |
+
|
| 82 |
+
### 3.6 Vulnerable and Outdated Components (A06)
|
| 83 |
+
|
| 84 |
+
**Finding:** Dependencies are tracked via `requirements.txt` and `go.mod`. No automated vulnerability scanning is integrated into CI. A manual review of the Go dependencies (`go.sum`) shows up‑to‑date packages; Python dependencies were not exhaustively audited.
|
| 85 |
+
|
| 86 |
+
**Code evidence:** `requirements.txt`, `go.mod`, `go.sum`.
|
| 87 |
+
|
| 88 |
+
**Residual risk:** Medium. Recommend integrating `pip‑audit` and `govulncheck` into CI.
|
| 89 |
+
|
| 90 |
+
### 3.7 Identification and Authentication Failures (A07)
|
| 91 |
+
|
| 92 |
+
**Finding:** The gateway implements salted SHA‑256 API key hashing with random 16‑byte salts (`auth/apikey.go`). The API’s internal key uses constant‑time comparison. No authentication bypasses were found during testing.
|
| 93 |
+
|
| 94 |
+
**Code evidence:** `auth/apikey.go`, `deps.py` (_constant_time_compare).
|
| 95 |
+
|
| 96 |
+
**Residual risk:** Low.
|
| 97 |
+
|
| 98 |
+
### 3.8 Software and Data Integrity Failures (A08)
|
| 99 |
+
|
| 100 |
+
**Finding:** The `HealingIntent` supports Ed25519 signatures, and audit logs are hash‑chained. This provides strong integrity guarantees. However, there is no mechanism to verify the integrity of the governance loop’s Python dependencies at runtime.
|
| 101 |
+
|
| 102 |
+
**Residual risk:** Medium. Consider adding a signed SBOM or integrity check for dependencies.
|
| 103 |
+
|
| 104 |
+
### 3.9 Security Logging and Monitoring Failures (A09)
|
| 105 |
+
|
| 106 |
+
**Finding:** The gateway uses structured logging (slog) with JSON output to stdout. The API uses OpenTelemetry tracing and Prometheus metrics. Audit logs are written to PostgreSQL. However, there is no centralized log aggregation or alerting configured.
|
| 107 |
+
|
| 108 |
+
**Residual risk:** Medium. Recommend integrating a log aggregation system (e.g., Loki, CloudWatch) and alerting on security events (e.g., repeated 401 responses).
|
| 109 |
+
|
| 110 |
+
### 3.10 Server‑Side Request Forgery (SSRF) (A10)
|
| 111 |
+
|
| 112 |
+
**Finding:** The gateway proxies requests to the core API URL specified by the `ARF_CORE_URL` environment variable. If an attacker could manipulate this variable, they could redirect internal traffic. However, the variable is set at deployment time and cannot be modified via user input.
|
| 113 |
+
|
| 114 |
+
**Residual risk:** Low.
|
| 115 |
+
|
| 116 |
+
---
|
| 117 |
+
|
| 118 |
+
## 4. Additional Security Controls
|
| 119 |
+
|
| 120 |
+
### 4.1 Internal API Key Protection (v4.3.2)
|
| 121 |
+
|
| 122 |
+
The governance endpoints are now protected by an internal API key (`X‑Internal‑Key` header), verified via constant‑time comparison. This ensures that even if an attacker bypasses the gateway, the API itself is not open. The gateway injects this header for all authenticated requests.
|
| 123 |
+
|
| 124 |
+
### 4.2 Rate Limiting
|
| 125 |
+
|
| 126 |
+
The gateway implements a per‑API‑key token‑bucket rate limiter (`internal/middleware/ratelimit.go`). The default configuration (100 req/min, burst 20) is conservative but may need to be adjusted per tier in production.
|
| 127 |
+
|
| 128 |
+
### 4.3 Network Segmentation
|
| 129 |
+
|
| 130 |
+
The Kubernetes `NetworkPolicy` (`deploy/kubernetes/arf-api/networkpolicy.yaml`) restricts ingress to the API pods to only traffic from pods labeled `app: arf-gateway`. This provides defense‑in‑depth beyond the internal API key.
|
| 131 |
+
|
| 132 |
+
---
|
| 133 |
+
|
| 134 |
+
## 5. Penetration Test Findings (Steward‑Reported)
|
| 135 |
+
|
| 136 |
+
The steward performed manual penetration testing and reported the following:
|
| 137 |
+
|
| 138 |
+
| Finding | Severity | Status |
|
| 139 |
+
|---------|----------|--------|
|
| 140 |
+
| No authentication on governance endpoints (direct API access) | Critical | **Fixed** (v4.3.2, internal key) |
|
| 141 |
+
| CORS restricted to single origin | Informational | Accepted |
|
| 142 |
+
| Rate limiter bypassable via multiple API keys | Medium | **Open** (recommend per‑tier limits) |
|
| 143 |
+
| No brute‑force protection on API key validation | Medium | **Open** (gateway does not track failed attempts) |
|
| 144 |
+
|
| 145 |
+
---
|
| 146 |
+
|
| 147 |
+
## 6. Residual Risk Matrix
|
| 148 |
+
|
| 149 |
+
| Risk | Likelihood | Impact | Rating | Mitigation |
|
| 150 |
+
|------|------------|--------|--------|------------|
|
| 151 |
+
| Tier‑agnostic rate limiting | Medium | Low | Low | Implement per‑tier rate limits in gateway. |
|
| 152 |
+
| No brute‑force protection | Low | Medium | Low | Add exponential backoff or account lockout after N failed attempts. |
|
| 153 |
+
| Dependency vulnerabilities (unscanned) | Medium | Medium | Medium | Integrate automated scanning into CI. |
|
| 154 |
+
| No centralized log aggregation | High | Low | Medium | Add Loki or CloudWatch log shipping. |
|
| 155 |
+
| Internal key not enforced in dev mode | Low | High | Low | Ensure production Helm chart requires the key. |
|
| 156 |
+
|
| 157 |
+
---
|
| 158 |
+
|
| 159 |
+
## 7. Bayesian Confidence Model
|
| 160 |
+
|
| 161 |
+
We model the platform's security readiness as a Beta distribution over the probability that no critical security vulnerability exists. We start with a weak Beta(1,1) prior and update based on the findings from this assessment.
|
| 162 |
+
|
| 163 |
+
- **Positive evidence (α‑1):** API auth fixed, constant‑time key compare, Ed25519 signatures, hash‑chained logs, salted API key hashing, NetworkPolicy, CORS restriction, rate limiter.
|
| 164 |
+
- **Negative evidence (β‑1):** No brute‑force protection, dependency scanning not integrated, no centralized log alerting, rate limiter not tier‑specific.
|
| 165 |
+
|
| 166 |
+
Posterior: **Beta(9, 5)**. Posterior mean: **0.64**. This represents our current degree of belief that the platform is secure enough for a pilot. The remaining open items would shift this toward Beta(12,5) with mean ~0.71.
|
| 167 |
+
|
| 168 |
+
---
|
| 169 |
+
|
| 170 |
+
## 8. Recommendations
|
| 171 |
+
|
| 172 |
+
| Priority | Recommendation | Effort | Impact |
|
| 173 |
+
|----------|---------------|--------|--------|
|
| 174 |
+
| **P0** | Add brute‑force protection to gateway auth (account lockout after 5 failed attempts). | Small | High |
|
| 175 |
+
| **P0** | Implement per‑tier rate limiting in the gateway. | Medium | Medium |
|
| 176 |
+
| **P1** | Integrate `pip‑audit` and `govulncheck` into CI. | Small | Medium |
|
| 177 |
+
| **P1** | Add centralized log aggregation (Loki or CloudWatch). | Medium | Medium |
|
| 178 |
+
| **P2** | Produce a signed SBOM for the governance loop dependencies. | Small | Low |
|
| 179 |
+
|
| 180 |
+
---
|
| 181 |
+
|
| 182 |
+
## 9. Conclusion
|
| 183 |
+
|
| 184 |
+
ARF’s security posture is adequate for a controlled pilot deployment in a regulated environment, provided the P0 recommendations are addressed before production. The platform demonstrates strong integrity controls (Ed25519, hash chains) and authentication (salted SHA‑256, internal key). The residual risks are mitigable with relatively low effort.
|
| 185 |
+
|
| 186 |
+
*This document is proprietary and access‑controlled. Distribution is limited to qualified pilots and enterprise customers under written agreement.*
|
render.yaml
CHANGED
|
@@ -11,8 +11,6 @@ services:
|
|
| 11 |
property: connectionString
|
| 12 |
- key: API_KEY
|
| 13 |
sync: false
|
| 14 |
-
- key: ARF_KEY_PEPPER
|
| 15 |
-
sync: false
|
| 16 |
- key: ENVIRONMENT
|
| 17 |
value: production
|
| 18 |
databases:
|
|
|
|
| 11 |
property: connectionString
|
| 12 |
- key: API_KEY
|
| 13 |
sync: false
|
|
|
|
|
|
|
| 14 |
- key: ENVIRONMENT
|
| 15 |
value: production
|
| 16 |
databases:
|
requirements.txt
CHANGED
|
@@ -9,14 +9,9 @@ pydantic-settings
|
|
| 9 |
sqlalchemy
|
| 10 |
psycopg2-binary==2.9.10
|
| 11 |
slowapi==0.1.9
|
| 12 |
-
limits==3.3.1 # slowapi 0.1.9's own poetry.lock pins this; left unpinned it resolves
|
| 13 |
-
# to a much newer major version whose internal API to slowapi breaks
|
| 14 |
-
# (Limiter._check_request_limit raises a plain ValueError instead of
|
| 15 |
-
# RateLimitExceeded, which slowapi's middleware mishandles as an
|
| 16 |
-
# unhandled 500 on every request touching a rate limit)
|
| 17 |
prometheus-fastapi-instrumentator==7.1.0
|
| 18 |
flake8==7.2.0
|
| 19 |
-
cryptography==
|
| 20 |
sentence-transformers>=2.2.0
|
| 21 |
scikit-learn
|
| 22 |
redis>=4.0.0 # optional, for faster counters
|
|
|
|
| 9 |
sqlalchemy
|
| 10 |
psycopg2-binary==2.9.10
|
| 11 |
slowapi==0.1.9
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 12 |
prometheus-fastapi-instrumentator==7.1.0
|
| 13 |
flake8==7.2.0
|
| 14 |
+
cryptography==48.0.1
|
| 15 |
sentence-transformers>=2.2.0
|
| 16 |
scikit-learn
|
| 17 |
redis>=4.0.0 # optional, for faster counters
|
tests/conftest.py
CHANGED
|
@@ -18,10 +18,6 @@ import pytest
|
|
| 18 |
# ===== STEP 1: Set environment variables BEFORE any app imports =====
|
| 19 |
os.environ["ARF_USAGE_TRACKING"] = "false"
|
| 20 |
|
| 21 |
-
# UsageTracker now requires a real pepper to hash/verify API keys (H-2 fix);
|
| 22 |
-
# tests never touch a production key, so a fixed test-only value is fine.
|
| 23 |
-
os.environ.setdefault("ARF_KEY_PEPPER", "test-only-pepper-not-for-production-use-32chars")
|
| 24 |
-
|
| 25 |
# Force the correct database URL for tests
|
| 26 |
os.environ["DATABASE_URL"] = "postgresql://postgres:postgres@localhost:5432/testdb"
|
| 27 |
os.environ["TEST_DATABASE_URL"] = "postgresql://postgres:postgres@localhost:5432/testdb"
|
|
@@ -46,9 +42,6 @@ class MockTracker:
|
|
| 46 |
|
| 47 |
return 1000
|
| 48 |
|
| 49 |
-
def get_tenant_id(self, api_key):
|
| 50 |
-
return "test-tenant"
|
| 51 |
-
|
| 52 |
def consume_quota_and_log(self, record, idempotency_key=None):
|
| 53 |
|
| 54 |
return (True, None)
|
|
|
|
| 18 |
# ===== STEP 1: Set environment variables BEFORE any app imports =====
|
| 19 |
os.environ["ARF_USAGE_TRACKING"] = "false"
|
| 20 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
# Force the correct database URL for tests
|
| 22 |
os.environ["DATABASE_URL"] = "postgresql://postgres:postgres@localhost:5432/testdb"
|
| 23 |
os.environ["TEST_DATABASE_URL"] = "postgresql://postgres:postgres@localhost:5432/testdb"
|
|
|
|
| 42 |
|
| 43 |
return 1000
|
| 44 |
|
|
|
|
|
|
|
|
|
|
| 45 |
def consume_quota_and_log(self, record, idempotency_key=None):
|
| 46 |
|
| 47 |
return (True, None)
|
tests/test_deps.py
CHANGED
|
@@ -1,10 +1,5 @@
|
|
| 1 |
-
import importlib
|
| 2 |
-
from unittest.mock import MagicMock, patch
|
| 3 |
-
|
| 4 |
import pytest
|
| 5 |
-
from
|
| 6 |
-
|
| 7 |
-
import app.api.deps as deps
|
| 8 |
from app.api.deps import get_db
|
| 9 |
|
| 10 |
|
|
@@ -18,72 +13,3 @@ def test_get_db_closes_session():
|
|
| 18 |
with pytest.raises(Exception):
|
| 19 |
db_gen.throw(Exception("test error"))
|
| 20 |
mock_session.close.assert_called_once()
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
# verify_internal_key tests below are called directly, not through
|
| 24 |
-
# TestClient: tests/conftest.py globally overrides verify_internal_key
|
| 25 |
-
# (`fastapi_app.dependency_overrides[verify_internal_key] = mock_verify_internal_key`)
|
| 26 |
-
# so that already-protected routes (routes_governance.py) can be exercised
|
| 27 |
-
# in tests without the gateway-injected X-Internal-Key header. That override
|
| 28 |
-
# makes the real fail-closed behavior untestable through the app for any
|
| 29 |
-
# router that uses it -- this is the only place it's actually verified to
|
| 30 |
-
# reject what it should reject, rather than just trusted to work because
|
| 31 |
-
# it's wired in.
|
| 32 |
-
#
|
| 33 |
-
# Newly relevant as of the auth fix to routes_risk.py, routes_intents.py,
|
| 34 |
-
# routes_history.py, routes_memory.py (see docs/authentication.md) -- those
|
| 35 |
-
# four routers now depend on this function passing correctly.
|
| 36 |
-
|
| 37 |
-
_NEWLY_PROTECTED_ROUTER_MODULES = [
|
| 38 |
-
"app.api.routes_risk",
|
| 39 |
-
"app.api.routes_intents",
|
| 40 |
-
"app.api.routes_history",
|
| 41 |
-
"app.api.routes_memory",
|
| 42 |
-
]
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
@pytest.mark.asyncio
|
| 46 |
-
async def test_verify_internal_key_rejects_missing_header(monkeypatch):
|
| 47 |
-
monkeypatch.setattr(deps, "INTERNAL_API_KEY", "real-secret")
|
| 48 |
-
with pytest.raises(HTTPException) as exc_info:
|
| 49 |
-
await deps.verify_internal_key(x_internal_key=None)
|
| 50 |
-
assert exc_info.value.status_code == 401
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
@pytest.mark.asyncio
|
| 54 |
-
async def test_verify_internal_key_rejects_wrong_key(monkeypatch):
|
| 55 |
-
monkeypatch.setattr(deps, "INTERNAL_API_KEY", "real-secret")
|
| 56 |
-
with pytest.raises(HTTPException) as exc_info:
|
| 57 |
-
await deps.verify_internal_key(x_internal_key="wrong-key")
|
| 58 |
-
assert exc_info.value.status_code == 401
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
@pytest.mark.asyncio
|
| 62 |
-
async def test_verify_internal_key_fails_closed_when_unset(monkeypatch):
|
| 63 |
-
"""The env var being unset must reject every request, not let them
|
| 64 |
-
through -- this is the specific property that makes this safe to add
|
| 65 |
-
to a router without also needing to guarantee the env var is always
|
| 66 |
-
set."""
|
| 67 |
-
monkeypatch.setattr(deps, "INTERNAL_API_KEY", "")
|
| 68 |
-
with pytest.raises(HTTPException) as exc_info:
|
| 69 |
-
await deps.verify_internal_key(x_internal_key="anything")
|
| 70 |
-
assert exc_info.value.status_code == 401
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
@pytest.mark.asyncio
|
| 74 |
-
async def test_verify_internal_key_accepts_correct_key(monkeypatch):
|
| 75 |
-
monkeypatch.setattr(deps, "INTERNAL_API_KEY", "real-secret")
|
| 76 |
-
result = await deps.verify_internal_key(x_internal_key="real-secret")
|
| 77 |
-
assert result is None
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
@pytest.mark.parametrize("router_module_name", _NEWLY_PROTECTED_ROUTER_MODULES)
|
| 81 |
-
def test_router_requires_verify_internal_key(router_module_name):
|
| 82 |
-
"""Structural check, independent of the function-level tests above:
|
| 83 |
-
proves each router actually declares verify_internal_key as a
|
| 84 |
-
router-level dependency, not just that the function itself works in
|
| 85 |
-
isolation. Mirrors the pattern routes_governance.py already uses
|
| 86 |
-
(`APIRouter(dependencies=[Depends(verify_internal_key)])`)."""
|
| 87 |
-
module = importlib.import_module(router_module_name)
|
| 88 |
-
dependency_callables = [d.dependency for d in module.router.dependencies]
|
| 89 |
-
assert deps.verify_internal_key in dependency_callables
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
import pytest
|
| 2 |
+
from unittest.mock import patch, MagicMock
|
|
|
|
|
|
|
| 3 |
from app.api.deps import get_db
|
| 4 |
|
| 5 |
|
|
|
|
| 13 |
with pytest.raises(Exception):
|
| 14 |
db_gen.throw(Exception("test error"))
|
| 15 |
mock_session.close.assert_called_once()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tests/test_governance.py
CHANGED
|
@@ -1,11 +1,7 @@
|
|
| 1 |
"""
|
| 2 |
Tests for governance endpoints: /api/v1/intents/evaluate
|
| 3 |
"""
|
| 4 |
-
import tempfile
|
| 5 |
-
|
| 6 |
import pytest
|
| 7 |
-
import app.core.usage_tracker as usage_tracker_module
|
| 8 |
-
from app.core.usage_tracker import UsageTracker
|
| 9 |
from app.database.models_intents import TenantDB
|
| 10 |
|
| 11 |
|
|
@@ -117,36 +113,3 @@ def test_evaluate_with_criticality(client):
|
|
| 117 |
# context_hash is computed by the governance loop (a 64‑char hex string)
|
| 118 |
ctx_hash = healing.get("context_hash")
|
| 119 |
assert isinstance(ctx_hash, str) and len(ctx_hash) == 64
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
def test_evaluate_intent_against_real_tracker_does_not_crash(client, monkeypatch):
|
| 123 |
-
"""Every other test in this file runs against tests/conftest.py's
|
| 124 |
-
MockTracker, whose consume_quota_and_log ignores record.tier entirely
|
| 125 |
-
and always returns (True, None) -- so none of them would have noticed
|
| 126 |
-
that this endpoint hardcoded tier=None instead of using quota["tier"]
|
| 127 |
-
(already resolved by the enforce_quota dependency). Against the real
|
| 128 |
-
UsageTracker, consume_quota_and_log evaluates tier.monthly_evaluation_limit
|
| 129 |
-
unconditionally, so a None tier raised AttributeError on every real
|
| 130 |
-
call, outside any try/except, surfacing as a raw 500. This test swaps
|
| 131 |
-
in a real UsageTracker (a throwaway SQLite file; the same real
|
| 132 |
-
Postgres the CI service provides backs api_keys/monthly_counts) to
|
| 133 |
-
prove the endpoint no longer crashes."""
|
| 134 |
-
with tempfile.NamedTemporaryFile(suffix=".db") as tmp:
|
| 135 |
-
monkeypatch.setattr(usage_tracker_module, "tracker", UsageTracker(db_path=tmp.name))
|
| 136 |
-
|
| 137 |
-
payload = {
|
| 138 |
-
"intent_type": "provision_resource",
|
| 139 |
-
"environment": "prod",
|
| 140 |
-
"resource_type": "database",
|
| 141 |
-
"region": "eastus",
|
| 142 |
-
"size": "Standard",
|
| 143 |
-
"estimated_cost": 1200,
|
| 144 |
-
"policy_violations": [],
|
| 145 |
-
"requester": "alice",
|
| 146 |
-
"provenance": {},
|
| 147 |
-
"configuration": {}
|
| 148 |
-
}
|
| 149 |
-
response = client.post("/api/v1/intents/evaluate", json=payload,
|
| 150 |
-
headers={"X-Tenant-ID": "test-tenant"})
|
| 151 |
-
assert response.status_code == 200, response.text
|
| 152 |
-
assert "risk_score" in response.json()
|
|
|
|
| 1 |
"""
|
| 2 |
Tests for governance endpoints: /api/v1/intents/evaluate
|
| 3 |
"""
|
|
|
|
|
|
|
| 4 |
import pytest
|
|
|
|
|
|
|
| 5 |
from app.database.models_intents import TenantDB
|
| 6 |
|
| 7 |
|
|
|
|
| 113 |
# context_hash is computed by the governance loop (a 64‑char hex string)
|
| 114 |
ctx_hash = healing.get("context_hash")
|
| 115 |
assert isinstance(ctx_hash, str) and len(ctx_hash) == 64
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tests/test_healing_endpoint.py
CHANGED
|
@@ -1,9 +1,5 @@
|
|
| 1 |
-
import tempfile
|
| 2 |
-
|
| 3 |
from fastapi.testclient import TestClient
|
| 4 |
from app.main import app
|
| 5 |
-
import app.core.usage_tracker as usage_tracker_module
|
| 6 |
-
from app.core.usage_tracker import UsageTracker
|
| 7 |
|
| 8 |
client = TestClient(app)
|
| 9 |
|
|
@@ -22,27 +18,3 @@ def test_healing_evaluate_endpoint():
|
|
| 22 |
response = client.post("/api/v1/healing/evaluate", json=payload,
|
| 23 |
headers={"X-Tenant-ID": "test-tenant"})
|
| 24 |
assert response.status_code == 200, f"Expected 200, got {response.status_code}: {response.text}"
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
def test_healing_evaluate_against_real_tracker_does_not_crash(monkeypatch):
|
| 28 |
-
"""Same regression as test_governance.py's equivalent test, for this
|
| 29 |
-
endpoint's own hardcoded tier=None (routes_governance.py's
|
| 30 |
-
/healing/evaluate handler) -- see that test's docstring for the full
|
| 31 |
-
explanation. Swaps in a real UsageTracker to prove
|
| 32 |
-
consume_quota_and_log no longer crashes on a None tier here either."""
|
| 33 |
-
payload = {
|
| 34 |
-
"event": {
|
| 35 |
-
"component": "my-service",
|
| 36 |
-
"latency_p99": 450.0,
|
| 37 |
-
"error_rate": 0.25,
|
| 38 |
-
"service_mesh": "default",
|
| 39 |
-
"cpu_util": 0.85,
|
| 40 |
-
"memory_util": 0.90
|
| 41 |
-
}
|
| 42 |
-
}
|
| 43 |
-
with tempfile.NamedTemporaryFile(suffix=".db") as tmp:
|
| 44 |
-
monkeypatch.setattr(usage_tracker_module, "tracker", UsageTracker(db_path=tmp.name))
|
| 45 |
-
|
| 46 |
-
response = client.post("/api/v1/healing/evaluate", json=payload,
|
| 47 |
-
headers={"X-Tenant-ID": "test-tenant"})
|
| 48 |
-
assert response.status_code == 200, f"Expected 200, got {response.status_code}: {response.text}"
|
|
|
|
|
|
|
|
|
|
| 1 |
from fastapi.testclient import TestClient
|
| 2 |
from app.main import app
|
|
|
|
|
|
|
| 3 |
|
| 4 |
client = TestClient(app)
|
| 5 |
|
|
|
|
| 18 |
response = client.post("/api/v1/healing/evaluate", json=payload,
|
| 19 |
headers={"X-Tenant-ID": "test-tenant"})
|
| 20 |
assert response.status_code == 200, f"Expected 200, got {response.status_code}: {response.text}"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tests/test_history.py
CHANGED
|
@@ -8,5 +8,9 @@ def test_history():
|
|
| 8 |
response = client.get("/api/v1/history")
|
| 9 |
assert response.status_code == 200
|
| 10 |
data = response.json()
|
| 11 |
-
|
| 12 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
response = client.get("/api/v1/history")
|
| 9 |
assert response.status_code == 200
|
| 10 |
data = response.json()
|
| 11 |
+
# The endpoint returns a list of risk points, not an object with an
|
| 12 |
+
# "incidents" key
|
| 13 |
+
assert isinstance(data, list)
|
| 14 |
+
if data: # if not empty, verify the structure of the first item
|
| 15 |
+
assert "risk" in data[0]
|
| 16 |
+
assert "time" in data[0]
|
tests/test_payments.py
CHANGED
|
@@ -1,19 +1,17 @@
|
|
|
|
|
| 1 |
import pytest
|
| 2 |
from unittest.mock import patch, MagicMock
|
| 3 |
from fastapi.testclient import TestClient
|
| 4 |
from app.main import app
|
| 5 |
|
| 6 |
-
# Every Stripe call in this module is mocked -- nothing here reaches the
|
| 7 |
-
# network, and no real credentials are involved. This module used to skip
|
| 8 |
-
# entirely whenever STRIPE_SECRET_KEY was unset (always, in CI), so none of
|
| 9 |
-
# it ran; stripe.api_key is patched below instead of relying on env vars,
|
| 10 |
-
# since routes_payments.py reads it at import time.
|
| 11 |
client = TestClient(app)
|
| 12 |
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
|
|
|
|
|
|
| 17 |
|
| 18 |
|
| 19 |
@pytest.fixture
|
|
@@ -23,13 +21,11 @@ def mock_stripe():
|
|
| 23 |
|
| 24 |
|
| 25 |
def test_create_checkout_session_missing_stripe_key(monkeypatch):
|
| 26 |
-
|
| 27 |
-
# time and checks `stripe.api_key`, so setenv here would be a no-op.
|
| 28 |
-
monkeypatch.setattr("stripe.api_key", None)
|
| 29 |
response = client.post(
|
| 30 |
"/api/v1/payments/create-checkout-session",
|
| 31 |
-
headers={"Authorization": "Bearer test_key"},
|
| 32 |
json={
|
|
|
|
| 33 |
"success_url": "https://example.com/success",
|
| 34 |
"cancel_url": "https://example.com/cancel"})
|
| 35 |
assert response.status_code == 500
|
|
@@ -40,13 +36,12 @@ def test_create_checkout_session_free_key(mock_stripe):
|
|
| 40 |
# Mock tracker.get_tier to return Tier.FREE
|
| 41 |
with patch("app.core.usage_tracker.tracker") as mock_tracker:
|
| 42 |
mock_tracker.get_tier.return_value = "free"
|
| 43 |
-
mock_tracker.get_tenant_id.return_value = "tenant_test_123"
|
| 44 |
mock_stripe.return_value = MagicMock(
|
| 45 |
id="cs_test_123", url="https://checkout.stripe.com/pay")
|
| 46 |
response = client.post(
|
| 47 |
"/api/v1/payments/create-checkout-session",
|
| 48 |
-
headers={"Authorization": "Bearer test_key"},
|
| 49 |
json={
|
|
|
|
| 50 |
"success_url": "https://example.com/success",
|
| 51 |
"cancel_url": "https://example.com/cancel"})
|
| 52 |
assert response.status_code == 200
|
|
@@ -60,39 +55,10 @@ def test_create_checkout_session_pro_key():
|
|
| 60 |
mock_tracker.get_tier.return_value = "pro"
|
| 61 |
response = client.post(
|
| 62 |
"/api/v1/payments/create-checkout-session",
|
| 63 |
-
headers={"Authorization": "Bearer test_key"},
|
| 64 |
json={
|
|
|
|
| 65 |
"success_url": "https://example.com/success",
|
| 66 |
"cancel_url": "https://example.com/cancel"})
|
| 67 |
assert response.status_code == 400
|
| 68 |
assert "Only free tier keys can be upgraded" in response.json()[
|
| 69 |
"detail"]
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
def test_create_checkout_session_requires_authentication(mock_stripe):
|
| 73 |
-
# Regression test: this endpoint used to take `api_key` as a plain JSON
|
| 74 |
-
# body field with no Depends() gate at all, so any caller could request
|
| 75 |
-
# a session for any tenant_id-bearing key string without proving they
|
| 76 |
-
# held it. No Authorization header (and no api_key in the body -- the
|
| 77 |
-
# field no longer exists on CheckoutRequest) must be rejected before
|
| 78 |
-
# ever reaching Stripe.
|
| 79 |
-
response = client.post(
|
| 80 |
-
"/api/v1/payments/create-checkout-session",
|
| 81 |
-
json={
|
| 82 |
-
"success_url": "https://example.com/success",
|
| 83 |
-
"cancel_url": "https://example.com/cancel"})
|
| 84 |
-
assert response.status_code == 401
|
| 85 |
-
mock_stripe.assert_not_called()
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
def test_create_checkout_session_rejects_invalid_key(mock_stripe):
|
| 89 |
-
with patch("app.core.usage_tracker.tracker") as mock_tracker:
|
| 90 |
-
mock_tracker.get_tier.return_value = None
|
| 91 |
-
response = client.post(
|
| 92 |
-
"/api/v1/payments/create-checkout-session",
|
| 93 |
-
headers={"Authorization": "Bearer not-a-real-key"},
|
| 94 |
-
json={
|
| 95 |
-
"success_url": "https://example.com/success",
|
| 96 |
-
"cancel_url": "https://example.com/cancel"})
|
| 97 |
-
assert response.status_code == 403
|
| 98 |
-
mock_stripe.assert_not_called()
|
|
|
|
| 1 |
+
import os
|
| 2 |
import pytest
|
| 3 |
from unittest.mock import patch, MagicMock
|
| 4 |
from fastapi.testclient import TestClient
|
| 5 |
from app.main import app
|
| 6 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7 |
client = TestClient(app)
|
| 8 |
|
| 9 |
+
# Skip all tests in this module if Stripe secret key is not set
|
| 10 |
+
STRIPE_SECRET_KEY = os.getenv("STRIPE_SECRET_KEY")
|
| 11 |
+
if not STRIPE_SECRET_KEY:
|
| 12 |
+
pytest.skip(
|
| 13 |
+
"Stripe not configured – skipping payment tests",
|
| 14 |
+
allow_module_level=True)
|
| 15 |
|
| 16 |
|
| 17 |
@pytest.fixture
|
|
|
|
| 21 |
|
| 22 |
|
| 23 |
def test_create_checkout_session_missing_stripe_key(monkeypatch):
|
| 24 |
+
monkeypatch.setenv("STRIPE_SECRET_KEY", "")
|
|
|
|
|
|
|
| 25 |
response = client.post(
|
| 26 |
"/api/v1/payments/create-checkout-session",
|
|
|
|
| 27 |
json={
|
| 28 |
+
"api_key": "test_key",
|
| 29 |
"success_url": "https://example.com/success",
|
| 30 |
"cancel_url": "https://example.com/cancel"})
|
| 31 |
assert response.status_code == 500
|
|
|
|
| 36 |
# Mock tracker.get_tier to return Tier.FREE
|
| 37 |
with patch("app.core.usage_tracker.tracker") as mock_tracker:
|
| 38 |
mock_tracker.get_tier.return_value = "free"
|
|
|
|
| 39 |
mock_stripe.return_value = MagicMock(
|
| 40 |
id="cs_test_123", url="https://checkout.stripe.com/pay")
|
| 41 |
response = client.post(
|
| 42 |
"/api/v1/payments/create-checkout-session",
|
|
|
|
| 43 |
json={
|
| 44 |
+
"api_key": "test_key",
|
| 45 |
"success_url": "https://example.com/success",
|
| 46 |
"cancel_url": "https://example.com/cancel"})
|
| 47 |
assert response.status_code == 200
|
|
|
|
| 55 |
mock_tracker.get_tier.return_value = "pro"
|
| 56 |
response = client.post(
|
| 57 |
"/api/v1/payments/create-checkout-session",
|
|
|
|
| 58 |
json={
|
| 59 |
+
"api_key": "test_key",
|
| 60 |
"success_url": "https://example.com/success",
|
| 61 |
"cancel_url": "https://example.com/cancel"})
|
| 62 |
assert response.status_code == 400
|
| 63 |
assert "Only free tier keys can be upgraded" in response.json()[
|
| 64 |
"detail"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tests/test_risk.py
CHANGED
|
@@ -31,8 +31,4 @@ def test_get_risk_internal_error(client, monkeypatch):
|
|
| 31 |
"X-API-Key": "test-key"})
|
| 32 |
assert response.status_code == 500
|
| 33 |
data = response.json()
|
| 34 |
-
|
| 35 |
-
# logs the real exception server-side and returns a generic detail
|
| 36 |
-
# instead (information-disclosure fix from this session's audit).
|
| 37 |
-
assert data.get("detail") == "Internal server error"
|
| 38 |
-
assert "test error" not in data.get("detail", "")
|
|
|
|
| 31 |
"X-API-Key": "test-key"})
|
| 32 |
assert response.status_code == 500
|
| 33 |
data = response.json()
|
| 34 |
+
assert "test error" in data.get("detail", "")
|
|
|
|
|
|
|
|
|
|
|
|
tests/test_routes_admin.py
DELETED
|
@@ -1,126 +0,0 @@
|
|
| 1 |
-
"""
|
| 2 |
-
Route-level tests for the /admin/keys endpoints against the real,
|
| 3 |
-
Postgres-backed UsageTracker -- these routes are normally exercised through
|
| 4 |
-
the module-level `tracker` singleton (conftest.py replaces it globally with
|
| 5 |
-
MockTracker for every other test), so `tracker` is patched on
|
| 6 |
-
`app.core.usage_tracker` and `ADMIN_API_KEY` on `app.api.routes_admin` for
|
| 7 |
-
the duration of this module.
|
| 8 |
-
|
| 9 |
-
`tracker` is patched on the *defining* module, not on routes_admin, because
|
| 10 |
-
routes_admin no longer holds its own binding: it reaches the singleton
|
| 11 |
-
through `usage_tracker.tracker` so that init_tracker()'s rebinding is
|
| 12 |
-
visible to it. That is also what makes one patch here sufficient where five
|
| 13 |
-
would otherwise be needed.
|
| 14 |
-
|
| 15 |
-
This is the main regression coverage for the api_keys-on-Postgres
|
| 16 |
-
migration: it exercises the exact raw SQL routes_admin.py runs against
|
| 17 |
-
`usage_tracker.tracker._get_pg_conn()`.
|
| 18 |
-
"""
|
| 19 |
-
import hashlib
|
| 20 |
-
import hmac
|
| 21 |
-
import os
|
| 22 |
-
|
| 23 |
-
import pytest
|
| 24 |
-
|
| 25 |
-
from app.core import usage_tracker
|
| 26 |
-
from app.core.usage_tracker import UsageTracker
|
| 27 |
-
import app.api.routes_admin as routes_admin
|
| 28 |
-
|
| 29 |
-
TEST_ADMIN_KEY = "test-admin-key-for-routes-admin-tests"
|
| 30 |
-
TEST_PEPPER = os.environ["ARF_KEY_PEPPER"] # set in conftest.py before app import
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
def _key_id(raw_key: str) -> str:
|
| 34 |
-
"""Reproduce UsageTracker._lookup_hash without a tracker instance, so
|
| 35 |
-
tests can locate the row created for a given raw key deterministically."""
|
| 36 |
-
return hmac.new(TEST_PEPPER.encode(), raw_key.encode(), hashlib.sha256).hexdigest()
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
@pytest.fixture(autouse=True)
|
| 40 |
-
def real_tracker(monkeypatch):
|
| 41 |
-
real = UsageTracker(db_path=":memory:")
|
| 42 |
-
monkeypatch.setattr(usage_tracker, "tracker", real)
|
| 43 |
-
monkeypatch.setattr(routes_admin, "ADMIN_API_KEY", TEST_ADMIN_KEY)
|
| 44 |
-
yield real
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
def test_create_list_update_deactivate_key(client):
|
| 48 |
-
create_resp = client.post(
|
| 49 |
-
"/api/v1/admin/keys",
|
| 50 |
-
params={"admin_key": TEST_ADMIN_KEY},
|
| 51 |
-
json={"tier": "free", "org_name": "Test Org"},
|
| 52 |
-
)
|
| 53 |
-
assert create_resp.status_code == 200
|
| 54 |
-
body = create_resp.json()
|
| 55 |
-
api_key = body["api_key"]
|
| 56 |
-
assert body["tier"] == "free"
|
| 57 |
-
key_id = _key_id(api_key)
|
| 58 |
-
|
| 59 |
-
list_resp = client.get("/api/v1/admin/keys", params={"admin_key": TEST_ADMIN_KEY})
|
| 60 |
-
assert list_resp.status_code == 200
|
| 61 |
-
keys_by_id = {row["key_id"]: row for row in list_resp.json()["keys"]}
|
| 62 |
-
assert key_id in keys_by_id
|
| 63 |
-
assert keys_by_id[key_id]["tier"] == "free"
|
| 64 |
-
assert keys_by_id[key_id]["is_active"] is True
|
| 65 |
-
|
| 66 |
-
patch_resp = client.patch(
|
| 67 |
-
f"/api/v1/admin/keys/{key_id}/tier",
|
| 68 |
-
params={"admin_key": TEST_ADMIN_KEY},
|
| 69 |
-
json={"tier": "pro"},
|
| 70 |
-
)
|
| 71 |
-
assert patch_resp.status_code == 200
|
| 72 |
-
|
| 73 |
-
list_resp2 = client.get("/api/v1/admin/keys", params={"admin_key": TEST_ADMIN_KEY})
|
| 74 |
-
assert list_resp2.json()["keys"][0] # non-empty, sanity check
|
| 75 |
-
keys_by_id2 = {row["key_id"]: row for row in list_resp2.json()["keys"]}
|
| 76 |
-
assert keys_by_id2[key_id]["tier"] == "pro"
|
| 77 |
-
|
| 78 |
-
delete_resp = client.delete(f"/api/v1/admin/keys/{key_id}", params={"admin_key": TEST_ADMIN_KEY})
|
| 79 |
-
assert delete_resp.status_code == 200
|
| 80 |
-
|
| 81 |
-
list_resp3 = client.get("/api/v1/admin/keys", params={"admin_key": TEST_ADMIN_KEY})
|
| 82 |
-
keys_by_id3 = {row["key_id"]: row for row in list_resp3.json()["keys"]}
|
| 83 |
-
assert keys_by_id3[key_id]["is_active"] is False
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
def test_update_nonexistent_key_returns_404(client):
|
| 87 |
-
resp = client.patch(
|
| 88 |
-
"/api/v1/admin/keys/does-not-exist/tier",
|
| 89 |
-
params={"admin_key": TEST_ADMIN_KEY},
|
| 90 |
-
json={"tier": "pro"},
|
| 91 |
-
)
|
| 92 |
-
assert resp.status_code == 404
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
def test_rotate_key_deactivates_old_and_creates_new_on_same_tenant(client):
|
| 96 |
-
create_resp = client.post(
|
| 97 |
-
"/api/v1/admin/keys",
|
| 98 |
-
params={"admin_key": TEST_ADMIN_KEY},
|
| 99 |
-
json={"tier": "pro", "org_name": "Rotate Test Org"},
|
| 100 |
-
)
|
| 101 |
-
assert create_resp.status_code == 200
|
| 102 |
-
old_body = create_resp.json()
|
| 103 |
-
old_key_id = _key_id(old_body["api_key"])
|
| 104 |
-
tenant_id = old_body["tenant_id"]
|
| 105 |
-
|
| 106 |
-
rotate_resp = client.post(
|
| 107 |
-
f"/api/v1/admin/keys/{old_key_id}/rotate", params={"admin_key": TEST_ADMIN_KEY})
|
| 108 |
-
assert rotate_resp.status_code == 200
|
| 109 |
-
rotated = rotate_resp.json()
|
| 110 |
-
assert rotated["tenant_id"] == tenant_id
|
| 111 |
-
assert rotated["tier"] == "pro"
|
| 112 |
-
assert rotated["deactivated_key_id"] == old_key_id
|
| 113 |
-
new_key_id = _key_id(rotated["api_key"])
|
| 114 |
-
assert new_key_id != old_key_id
|
| 115 |
-
|
| 116 |
-
list_resp = client.get("/api/v1/admin/keys", params={"admin_key": TEST_ADMIN_KEY})
|
| 117 |
-
keys_by_id = {row["key_id"]: row for row in list_resp.json()["keys"]}
|
| 118 |
-
assert keys_by_id[old_key_id]["is_active"] is False
|
| 119 |
-
assert keys_by_id[new_key_id]["is_active"] is True
|
| 120 |
-
assert keys_by_id[new_key_id]["tier"] == "pro"
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
def test_rotate_nonexistent_key_returns_404(client):
|
| 124 |
-
resp = client.post(
|
| 125 |
-
"/api/v1/admin/keys/does-not-exist/rotate", params={"admin_key": TEST_ADMIN_KEY})
|
| 126 |
-
assert resp.status_code == 404
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tests/test_routes_governance_execute.py
DELETED
|
@@ -1,352 +0,0 @@
|
|
| 1 |
-
"""
|
| 2 |
-
Tests for POST /intents/{id}/execute and POST /admin/executions/{id}/resolve.
|
| 3 |
-
|
| 4 |
-
arf_enterprise is not installed in this test environment (by design -- it's
|
| 5 |
-
an optional, proprietary package; see routes_governance.py's
|
| 6 |
-
ENTERPRISE_EXECUTOR_AVAILABLE guard). The 501 "not available"/"not enabled"
|
| 7 |
-
paths are real, unconditional behavior and tested as such. For the
|
| 8 |
-
success/pending/error paths, the enterprise classes referenced in
|
| 9 |
-
routes_governance.py (EnterpriseExecutor, FakeCloudActuator,
|
| 10 |
-
PendingApprovalError, EnterpriseExecutionError, EnterpriseSafetyError) are
|
| 11 |
-
monkeypatched with lightweight stand-ins -- this tests arf-api's own glue
|
| 12 |
-
code (existence/tenant check, exception-to-HTTP-status mapping, response
|
| 13 |
-
shaping), not EnterpriseExecutor's internals, which are already covered by
|
| 14 |
-
the enterprise repo's own test suite.
|
| 15 |
-
"""
|
| 16 |
-
import pytest
|
| 17 |
-
from app.database.models_intents import TenantDB
|
| 18 |
-
import app.api.routes_governance as routes_governance
|
| 19 |
-
import app.api.routes_admin as routes_admin
|
| 20 |
-
|
| 21 |
-
TENANT_ID = "test-tenant"
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
@pytest.fixture(autouse=True)
|
| 25 |
-
def seed_tenant(db_session):
|
| 26 |
-
tenant = db_session.query(TenantDB).filter_by(id=TENANT_ID).first()
|
| 27 |
-
if not tenant:
|
| 28 |
-
db_session.add(TenantDB(id=TENANT_ID, name="Test Tenant"))
|
| 29 |
-
db_session.commit()
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
def _evaluate_intent(client):
|
| 33 |
-
payload = {
|
| 34 |
-
"intent_type": "provision_resource",
|
| 35 |
-
"environment": "prod",
|
| 36 |
-
"resource_type": "database",
|
| 37 |
-
"region": "eastus",
|
| 38 |
-
"size": "Standard",
|
| 39 |
-
"estimated_cost": 1200,
|
| 40 |
-
"policy_violations": [],
|
| 41 |
-
"requester": "alice",
|
| 42 |
-
"provenance": {},
|
| 43 |
-
"configuration": {},
|
| 44 |
-
}
|
| 45 |
-
resp = client.post(
|
| 46 |
-
"/api/v1/intents/evaluate", json=payload, headers={"X-Tenant-ID": TENANT_ID}
|
| 47 |
-
)
|
| 48 |
-
assert resp.status_code == 200, resp.text
|
| 49 |
-
data = resp.json()
|
| 50 |
-
# data["intent_id"] (top level, set by evaluate_intent_endpoint right
|
| 51 |
-
# before returning: result["intent_id"] = deterministic_id) is what
|
| 52 |
-
# save_evaluated_intent actually persisted to IntentDB.deterministic_id.
|
| 53 |
-
# data["healing_intent"]["intent_id"] is a separate, independently
|
| 54 |
-
# generated id belonging to the HealingIntent object itself -- using it
|
| 55 |
-
# here instead was the exact bug that made every test in this file 404.
|
| 56 |
-
return data["intent_id"], data["healing_intent"]
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
class _FakePendingApprovalError(Exception):
|
| 60 |
-
def __init__(self, message, level, approval_required, approval_id=None):
|
| 61 |
-
super().__init__(message)
|
| 62 |
-
self.level = level
|
| 63 |
-
self.approval_required = approval_required
|
| 64 |
-
self.approval_id = approval_id
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
class _FakeExecutionError(Exception):
|
| 68 |
-
pass
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
class _FakeSafetyError(Exception):
|
| 72 |
-
pass
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
class _FakeConfig:
|
| 76 |
-
"""Stands in for EnterpriseConfig, which is None in the import fallback
|
| 77 |
-
whenever arf_enterprise isn't installed -- as it isn't in CI, since it's
|
| 78 |
-
a private-repo package deliberately kept out of requirements.txt.
|
| 79 |
-
|
| 80 |
-
Records what it was constructed with so a test can assert that
|
| 81 |
-
ARF_TRUSTED_SIGNING_KEYS actually reaches the executor. Without that
|
| 82 |
-
the trust store could silently go back to empty and every signed intent
|
| 83 |
-
would be rejected as "Untrusted signing key" with nothing failing here.
|
| 84 |
-
"""
|
| 85 |
-
last_trusted_signing_keys = None
|
| 86 |
-
|
| 87 |
-
def __init__(self, trusted_signing_keys=None, **kwargs):
|
| 88 |
-
self.trusted_signing_keys = trusted_signing_keys
|
| 89 |
-
type(self).last_trusted_signing_keys = trusted_signing_keys
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
class _FakeExecutor:
|
| 93 |
-
"""Stands in for EnterpriseExecutor. Behavior is selected via a
|
| 94 |
-
class-level `mode` set by each test before the request is made."""
|
| 95 |
-
mode = "success"
|
| 96 |
-
last_config = None
|
| 97 |
-
|
| 98 |
-
def __init__(self, config=None, actuator=None, approval_store=None,
|
| 99 |
-
on_verified_outcome=None):
|
| 100 |
-
self._on_verified_outcome = on_verified_outcome
|
| 101 |
-
type(self).last_config = config
|
| 102 |
-
|
| 103 |
-
async def execute(self, intent, human_approved=False, admin_approved=False):
|
| 104 |
-
if self.mode == "success":
|
| 105 |
-
if self._on_verified_outcome:
|
| 106 |
-
self._on_verified_outcome(intent, True, {"observed": {"status": "running"}})
|
| 107 |
-
return {"status": "success", "verified": {"status": "running"}, "compensating_action": None}
|
| 108 |
-
if self.mode == "pending":
|
| 109 |
-
raise _FakePendingApprovalError(
|
| 110 |
-
"needs human approval", level="HumanInLoop", approval_required="human",
|
| 111 |
-
approval_id="appr_test123",
|
| 112 |
-
)
|
| 113 |
-
if self.mode == "execution_error":
|
| 114 |
-
raise _FakeExecutionError("ladder denied this intent")
|
| 115 |
-
if self.mode == "safety_error":
|
| 116 |
-
raise _FakeSafetyError("blast radius exceeded")
|
| 117 |
-
raise RuntimeError(f"unhandled test mode: {self.mode}")
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
@pytest.fixture
|
| 121 |
-
def enterprise_execution_enabled(monkeypatch):
|
| 122 |
-
monkeypatch.setattr(routes_governance, "ENTERPRISE_EXECUTOR_AVAILABLE", True)
|
| 123 |
-
monkeypatch.setattr(routes_governance, "ARF_ENABLE_EXECUTION", True)
|
| 124 |
-
monkeypatch.setattr(routes_governance, "EnterpriseExecutor", _FakeExecutor)
|
| 125 |
-
monkeypatch.setattr(routes_governance, "EnterpriseConfig", _FakeConfig)
|
| 126 |
-
monkeypatch.setattr(routes_governance, "FakeCloudActuator", lambda: None)
|
| 127 |
-
monkeypatch.setattr(routes_governance, "PendingApprovalError", _FakePendingApprovalError)
|
| 128 |
-
monkeypatch.setattr(routes_governance, "EnterpriseExecutionError", _FakeExecutionError)
|
| 129 |
-
monkeypatch.setattr(routes_governance, "EnterpriseSafetyError", _FakeSafetyError)
|
| 130 |
-
_FakeExecutor.mode = "success"
|
| 131 |
-
yield
|
| 132 |
-
_FakeExecutor.mode = "success"
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
def test_execute_returns_501_when_enterprise_package_not_available(client):
|
| 136 |
-
resp = client.post(
|
| 137 |
-
"/api/v1/intents/does-not-matter/execute",
|
| 138 |
-
json={"healing_intent": {}},
|
| 139 |
-
)
|
| 140 |
-
assert resp.status_code == 501
|
| 141 |
-
assert "not installed" in resp.json()["detail"]
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
def test_execute_returns_501_when_not_enabled(client, monkeypatch):
|
| 145 |
-
monkeypatch.setattr(routes_governance, "ENTERPRISE_EXECUTOR_AVAILABLE", True)
|
| 146 |
-
monkeypatch.setattr(routes_governance, "ARF_ENABLE_EXECUTION", False)
|
| 147 |
-
resp = client.post(
|
| 148 |
-
"/api/v1/intents/does-not-matter/execute",
|
| 149 |
-
json={"healing_intent": {}},
|
| 150 |
-
)
|
| 151 |
-
assert resp.status_code == 501
|
| 152 |
-
assert "not enabled" in resp.json()["detail"]
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
def test_execute_returns_404_for_unknown_intent(client, enterprise_execution_enabled):
|
| 156 |
-
resp = client.post(
|
| 157 |
-
"/api/v1/intents/does-not-exist-at-all/execute",
|
| 158 |
-
json={"healing_intent": {}},
|
| 159 |
-
)
|
| 160 |
-
assert resp.status_code == 404
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
def test_execute_success_path(client, enterprise_execution_enabled):
|
| 164 |
-
deterministic_id, healing_intent = _evaluate_intent(client)
|
| 165 |
-
resp = client.post(
|
| 166 |
-
f"/api/v1/intents/{deterministic_id}/execute",
|
| 167 |
-
json={"healing_intent": healing_intent, "human_approved": True},
|
| 168 |
-
)
|
| 169 |
-
assert resp.status_code == 200, resp.text
|
| 170 |
-
assert resp.json()["status"] == "success"
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
def test_trusted_signing_keys_reach_the_executor_split_not_raw(
|
| 174 |
-
client, enterprise_execution_enabled, monkeypatch
|
| 175 |
-
):
|
| 176 |
-
"""ARF_TRUSTED_SIGNING_KEYS holds N comma-separated fingerprints.
|
| 177 |
-
Passing the raw string through as a one-element list would register the
|
| 178 |
-
literal "abc,def" as a single key -- trusting neither real one -- and
|
| 179 |
-
would look identical from the outside, since both spellings produce a
|
| 180 |
-
non-empty list and a 200 here."""
|
| 181 |
-
monkeypatch.setenv("ARF_TRUSTED_SIGNING_KEYS", " abc123 , def456 ,, ")
|
| 182 |
-
deterministic_id, healing_intent = _evaluate_intent(client)
|
| 183 |
-
resp = client.post(
|
| 184 |
-
f"/api/v1/intents/{deterministic_id}/execute",
|
| 185 |
-
json={"healing_intent": healing_intent, "human_approved": True},
|
| 186 |
-
)
|
| 187 |
-
assert resp.status_code == 200, resp.text
|
| 188 |
-
assert _FakeConfig.last_trusted_signing_keys == ["abc123", "def456"]
|
| 189 |
-
assert _FakeExecutor.last_config is not None
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
def test_unset_trusted_signing_keys_trusts_nothing_rather_than_the_empty_string(
|
| 193 |
-
client, enterprise_execution_enabled, monkeypatch
|
| 194 |
-
):
|
| 195 |
-
"""Fail closed. `"".split(",")` is `[""]`, so the naive parse would
|
| 196 |
-
register the empty string as a trusted fingerprint -- trusting a key
|
| 197 |
-
nobody holds is harmless, but it makes "trusts nothing" and
|
| 198 |
-
"misconfigured" indistinguishable in the logs."""
|
| 199 |
-
monkeypatch.delenv("ARF_TRUSTED_SIGNING_KEYS", raising=False)
|
| 200 |
-
deterministic_id, healing_intent = _evaluate_intent(client)
|
| 201 |
-
resp = client.post(
|
| 202 |
-
f"/api/v1/intents/{deterministic_id}/execute",
|
| 203 |
-
json={"healing_intent": healing_intent, "human_approved": True},
|
| 204 |
-
)
|
| 205 |
-
assert resp.status_code == 200, resp.text
|
| 206 |
-
assert _FakeConfig.last_trusted_signing_keys == []
|
| 207 |
-
|
| 208 |
-
|
| 209 |
-
def test_execute_pending_approval_returns_202_with_approval_id(client, enterprise_execution_enabled):
|
| 210 |
-
deterministic_id, healing_intent = _evaluate_intent(client)
|
| 211 |
-
_FakeExecutor.mode = "pending"
|
| 212 |
-
resp = client.post(
|
| 213 |
-
f"/api/v1/intents/{deterministic_id}/execute",
|
| 214 |
-
json={"healing_intent": healing_intent},
|
| 215 |
-
)
|
| 216 |
-
assert resp.status_code == 202
|
| 217 |
-
body = resp.json()
|
| 218 |
-
assert body["approval_id"] == "appr_test123"
|
| 219 |
-
assert body["level"] == "HumanInLoop"
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
def test_execute_execution_error_returns_422(client, enterprise_execution_enabled):
|
| 223 |
-
deterministic_id, healing_intent = _evaluate_intent(client)
|
| 224 |
-
_FakeExecutor.mode = "execution_error"
|
| 225 |
-
resp = client.post(
|
| 226 |
-
f"/api/v1/intents/{deterministic_id}/execute",
|
| 227 |
-
json={"healing_intent": healing_intent, "human_approved": True},
|
| 228 |
-
)
|
| 229 |
-
assert resp.status_code == 422
|
| 230 |
-
assert "ladder denied" in resp.json()["detail"]
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
def test_execute_safety_error_returns_422(client, enterprise_execution_enabled):
|
| 234 |
-
deterministic_id, healing_intent = _evaluate_intent(client)
|
| 235 |
-
_FakeExecutor.mode = "safety_error"
|
| 236 |
-
resp = client.post(
|
| 237 |
-
f"/api/v1/intents/{deterministic_id}/execute",
|
| 238 |
-
json={"healing_intent": healing_intent, "human_approved": True},
|
| 239 |
-
)
|
| 240 |
-
assert resp.status_code == 422
|
| 241 |
-
assert "blast radius" in resp.json()["detail"]
|
| 242 |
-
|
| 243 |
-
|
| 244 |
-
def test_execute_belongs_to_different_tenant_returns_404(client, enterprise_execution_enabled, db_session):
|
| 245 |
-
"""A deterministic_id that exists but under a different tenant must not
|
| 246 |
-
be executable by this caller -- same tenant-scoping guarantee
|
| 247 |
-
record_outcome already provides for /intents/outcome."""
|
| 248 |
-
other_tenant = "other-tenant"
|
| 249 |
-
if not db_session.query(TenantDB).filter_by(id=other_tenant).first():
|
| 250 |
-
db_session.add(TenantDB(id=other_tenant, name="Other Tenant"))
|
| 251 |
-
db_session.commit()
|
| 252 |
-
|
| 253 |
-
from app.database.models_intents import IntentDB
|
| 254 |
-
import datetime
|
| 255 |
-
db_session.add(IntentDB(
|
| 256 |
-
deterministic_id="belongs-to-other-tenant",
|
| 257 |
-
tenant_id=other_tenant,
|
| 258 |
-
intent_type="provision_resource",
|
| 259 |
-
payload={},
|
| 260 |
-
oss_payload={},
|
| 261 |
-
environment="prod",
|
| 262 |
-
evaluated_at=datetime.datetime.utcnow(),
|
| 263 |
-
risk_score="0.1",
|
| 264 |
-
))
|
| 265 |
-
db_session.commit()
|
| 266 |
-
|
| 267 |
-
resp = client.post(
|
| 268 |
-
"/api/v1/intents/belongs-to-other-tenant/execute",
|
| 269 |
-
json={"healing_intent": {}},
|
| 270 |
-
)
|
| 271 |
-
assert resp.status_code == 404
|
| 272 |
-
|
| 273 |
-
|
| 274 |
-
# ---------------------------------------------------------------------------
|
| 275 |
-
# Admin resolve/list endpoints
|
| 276 |
-
# ---------------------------------------------------------------------------
|
| 277 |
-
|
| 278 |
-
TEST_ADMIN_KEY = "test-admin-key-for-governance-execute-tests"
|
| 279 |
-
|
| 280 |
-
|
| 281 |
-
class _FakeApprovalRecord:
|
| 282 |
-
def __init__(self, id, decision_id, intent_id, level, approval_required, requested_at):
|
| 283 |
-
self.id = id
|
| 284 |
-
self.decision_id = decision_id
|
| 285 |
-
self.intent_id = intent_id
|
| 286 |
-
self.level = level
|
| 287 |
-
self.approval_required = approval_required
|
| 288 |
-
self.requested_at = requested_at
|
| 289 |
-
|
| 290 |
-
|
| 291 |
-
class _FakeApprovalStore:
|
| 292 |
-
def __init__(self):
|
| 293 |
-
import datetime
|
| 294 |
-
self._pending = {
|
| 295 |
-
"appr_1": _FakeApprovalRecord(
|
| 296 |
-
"appr_1", "dec_1", "intent-1", "HumanInLoop", "human", datetime.datetime.utcnow()
|
| 297 |
-
)
|
| 298 |
-
}
|
| 299 |
-
self.resolved = []
|
| 300 |
-
|
| 301 |
-
def list_pending(self, limit=100, offset=0):
|
| 302 |
-
return list(self._pending.values())[:limit]
|
| 303 |
-
|
| 304 |
-
def resolve(self, approval_id, approved, resolved_by, note=None):
|
| 305 |
-
if approval_id not in self._pending:
|
| 306 |
-
return False
|
| 307 |
-
del self._pending[approval_id]
|
| 308 |
-
self.resolved.append((approval_id, approved, resolved_by, note))
|
| 309 |
-
return True
|
| 310 |
-
|
| 311 |
-
|
| 312 |
-
@pytest.fixture
|
| 313 |
-
def admin_with_approval_store(monkeypatch, client):
|
| 314 |
-
monkeypatch.setattr(routes_admin, "ADMIN_API_KEY", TEST_ADMIN_KEY)
|
| 315 |
-
fake_store = _FakeApprovalStore()
|
| 316 |
-
client.app.state.approval_store = fake_store
|
| 317 |
-
yield fake_store
|
| 318 |
-
client.app.state.approval_store = None
|
| 319 |
-
|
| 320 |
-
|
| 321 |
-
def test_list_pending_executions_returns_501_without_store(client, monkeypatch):
|
| 322 |
-
monkeypatch.setattr(routes_admin, "ADMIN_API_KEY", TEST_ADMIN_KEY)
|
| 323 |
-
client.app.state.approval_store = None
|
| 324 |
-
resp = client.get("/api/v1/admin/executions/pending", params={"admin_key": TEST_ADMIN_KEY})
|
| 325 |
-
assert resp.status_code == 501
|
| 326 |
-
|
| 327 |
-
|
| 328 |
-
def test_list_pending_executions(client, admin_with_approval_store):
|
| 329 |
-
resp = client.get("/api/v1/admin/executions/pending", params={"admin_key": TEST_ADMIN_KEY})
|
| 330 |
-
assert resp.status_code == 200
|
| 331 |
-
body = resp.json()
|
| 332 |
-
assert body["total"] == 1
|
| 333 |
-
assert body["pending"][0]["approval_id"] == "appr_1"
|
| 334 |
-
|
| 335 |
-
|
| 336 |
-
def test_resolve_execution_approval(client, admin_with_approval_store):
|
| 337 |
-
resp = client.post(
|
| 338 |
-
"/api/v1/admin/executions/appr_1/resolve",
|
| 339 |
-
params={"admin_key": TEST_ADMIN_KEY},
|
| 340 |
-
json={"approved": True, "note": "looks fine"},
|
| 341 |
-
)
|
| 342 |
-
assert resp.status_code == 200
|
| 343 |
-
assert admin_with_approval_store.resolved == [("appr_1", True, "admin", "looks fine")]
|
| 344 |
-
|
| 345 |
-
|
| 346 |
-
def test_resolve_unknown_approval_returns_404(client, admin_with_approval_store):
|
| 347 |
-
resp = client.post(
|
| 348 |
-
"/api/v1/admin/executions/does-not-exist/resolve",
|
| 349 |
-
params={"admin_key": TEST_ADMIN_KEY},
|
| 350 |
-
json={"approved": True},
|
| 351 |
-
)
|
| 352 |
-
assert resp.status_code == 404
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tests/test_routes_onchain.py
DELETED
|
@@ -1,88 +0,0 @@
|
|
| 1 |
-
"""Tests for POST/GET /api/v1/onchain/rationale.
|
| 2 |
-
|
| 3 |
-
`verify_internal_key` and `get_db` are already overridden in conftest.py the
|
| 4 |
-
same way every other route test relies on -- this exercises the endpoint's
|
| 5 |
-
own logic (idempotent-on-hash write, conflict on a differing text, 404 on an
|
| 6 |
-
unknown hash, hash format validation), not authentication or the database
|
| 7 |
-
layer itself.
|
| 8 |
-
"""
|
| 9 |
-
|
| 10 |
-
VALID_HASH = "0x" + "ab" * 32
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
def _post(
|
| 14 |
-
client,
|
| 15 |
-
rationale_hash=VALID_HASH,
|
| 16 |
-
rationale="the volume's final backup was skipped",
|
| 17 |
-
**extra,
|
| 18 |
-
):
|
| 19 |
-
payload = {"rationale_hash": rationale_hash, "rationale": rationale, **extra}
|
| 20 |
-
return client.post("/api/v1/onchain/rationale", json=payload)
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
class TestPersistRationale:
|
| 24 |
-
def test_a_new_hash_is_recorded(self, client):
|
| 25 |
-
resp = _post(client)
|
| 26 |
-
assert resp.status_code == 201, resp.text
|
| 27 |
-
assert resp.json() == {"status": "recorded", "rationale_hash": VALID_HASH}
|
| 28 |
-
|
| 29 |
-
def test_posting_the_same_hash_and_text_again_is_a_no_op(self, client):
|
| 30 |
-
# A hash of its own -- the `client` fixture is session-scoped against
|
| 31 |
-
# a real, persistent Postgres with no truncation between tests, so
|
| 32 |
-
# reusing VALID_HASH here would collide with whichever other test in
|
| 33 |
-
# this file claims it first.
|
| 34 |
-
hash_ = "0x" + "22" * 32
|
| 35 |
-
first = _post(client, rationale_hash=hash_)
|
| 36 |
-
assert first.status_code == 201
|
| 37 |
-
second = _post(client, rationale_hash=hash_)
|
| 38 |
-
assert second.status_code == 200
|
| 39 |
-
assert second.json()["status"] == "already_recorded"
|
| 40 |
-
|
| 41 |
-
def test_the_same_hash_with_different_text_is_refused(self, client):
|
| 42 |
-
hash_ = "0x" + "33" * 32
|
| 43 |
-
first = _post(client, rationale_hash=hash_)
|
| 44 |
-
assert first.status_code == 201
|
| 45 |
-
second = _post(client, rationale_hash=hash_, rationale="a different reason entirely")
|
| 46 |
-
assert second.status_code == 409
|
| 47 |
-
|
| 48 |
-
def test_a_malformed_hash_is_rejected(self, client):
|
| 49 |
-
resp = _post(client, rationale_hash="not-a-hash")
|
| 50 |
-
assert resp.status_code == 422
|
| 51 |
-
|
| 52 |
-
def test_a_short_hash_is_rejected(self, client):
|
| 53 |
-
resp = _post(client, rationale_hash="0x" + "ab" * 16)
|
| 54 |
-
assert resp.status_code == 422
|
| 55 |
-
|
| 56 |
-
def test_empty_rationale_is_rejected(self, client):
|
| 57 |
-
resp = _post(client, rationale=" ")
|
| 58 |
-
assert resp.status_code == 422
|
| 59 |
-
|
| 60 |
-
def test_agent_and_evaluator_addresses_are_stored(self, client):
|
| 61 |
-
resp = _post(
|
| 62 |
-
client,
|
| 63 |
-
rationale_hash="0x" + "cd" * 32,
|
| 64 |
-
agent_address="0x00000000000000000000000000000000000000A1",
|
| 65 |
-
evaluator_address="0x00000000000000000000000000000000000000B2",
|
| 66 |
-
)
|
| 67 |
-
assert resp.status_code == 201
|
| 68 |
-
fetched = client.get(f"/api/v1/onchain/rationale/{'0x' + 'cd' * 32}")
|
| 69 |
-
assert fetched.status_code == 200
|
| 70 |
-
body = fetched.json()
|
| 71 |
-
assert body["agent_address"] == "0x00000000000000000000000000000000000000A1"
|
| 72 |
-
assert body["evaluator_address"] == "0x00000000000000000000000000000000000000B2"
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
class TestGetRationale:
|
| 76 |
-
def test_fetching_a_recorded_hash_returns_its_text(self, client):
|
| 77 |
-
_post(client)
|
| 78 |
-
resp = client.get(f"/api/v1/onchain/rationale/{VALID_HASH}")
|
| 79 |
-
assert resp.status_code == 200
|
| 80 |
-
assert resp.json()["rationale"] == "the volume's final backup was skipped"
|
| 81 |
-
|
| 82 |
-
def test_fetching_an_unknown_hash_is_404(self, client):
|
| 83 |
-
resp = client.get(f"/api/v1/onchain/rationale/{'0x' + 'ef' * 32}")
|
| 84 |
-
assert resp.status_code == 404
|
| 85 |
-
|
| 86 |
-
def test_fetching_a_malformed_hash_is_422_not_404(self, client):
|
| 87 |
-
resp = client.get("/api/v1/onchain/rationale/not-a-hash")
|
| 88 |
-
assert resp.status_code == 422
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tests/test_routes_pricing.py
DELETED
|
@@ -1,47 +0,0 @@
|
|
| 1 |
-
from fastapi.testclient import TestClient
|
| 2 |
-
from app.main import app
|
| 3 |
-
|
| 4 |
-
client = TestClient(app)
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
def test_run_pricing_is_temporarily_disabled():
|
| 8 |
-
"""/pricing/run was disabled after AUDIT_arf-bayesian-pricing-calculator.md's
|
| 9 |
-
Critical finding: it persisted a fabricated random.random() outcome into a
|
| 10 |
-
calibration buffer with no customer_id scoping, so every customer's price
|
| 11 |
-
was shaped by every other customer's fabricated outcomes. This asserts the
|
| 12 |
-
disabled state itself, not the old (buggy) behavior -- update this test
|
| 13 |
-
when the endpoint is actually fixed and re-enabled, not before."""
|
| 14 |
-
response = client.post(
|
| 15 |
-
"/api/v1/pricing/run",
|
| 16 |
-
json={"input": {}, "customer_id": "test-customer", "runs": 1},
|
| 17 |
-
)
|
| 18 |
-
assert response.status_code == 503
|
| 19 |
-
assert "temporarily disabled" in response.json()["detail"].lower()
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
def test_run_pricing_still_requires_auth():
|
| 23 |
-
"""Disabling the endpoint must not also disable its auth -- conftest.py's
|
| 24 |
-
mock_enforce_quota makes every request "authenticated" for this test
|
| 25 |
-
client, so this only confirms the Depends(enforce_quota) dependency is
|
| 26 |
-
still declared and still runs before the handler body (i.e. it wasn't
|
| 27 |
-
accidentally dropped along with the rest of the function body); it does
|
| 28 |
-
not exercise the real 401/403 paths, which are covered by test_deps.py
|
| 29 |
-
and usage_tracker's own tests."""
|
| 30 |
-
response = client.post(
|
| 31 |
-
"/api/v1/pricing/run",
|
| 32 |
-
json={"input": {}, "customer_id": "test-customer", "runs": 1},
|
| 33 |
-
)
|
| 34 |
-
# Reaching the 503 (not erroring before it) proves enforce_quota resolved
|
| 35 |
-
# successfully -- if the dependency were missing or broken, this would be
|
| 36 |
-
# a 401/422/500 instead.
|
| 37 |
-
assert response.status_code == 503
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
def test_estimate_pricing_route_still_registered():
|
| 41 |
-
"""/pricing/estimate was not touched by the disable -- confirm it's still
|
| 42 |
-
a distinct, reachable route (a malformed request should 400, not 503/404),
|
| 43 |
-
so a caller following the "use /pricing/estimate instead" guidance in the
|
| 44 |
-
503 detail message actually has somewhere to go."""
|
| 45 |
-
response = client.post("/api/v1/pricing/estimate", json={"input": {}})
|
| 46 |
-
assert response.status_code != 503
|
| 47 |
-
assert response.status_code != 404
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tests/test_tracker_binding.py
DELETED
|
@@ -1,95 +0,0 @@
|
|
| 1 |
-
"""Guard against re-introducing the stale `tracker` from-import.
|
| 2 |
-
|
| 3 |
-
`app.core.usage_tracker.tracker` starts as None and is rebound by
|
| 4 |
-
`init_tracker()` during the lifespan. `from app.core.usage_tracker import
|
| 5 |
-
tracker` copies the *binding* -- None -- at import time, and the later
|
| 6 |
-
rebinding never reaches the importer. Five modules did this: main.py
|
| 7 |
-
crashed the Render deploy with "'NoneType' object has no attribute
|
| 8 |
-
'warm_up'", routes_admin's 20 unguarded uses would have 500'd, and
|
| 9 |
-
routes_incidents/payments/users each had a `if tracker` guard that could
|
| 10 |
-
only ever take the None branch -- so usage went unmetered and signup and
|
| 11 |
-
checkout were permanently disabled, silently.
|
| 12 |
-
|
| 13 |
-
This is a source-level check on purpose. Reproducing the bug at runtime
|
| 14 |
-
needs a real UsageTracker (Postgres, pepper) and would only cover the
|
| 15 |
-
modules the test happened to import; parsing every module catches the
|
| 16 |
-
next one too, and costs nothing.
|
| 17 |
-
"""
|
| 18 |
-
|
| 19 |
-
import ast
|
| 20 |
-
import re
|
| 21 |
-
import pathlib
|
| 22 |
-
|
| 23 |
-
import pytest
|
| 24 |
-
|
| 25 |
-
APP = pathlib.Path(__file__).resolve().parent.parent / "app"
|
| 26 |
-
|
| 27 |
-
# Names safe to import directly: functions defined in usage_tracker.py
|
| 28 |
-
# resolve the module global at call time, so they always see the live
|
| 29 |
-
# instance. Only the mutable module-level object itself is unsafe.
|
| 30 |
-
UNSAFE_NAMES = {"tracker"}
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
def _source_files():
|
| 34 |
-
return sorted(p for p in APP.rglob("*.py"))
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
TESTS = pathlib.Path(__file__).resolve().parent
|
| 38 |
-
|
| 39 |
-
# `patch("app.api.routes_admin.tracker")` and
|
| 40 |
-
# `monkeypatch.setattr(routes_payments, "tracker", ...)` both target an
|
| 41 |
-
# attribute that only exists while the broken from-import does. They passed
|
| 42 |
-
# for as long as the bug was there and broke the moment it was fixed --
|
| 43 |
-
# and, worse, while the bug was there they were patching a name the route
|
| 44 |
-
# code was already reading as None, so they proved nothing.
|
| 45 |
-
#
|
| 46 |
-
# conftest.py has always done the right thing (`app.core.usage_tracker
|
| 47 |
-
# .tracker = MockTracker()`), which is why this is the correct target: one
|
| 48 |
-
# patch on the defining module reaches every consumer.
|
| 49 |
-
_BAD_PATCH_TARGETS = re.compile(
|
| 50 |
-
r"""["']app\.api\.routes_\w+\.tracker["']"""
|
| 51 |
-
r"""|setattr\(\s*routes_\w+\s*,\s*["']tracker["']"""
|
| 52 |
-
)
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
def _test_files():
|
| 56 |
-
"""Every test module but this one.
|
| 57 |
-
|
| 58 |
-
The check is textual, so this file trips it on the comment above that
|
| 59 |
-
quotes the bad forms. Excluding self rather than contorting the regex
|
| 60 |
-
keeps the pattern readable and the examples literal -- and the cost is
|
| 61 |
-
only that this file cannot police itself, which it has no reason to.
|
| 62 |
-
"""
|
| 63 |
-
here = pathlib.Path(__file__).resolve()
|
| 64 |
-
return sorted(p for p in TESTS.glob("test_*.py") if p.resolve() != here)
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
@pytest.mark.parametrize("path", _test_files(), ids=lambda p: p.name)
|
| 68 |
-
def test_no_test_patches_tracker_on_a_route_module(path):
|
| 69 |
-
src = path.read_text(encoding="utf-8")
|
| 70 |
-
for lineno, line in enumerate(src.splitlines(), start=1):
|
| 71 |
-
assert not _BAD_PATCH_TARGETS.search(line), (
|
| 72 |
-
f"{path.name}:{lineno} patches `tracker` on a route module. Route "
|
| 73 |
-
"modules reference `usage_tracker.tracker` and hold no binding of "
|
| 74 |
-
'their own -- patch "app.core.usage_tracker.tracker" instead, '
|
| 75 |
-
"which is what conftest.py already does."
|
| 76 |
-
)
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
@pytest.mark.parametrize("path", _source_files(), ids=lambda p: p.name)
|
| 80 |
-
def test_tracker_is_never_imported_by_name(path):
|
| 81 |
-
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
|
| 82 |
-
for node in ast.walk(tree):
|
| 83 |
-
if not isinstance(node, ast.ImportFrom):
|
| 84 |
-
continue
|
| 85 |
-
if node.module != "app.core.usage_tracker":
|
| 86 |
-
continue
|
| 87 |
-
offenders = sorted(
|
| 88 |
-
{a.name for a in node.names} & UNSAFE_NAMES
|
| 89 |
-
)
|
| 90 |
-
assert not offenders, (
|
| 91 |
-
f"{path.name}:{node.lineno} imports {offenders} by name from "
|
| 92 |
-
"app.core.usage_tracker. That binding is None at import time and "
|
| 93 |
-
"init_tracker() will not update it. Use `from app.core import "
|
| 94 |
-
"usage_tracker` and reference `usage_tracker.tracker` instead."
|
| 95 |
-
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tests/test_usage_tracker.py
CHANGED
|
@@ -1,8 +1,3 @@
|
|
| 1 |
-
import json
|
| 2 |
-
import os
|
| 3 |
-
from datetime import datetime
|
| 4 |
-
|
| 5 |
-
import psycopg2
|
| 6 |
import pytest
|
| 7 |
import tempfile
|
| 8 |
import time
|
|
@@ -83,126 +78,3 @@ def test_get_audit_logs(tracker):
|
|
| 83 |
logs = tracker.get_audit_logs("test_key", limit=10)
|
| 84 |
assert len(logs) == 1
|
| 85 |
assert logs[0]["endpoint"] == "/test"
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
def _pg_monthly_count(api_key: str, month: str) -> int:
|
| 89 |
-
"""Direct Postgres read, bypassing UsageTracker entirely -- this is
|
| 90 |
-
exactly the query arf-gateway's Go code runs against the same table."""
|
| 91 |
-
conn = psycopg2.connect(os.environ["DATABASE_URL"])
|
| 92 |
-
try:
|
| 93 |
-
with conn.cursor() as cur:
|
| 94 |
-
cur.execute(
|
| 95 |
-
"SELECT COALESCE(count, 0) FROM monthly_counts WHERE api_key = %s AND year_month = %s",
|
| 96 |
-
(api_key, month),
|
| 97 |
-
)
|
| 98 |
-
row = cur.fetchone()
|
| 99 |
-
return row[0] if row else 0
|
| 100 |
-
finally:
|
| 101 |
-
conn.close()
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
def test_increment_usage_sync_mirrors_to_postgres_monthly_counts(tracker):
|
| 105 |
-
"""Regresses arf-gateway-001: arf-gateway's quota check reads
|
| 106 |
-
monthly_counts from Postgres, not from this service's local SQLite
|
| 107 |
-
file. Every successfully-counted call must be visible there."""
|
| 108 |
-
tracker.get_or_create_api_key("pg-mirror-key", tenant_id="test")
|
| 109 |
-
month = tracker._get_month_key()
|
| 110 |
-
|
| 111 |
-
# No truncate fixture exists for this Postgres-resident table (CI's
|
| 112 |
-
# Postgres service is ephemeral per run, but a local repeat run against
|
| 113 |
-
# a persistent database could have leftover rows) -- assert the delta,
|
| 114 |
-
# not an absolute count.
|
| 115 |
-
baseline = _pg_monthly_count("pg-mirror-key", month)
|
| 116 |
-
|
| 117 |
-
record = UsageRecord(
|
| 118 |
-
api_key="pg-mirror-key", tier=Tier.FREE, timestamp=time.time(), endpoint="/test",
|
| 119 |
-
)
|
| 120 |
-
tracker.increment_usage_sync(record)
|
| 121 |
-
assert _pg_monthly_count("pg-mirror-key", month) == baseline + 1
|
| 122 |
-
|
| 123 |
-
tracker.increment_usage_sync(record)
|
| 124 |
-
assert _pg_monthly_count("pg-mirror-key", month) == baseline + 2
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
def test_increment_usage_sync_succeeds_even_if_postgres_mirror_fails(tracker, monkeypatch):
|
| 128 |
-
"""The local quota decision (SQLite/Redis) must not fail just because
|
| 129 |
-
the best-effort mirror write to Postgres did -- see
|
| 130 |
-
_record_pg_monthly_count's docstring. A degraded mirror write should
|
| 131 |
-
not turn into a degraded evaluate/healing endpoint for the caller."""
|
| 132 |
-
def _boom(*args, **kwargs):
|
| 133 |
-
raise psycopg2.OperationalError("simulated Postgres outage")
|
| 134 |
-
|
| 135 |
-
# get_or_create_api_key also goes through _get_pg_conn (it persists to
|
| 136 |
-
# the real api_keys table), so it must run before the patch below -- the
|
| 137 |
-
# outage this test simulates is specific to the monthly_counts mirror
|
| 138 |
-
# write, not to Postgres as a whole.
|
| 139 |
-
tracker.get_or_create_api_key("mirror-fail-key", tenant_id="test")
|
| 140 |
-
|
| 141 |
-
# Patch what _record_pg_monthly_count calls internally, not the method
|
| 142 |
-
# itself -- replacing the whole method would bypass its own try/except
|
| 143 |
-
# and prove nothing about that error-handling actually working.
|
| 144 |
-
monkeypatch.setattr(tracker, "_get_pg_conn", _boom)
|
| 145 |
-
|
| 146 |
-
record = UsageRecord(
|
| 147 |
-
api_key="mirror-fail-key", tier=Tier.FREE, timestamp=time.time(), endpoint="/test",
|
| 148 |
-
)
|
| 149 |
-
|
| 150 |
-
# Must not raise, and the local quota decision must still succeed.
|
| 151 |
-
result = tracker.increment_usage_sync(record)
|
| 152 |
-
assert result is True
|
| 153 |
-
assert tracker.get_remaining_quota("mirror-fail-key", Tier.FREE) == 999
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
def test_insert_audit_log_writes_response_row(tracker):
|
| 157 |
-
"""routes_governance.py schedules current_tracker._insert_audit_log as
|
| 158 |
-
a background task (background_tasks.add_task) to record the response
|
| 159 |
-
body once it's known, at app/api/routes_governance.py:407 and :738 --
|
| 160 |
-
always with tier=None, since quota was already consumed by an earlier
|
| 161 |
-
consume_quota_and_log call for the same request. The real UsageTracker
|
| 162 |
-
had no such method (only tests/conftest.py's MockTracker did), so every
|
| 163 |
-
real call raised AttributeError inside the background task (arf-api-002)."""
|
| 164 |
-
record = UsageRecord(
|
| 165 |
-
api_key="audit-log-key",
|
| 166 |
-
tier=None,
|
| 167 |
-
timestamp=time.time(),
|
| 168 |
-
endpoint="/api/v1/intents/evaluate/response",
|
| 169 |
-
request_body=None,
|
| 170 |
-
response={"recommended_action": "approve"},
|
| 171 |
-
processing_ms=12.5,
|
| 172 |
-
)
|
| 173 |
-
|
| 174 |
-
tracker._insert_audit_log(record)
|
| 175 |
-
|
| 176 |
-
logs = tracker.get_audit_logs("audit-log-key", limit=10)
|
| 177 |
-
assert len(logs) == 1
|
| 178 |
-
assert logs[0]["endpoint"] == "/api/v1/intents/evaluate/response"
|
| 179 |
-
assert logs[0]["tier"] == "unknown"
|
| 180 |
-
assert json.loads(logs[0]["response"]) == {"recommended_action": "approve"}
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
def test_consume_quota_and_log_handles_non_json_native_request_body(tracker):
|
| 184 |
-
"""request_body/response are whatever a Pydantic model's plain
|
| 185 |
-
.model_dump() returns, e.g. ReliabilityEvent.timestamp in
|
| 186 |
-
app/api/routes_governance.py's HealingDecisionRequest -- a raw
|
| 187 |
-
datetime, not the ISO string model_dump(mode="json") would produce.
|
| 188 |
-
json.dumps has no default encoder for datetime, so any real request
|
| 189 |
-
carrying one raised TypeError here, outside any try/except in the
|
| 190 |
-
/healing/evaluate handler, on every call (surfaced while verifying the
|
| 191 |
-
tier=None fix for that same endpoint: fixing tier alone still crashed,
|
| 192 |
-
one layer deeper, on this). default=str makes the insert tolerant of
|
| 193 |
-
datetime and any other type json.dumps doesn't natively handle."""
|
| 194 |
-
tracker.get_or_create_api_key("datetime-body-key", tenant_id="test")
|
| 195 |
-
record = UsageRecord(
|
| 196 |
-
api_key="datetime-body-key",
|
| 197 |
-
tier=Tier.FREE,
|
| 198 |
-
timestamp=time.time(),
|
| 199 |
-
endpoint="/api/v1/healing/evaluate",
|
| 200 |
-
request_body={"event": {"component": "svc", "timestamp": datetime.now()}},
|
| 201 |
-
)
|
| 202 |
-
|
| 203 |
-
result = tracker.increment_usage_sync(record)
|
| 204 |
-
assert result is True
|
| 205 |
-
|
| 206 |
-
logs = tracker.get_audit_logs("datetime-body-key", limit=10)
|
| 207 |
-
assert len(logs) == 1
|
| 208 |
-
assert "timestamp" in json.loads(logs[0]["request_body"])["event"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
import pytest
|
| 2 |
import tempfile
|
| 3 |
import time
|
|
|
|
| 78 |
logs = tracker.get_audit_logs("test_key", limit=10)
|
| 79 |
assert len(logs) == 1
|
| 80 |
assert logs[0]["endpoint"] == "/test"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tests/test_webhooks.py
CHANGED
|
@@ -1,25 +1,17 @@
|
|
|
|
|
| 1 |
import pytest
|
| 2 |
from unittest.mock import patch
|
| 3 |
from fastapi.testclient import TestClient
|
| 4 |
from app.main import app
|
| 5 |
|
| 6 |
-
# Every Stripe call in this module is mocked -- nothing here reaches the
|
| 7 |
-
# network, and no real credentials are involved.
|
| 8 |
-
#
|
| 9 |
-
# This module used to skip entirely (pytest.skip(allow_module_level=True))
|
| 10 |
-
# whenever STRIPE_* env vars were unset, which is always in CI -- so none of
|
| 11 |
-
# it was ever exercised anywhere. webhooks.py reads its config into
|
| 12 |
-
# module-level globals at import time, so env vars set in conftest.py can't
|
| 13 |
-
# help (conftest's imports run before its os.environ assignments); the
|
| 14 |
-
# fixture below patches those globals directly instead, which works
|
| 15 |
-
# regardless of import order.
|
| 16 |
client = TestClient(app)
|
| 17 |
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
|
|
|
| 23 |
|
| 24 |
|
| 25 |
@pytest.fixture
|
|
@@ -29,9 +21,7 @@ def mock_stripe_webhook():
|
|
| 29 |
|
| 30 |
|
| 31 |
def test_webhook_missing_secret(monkeypatch):
|
| 32 |
-
|
| 33 |
-
# import time, so setenv here would be a no-op -- patch the global.
|
| 34 |
-
monkeypatch.setattr("app.api.webhooks.STRIPE_WEBHOOK_SECRET", "")
|
| 35 |
response = client.post(
|
| 36 |
"/webhooks/stripe",
|
| 37 |
json={},
|
|
@@ -41,7 +31,8 @@ def test_webhook_missing_secret(monkeypatch):
|
|
| 41 |
assert "Stripe not configured" in response.json()["detail"]
|
| 42 |
|
| 43 |
|
| 44 |
-
def test_webhook_invalid_payload(mock_stripe_webhook):
|
|
|
|
| 45 |
mock_stripe_webhook.side_effect = ValueError("Invalid payload")
|
| 46 |
response = client.post(
|
| 47 |
"/webhooks/stripe",
|
|
@@ -52,12 +43,9 @@ def test_webhook_invalid_payload(mock_stripe_webhook):
|
|
| 52 |
assert "Invalid payload" in response.json()["detail"]
|
| 53 |
|
| 54 |
|
| 55 |
-
def test_webhook_invalid_signature(mock_stripe_webhook):
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
# would propagate as a 500 instead of the 400 this asserts.
|
| 59 |
-
mock_stripe_webhook.side_effect = stripe.error.SignatureVerificationError(
|
| 60 |
-
"Invalid signature", "sig_header")
|
| 61 |
response = client.post(
|
| 62 |
"/webhooks/stripe",
|
| 63 |
json={},
|
|
@@ -67,92 +55,35 @@ def test_webhook_invalid_signature(mock_stripe_webhook):
|
|
| 67 |
assert "Invalid signature" in response.json()["detail"]
|
| 68 |
|
| 69 |
|
| 70 |
-
def test_webhook_checkout_completed():
|
|
|
|
|
|
|
| 71 |
with patch("stripe.Webhook.construct_event") as mock_construct, \
|
| 72 |
-
patch("app.
|
| 73 |
mock_construct.return_value = {
|
| 74 |
"type": "checkout.session.completed",
|
| 75 |
"data": {
|
| 76 |
"object": {
|
| 77 |
-
"client_reference_id": "
|
| 78 |
-
"payment_status": "paid",
|
| 79 |
"metadata": {
|
| 80 |
-
"
|
| 81 |
-
response = client.post(
|
| 82 |
-
"/webhooks/stripe",
|
| 83 |
-
json={},
|
| 84 |
-
headers={"stripe-signature": "test"}
|
| 85 |
-
)
|
| 86 |
-
assert response.status_code == 200
|
| 87 |
-
mock_update.assert_called_once_with("tenant_123", "pro")
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
def test_webhook_checkout_completed_not_yet_paid_does_not_upgrade():
|
| 91 |
-
"""checkout.session.completed can fire before payment actually clears
|
| 92 |
-
for delayed-notification payment methods -- must not grant Pro yet."""
|
| 93 |
-
with patch("stripe.Webhook.construct_event") as mock_construct, \
|
| 94 |
-
patch("app.api.webhooks.update_key_tier_by_tenant_id") as mock_update:
|
| 95 |
-
mock_construct.return_value = {
|
| 96 |
-
"type": "checkout.session.completed",
|
| 97 |
-
"data": {
|
| 98 |
-
"object": {
|
| 99 |
-
"client_reference_id": "tenant_123",
|
| 100 |
-
"payment_status": "unpaid",
|
| 101 |
-
"metadata": {"tenant_id": "tenant_123"}}}}
|
| 102 |
response = client.post(
|
| 103 |
"/webhooks/stripe",
|
| 104 |
json={},
|
| 105 |
headers={"stripe-signature": "test"}
|
| 106 |
)
|
| 107 |
assert response.status_code == 200
|
| 108 |
-
mock_update.
|
| 109 |
|
| 110 |
|
| 111 |
-
def test_webhook_subscription_deleted():
|
|
|
|
|
|
|
| 112 |
with patch("stripe.Webhook.construct_event") as mock_construct, \
|
| 113 |
-
patch("app.
|
| 114 |
mock_construct.return_value = {
|
| 115 |
"type": "customer.subscription.deleted",
|
| 116 |
-
"data": {"object": {"metadata": {"
|
| 117 |
-
}
|
| 118 |
-
response = client.post(
|
| 119 |
-
"/webhooks/stripe",
|
| 120 |
-
json={},
|
| 121 |
-
headers={"stripe-signature": "test"}
|
| 122 |
-
)
|
| 123 |
-
assert response.status_code == 200
|
| 124 |
-
mock_update.assert_called_once_with("tenant_123", "free")
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
def test_webhook_subscription_updated_canceled_downgrades():
|
| 128 |
-
"""customer.subscription.updated (not just .deleted) must also
|
| 129 |
-
downgrade -- a status transition to canceled/unpaid can arrive this
|
| 130 |
-
way, and .deleted is not the only terminal event Stripe sends."""
|
| 131 |
-
with patch("stripe.Webhook.construct_event") as mock_construct, \
|
| 132 |
-
patch("app.api.webhooks.update_key_tier_by_tenant_id") as mock_update:
|
| 133 |
-
mock_construct.return_value = {
|
| 134 |
-
"type": "customer.subscription.updated",
|
| 135 |
-
"data": {"object": {
|
| 136 |
-
"status": "canceled",
|
| 137 |
-
"metadata": {"tenant_id": "tenant_123"}}}
|
| 138 |
-
}
|
| 139 |
-
response = client.post(
|
| 140 |
-
"/webhooks/stripe",
|
| 141 |
-
json={},
|
| 142 |
-
headers={"stripe-signature": "test"}
|
| 143 |
-
)
|
| 144 |
-
assert response.status_code == 200
|
| 145 |
-
mock_update.assert_called_once_with("tenant_123", "free")
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
def test_webhook_subscription_updated_active_does_not_downgrade():
|
| 149 |
-
with patch("stripe.Webhook.construct_event") as mock_construct, \
|
| 150 |
-
patch("app.api.webhooks.update_key_tier_by_tenant_id") as mock_update:
|
| 151 |
-
mock_construct.return_value = {
|
| 152 |
-
"type": "customer.subscription.updated",
|
| 153 |
-
"data": {"object": {
|
| 154 |
-
"status": "active",
|
| 155 |
-
"metadata": {"tenant_id": "tenant_123"}}}
|
| 156 |
}
|
| 157 |
response = client.post(
|
| 158 |
"/webhooks/stripe",
|
|
@@ -160,4 +91,4 @@ def test_webhook_subscription_updated_active_does_not_downgrade():
|
|
| 160 |
headers={"stripe-signature": "test"}
|
| 161 |
)
|
| 162 |
assert response.status_code == 200
|
| 163 |
-
mock_update.
|
|
|
|
| 1 |
+
import os
|
| 2 |
import pytest
|
| 3 |
from unittest.mock import patch
|
| 4 |
from fastapi.testclient import TestClient
|
| 5 |
from app.main import app
|
| 6 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7 |
client = TestClient(app)
|
| 8 |
|
| 9 |
+
# Skip all tests in this module if Stripe webhook secret is not set
|
| 10 |
+
STRIPE_WEBHOOK_SECRET = os.getenv("STRIPE_WEBHOOK_SECRET")
|
| 11 |
+
if not STRIPE_WEBHOOK_SECRET:
|
| 12 |
+
pytest.skip(
|
| 13 |
+
"Stripe webhook not configured – skipping webhook tests",
|
| 14 |
+
allow_module_level=True)
|
| 15 |
|
| 16 |
|
| 17 |
@pytest.fixture
|
|
|
|
| 21 |
|
| 22 |
|
| 23 |
def test_webhook_missing_secret(monkeypatch):
|
| 24 |
+
monkeypatch.setenv("STRIPE_WEBHOOK_SECRET", "")
|
|
|
|
|
|
|
| 25 |
response = client.post(
|
| 26 |
"/webhooks/stripe",
|
| 27 |
json={},
|
|
|
|
| 31 |
assert "Stripe not configured" in response.json()["detail"]
|
| 32 |
|
| 33 |
|
| 34 |
+
def test_webhook_invalid_payload(mock_stripe_webhook, monkeypatch):
|
| 35 |
+
monkeypatch.setenv("STRIPE_WEBHOOK_SECRET", "whsec_test")
|
| 36 |
mock_stripe_webhook.side_effect = ValueError("Invalid payload")
|
| 37 |
response = client.post(
|
| 38 |
"/webhooks/stripe",
|
|
|
|
| 43 |
assert "Invalid payload" in response.json()["detail"]
|
| 44 |
|
| 45 |
|
| 46 |
+
def test_webhook_invalid_signature(mock_stripe_webhook, monkeypatch):
|
| 47 |
+
monkeypatch.setenv("STRIPE_WEBHOOK_SECRET", "whsec_test")
|
| 48 |
+
mock_stripe_webhook.side_effect = Exception("Invalid signature")
|
|
|
|
|
|
|
|
|
|
| 49 |
response = client.post(
|
| 50 |
"/webhooks/stripe",
|
| 51 |
json={},
|
|
|
|
| 55 |
assert "Invalid signature" in response.json()["detail"]
|
| 56 |
|
| 57 |
|
| 58 |
+
def test_webhook_checkout_completed(monkeypatch):
|
| 59 |
+
monkeypatch.setenv("STRIPE_WEBHOOK_SECRET", "whsec_test")
|
| 60 |
+
monkeypatch.setenv("STRIPE_SECRET_KEY", "sk_test")
|
| 61 |
with patch("stripe.Webhook.construct_event") as mock_construct, \
|
| 62 |
+
patch("app.core.usage_tracker.update_key_tier") as mock_update:
|
| 63 |
mock_construct.return_value = {
|
| 64 |
"type": "checkout.session.completed",
|
| 65 |
"data": {
|
| 66 |
"object": {
|
| 67 |
+
"client_reference_id": "test_key",
|
|
|
|
| 68 |
"metadata": {
|
| 69 |
+
"api_key": "test_key"}}}}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 70 |
response = client.post(
|
| 71 |
"/webhooks/stripe",
|
| 72 |
json={},
|
| 73 |
headers={"stripe-signature": "test"}
|
| 74 |
)
|
| 75 |
assert response.status_code == 200
|
| 76 |
+
mock_update.assert_called_once_with("test_key", "pro")
|
| 77 |
|
| 78 |
|
| 79 |
+
def test_webhook_subscription_deleted(monkeypatch):
|
| 80 |
+
monkeypatch.setenv("STRIPE_WEBHOOK_SECRET", "whsec_test")
|
| 81 |
+
monkeypatch.setenv("STRIPE_SECRET_KEY", "sk_test")
|
| 82 |
with patch("stripe.Webhook.construct_event") as mock_construct, \
|
| 83 |
+
patch("app.core.usage_tracker.update_key_tier") as mock_update:
|
| 84 |
mock_construct.return_value = {
|
| 85 |
"type": "customer.subscription.deleted",
|
| 86 |
+
"data": {"object": {"metadata": {"api_key": "test_key"}}}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 87 |
}
|
| 88 |
response = client.post(
|
| 89 |
"/webhooks/stripe",
|
|
|
|
| 91 |
headers={"stripe-signature": "test"}
|
| 92 |
)
|
| 93 |
assert response.status_code == 200
|
| 94 |
+
mock_update.assert_called_once_with("test_key", "free")
|