Spaces:
Sleeping
Sleeping
GitHub Action commited on
Commit ·
b88d8cd
1
Parent(s): 6a5fc90
Sync from GitHub: 564e0e9e7b5ba7ec7b40c0e05479a97951f38ffd
Browse files- .gitattributes +2 -31
- DEPLOYMENT_GUIDE.md +419 -0
- Dockerfile +55 -0
- HUGGINGFACE_DEPLOYMENT.md +123 -0
- Models/fraud_pipeline_final.pkl +3 -0
- QUICK_START.md +92 -0
- README.md +306 -7
- dataset/test_dataset.csv.gz +3 -0
- entrypoint.sh +16 -0
- env.template +25 -0
- feature_engineering.py +166 -0
- inference.py +590 -0
- main.py +530 -0
- railway.json +13 -0
- render.yaml +26 -0
- requirements.txt +35 -0
- vercel.json +20 -0
.gitattributes
CHANGED
|
@@ -1,35 +1,6 @@
|
|
| 1 |
-
*.
|
| 2 |
-
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
-
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
-
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
-
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
-
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
-
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
-
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
-
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
-
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
-
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
-
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
-
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
-
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
-
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
-
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
-
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
-
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
-
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
-
*.
|
| 25 |
-
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
-
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
-
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
-
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
-
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
-
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
-
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
-
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
-
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
-
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
-
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
| 1 |
+
*.pkl filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2 |
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 3 |
*.h5 filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 5 |
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 6 |
+
*.bin filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DEPLOYMENT_GUIDE.md
ADDED
|
@@ -0,0 +1,419 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# CloverShield ML API Deployment Guide
|
| 2 |
+
|
| 3 |
+
Complete guide for deploying the ML Inference API to various platforms.
|
| 4 |
+
|
| 5 |
+
## 📋 Prerequisites
|
| 6 |
+
|
| 7 |
+
- Docker installed (for containerized deployment)
|
| 8 |
+
- Model file: `Models/fraud_pipeline_final.pkl`
|
| 9 |
+
- Environment variables configured
|
| 10 |
+
- Platform account (Vercel/Render/Railway)
|
| 11 |
+
|
| 12 |
+
## 🐳 Docker Deployment
|
| 13 |
+
|
| 14 |
+
### Local Docker
|
| 15 |
+
|
| 16 |
+
```bash
|
| 17 |
+
# Build image
|
| 18 |
+
docker build -t clovershield-ml-api .
|
| 19 |
+
|
| 20 |
+
# Run container
|
| 21 |
+
docker run -d \
|
| 22 |
+
--name clovershield-api \
|
| 23 |
+
-p 8000:8000 \
|
| 24 |
+
-v $(pwd)/Models:/app/Models \
|
| 25 |
+
-e GROQ_API_KEY=your-key \
|
| 26 |
+
clovershield-ml-api
|
| 27 |
+
|
| 28 |
+
# Check logs
|
| 29 |
+
docker logs clovershield-api
|
| 30 |
+
|
| 31 |
+
# Test
|
| 32 |
+
curl http://localhost:8000/health
|
| 33 |
+
```
|
| 34 |
+
|
| 35 |
+
### Docker Compose
|
| 36 |
+
|
| 37 |
+
Create `docker-compose.yml`:
|
| 38 |
+
|
| 39 |
+
```yaml
|
| 40 |
+
version: '3.8'
|
| 41 |
+
|
| 42 |
+
services:
|
| 43 |
+
ml-api:
|
| 44 |
+
build: .
|
| 45 |
+
ports:
|
| 46 |
+
- "8000:8000"
|
| 47 |
+
volumes:
|
| 48 |
+
- ./Models:/app/Models
|
| 49 |
+
environment:
|
| 50 |
+
- MODEL_PATH=Models/fraud_pipeline_final.pkl
|
| 51 |
+
- GROQ_API_KEY=${GROQ_API_KEY}
|
| 52 |
+
healthcheck:
|
| 53 |
+
test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
|
| 54 |
+
interval: 30s
|
| 55 |
+
timeout: 10s
|
| 56 |
+
retries: 3
|
| 57 |
+
```
|
| 58 |
+
|
| 59 |
+
Run:
|
| 60 |
+
```bash
|
| 61 |
+
docker-compose up -d
|
| 62 |
+
```
|
| 63 |
+
|
| 64 |
+
## ☁️ Platform-Specific Deployment
|
| 65 |
+
|
| 66 |
+
### Option 1: Render.com (Recommended)
|
| 67 |
+
|
| 68 |
+
**Pros:**
|
| 69 |
+
- Easy setup
|
| 70 |
+
- Good for ML workloads
|
| 71 |
+
- Free tier available
|
| 72 |
+
- Supports Docker
|
| 73 |
+
|
| 74 |
+
**Steps:**
|
| 75 |
+
|
| 76 |
+
1. **Create Account**: Sign up at [render.com](https://render.com)
|
| 77 |
+
|
| 78 |
+
2. **Create New Web Service**:
|
| 79 |
+
- Connect GitHub repository
|
| 80 |
+
- Select "Docker" as environment
|
| 81 |
+
- Use `render.yaml` configuration
|
| 82 |
+
|
| 83 |
+
3. **Configure Environment Variables**:
|
| 84 |
+
```
|
| 85 |
+
MODEL_PATH=Models/fraud_pipeline_final.pkl
|
| 86 |
+
MODEL_THRESHOLD=0.00754482
|
| 87 |
+
GROQ_API_KEY=your-key-here
|
| 88 |
+
```
|
| 89 |
+
|
| 90 |
+
4. **Deploy**:
|
| 91 |
+
- Render will auto-detect `render.yaml`
|
| 92 |
+
- Build and deploy automatically
|
| 93 |
+
- Get public URL: `https://clovershield-ml-api.onrender.com`
|
| 94 |
+
|
| 95 |
+
5. **Update Model File**:
|
| 96 |
+
- Upload model to cloud storage (S3, etc.)
|
| 97 |
+
- Modify Dockerfile to download on startup
|
| 98 |
+
- Or use Render's persistent disk
|
| 99 |
+
|
| 100 |
+
**Model File Options:**
|
| 101 |
+
- **Option A**: Include in Docker image (not recommended - large)
|
| 102 |
+
- **Option B**: Download from S3/GCS on startup
|
| 103 |
+
- **Option C**: Use Render persistent disk
|
| 104 |
+
|
| 105 |
+
### Option 2: Railway.app
|
| 106 |
+
|
| 107 |
+
**Pros:**
|
| 108 |
+
- Simple deployment
|
| 109 |
+
- Good developer experience
|
| 110 |
+
- Automatic HTTPS
|
| 111 |
+
|
| 112 |
+
**Steps:**
|
| 113 |
+
|
| 114 |
+
1. **Create Account**: Sign up at [railway.app](https://railway.app)
|
| 115 |
+
|
| 116 |
+
2. **Create New Project**:
|
| 117 |
+
- Connect GitHub repository
|
| 118 |
+
- Railway auto-detects `railway.json`
|
| 119 |
+
|
| 120 |
+
3. **Configure Environment Variables**:
|
| 121 |
+
- Go to Variables tab
|
| 122 |
+
- Add required variables
|
| 123 |
+
|
| 124 |
+
4. **Deploy**:
|
| 125 |
+
- Railway builds and deploys automatically
|
| 126 |
+
- Get public URL: `https://clovershield-ml-api.up.railway.app`
|
| 127 |
+
|
| 128 |
+
### Option 3: Vercel (Limited)
|
| 129 |
+
|
| 130 |
+
**Pros:**
|
| 131 |
+
- Fast deployment
|
| 132 |
+
- Great for frontend
|
| 133 |
+
|
| 134 |
+
**Cons:**
|
| 135 |
+
- Limited for ML workloads
|
| 136 |
+
- Serverless function timeout limits
|
| 137 |
+
- Model size restrictions
|
| 138 |
+
|
| 139 |
+
**Steps:**
|
| 140 |
+
|
| 141 |
+
1. **Install Vercel CLI**: `npm i -g vercel`
|
| 142 |
+
|
| 143 |
+
2. **Deploy**:
|
| 144 |
+
```bash
|
| 145 |
+
vercel
|
| 146 |
+
```
|
| 147 |
+
|
| 148 |
+
3. **Configure**:
|
| 149 |
+
- Set environment variables in dashboard
|
| 150 |
+
- Note: Model file must be < 50MB
|
| 151 |
+
|
| 152 |
+
**Note**: Vercel is not ideal for ML APIs. Use Render or Railway instead.
|
| 153 |
+
|
| 154 |
+
### Option 4: AWS/GCP/Azure
|
| 155 |
+
|
| 156 |
+
For production deployments, consider:
|
| 157 |
+
|
| 158 |
+
- **AWS**: ECS, EKS, or Lambda (with container support)
|
| 159 |
+
- **GCP**: Cloud Run or GKE
|
| 160 |
+
- **Azure**: Container Instances or AKS
|
| 161 |
+
|
| 162 |
+
## 🔧 Model File Management
|
| 163 |
+
|
| 164 |
+
### Option 1: Include in Docker Image
|
| 165 |
+
|
| 166 |
+
**Pros**: Simple
|
| 167 |
+
**Cons**: Large image size, slower builds
|
| 168 |
+
|
| 169 |
+
```dockerfile
|
| 170 |
+
COPY Models/fraud_pipeline_final.pkl Models/
|
| 171 |
+
```
|
| 172 |
+
|
| 173 |
+
### Option 2: Download from Cloud Storage
|
| 174 |
+
|
| 175 |
+
**Pros**: Smaller image, flexible
|
| 176 |
+
**Cons**: Requires cloud storage setup
|
| 177 |
+
|
| 178 |
+
Modify `main.py` startup:
|
| 179 |
+
|
| 180 |
+
```python
|
| 181 |
+
import boto3
|
| 182 |
+
import os
|
| 183 |
+
|
| 184 |
+
def download_model():
|
| 185 |
+
s3 = boto3.client('s3')
|
| 186 |
+
s3.download_file('your-bucket', 'fraud_pipeline_final.pkl', 'Models/fraud_pipeline_final.pkl')
|
| 187 |
+
```
|
| 188 |
+
|
| 189 |
+
### Option 3: Environment Variable Path
|
| 190 |
+
|
| 191 |
+
**Pros**: Flexible
|
| 192 |
+
**Cons**: Requires manual setup
|
| 193 |
+
|
| 194 |
+
Set `MODEL_PATH` to point to mounted volume or cloud path.
|
| 195 |
+
|
| 196 |
+
## 🔒 Production Security
|
| 197 |
+
|
| 198 |
+
### 1. API Authentication
|
| 199 |
+
|
| 200 |
+
Add API key middleware:
|
| 201 |
+
|
| 202 |
+
```python
|
| 203 |
+
from fastapi import Header, HTTPException
|
| 204 |
+
|
| 205 |
+
API_KEY = os.getenv("API_KEY")
|
| 206 |
+
|
| 207 |
+
@app.middleware("http")
|
| 208 |
+
async def verify_api_key(request: Request, call_next):
|
| 209 |
+
if request.url.path.startswith("/docs"):
|
| 210 |
+
return await call_next(request)
|
| 211 |
+
|
| 212 |
+
api_key = request.headers.get("X-API-Key")
|
| 213 |
+
if api_key != API_KEY:
|
| 214 |
+
raise HTTPException(status_code=401, detail="Invalid API key")
|
| 215 |
+
|
| 216 |
+
return await call_next(request)
|
| 217 |
+
```
|
| 218 |
+
|
| 219 |
+
### 2. Rate Limiting
|
| 220 |
+
|
| 221 |
+
Install: `pip install slowapi`
|
| 222 |
+
|
| 223 |
+
```python
|
| 224 |
+
from slowapi import Limiter, _rate_limit_exceeded_handler
|
| 225 |
+
from slowapi.util import get_remote_address
|
| 226 |
+
|
| 227 |
+
limiter = Limiter(key_func=get_remote_address)
|
| 228 |
+
app.state.limiter = limiter
|
| 229 |
+
app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler)
|
| 230 |
+
|
| 231 |
+
@app.post("/predict")
|
| 232 |
+
@limiter.limit("10/minute")
|
| 233 |
+
async def predict(...):
|
| 234 |
+
...
|
| 235 |
+
```
|
| 236 |
+
|
| 237 |
+
### 3. HTTPS
|
| 238 |
+
|
| 239 |
+
- Use platform's built-in HTTPS (Render, Railway)
|
| 240 |
+
- Or configure reverse proxy (nginx, Traefik)
|
| 241 |
+
|
| 242 |
+
### 4. Monitoring
|
| 243 |
+
|
| 244 |
+
Add logging and monitoring:
|
| 245 |
+
|
| 246 |
+
```python
|
| 247 |
+
import logging
|
| 248 |
+
from prometheus_client import Counter, Histogram
|
| 249 |
+
|
| 250 |
+
request_count = Counter('requests_total', 'Total requests')
|
| 251 |
+
request_duration = Histogram('request_duration_seconds', 'Request duration')
|
| 252 |
+
|
| 253 |
+
@app.middleware("http")
|
| 254 |
+
async def log_requests(request: Request, call_next):
|
| 255 |
+
start_time = time.time()
|
| 256 |
+
response = await call_next(request)
|
| 257 |
+
duration = time.time() - start_time
|
| 258 |
+
request_duration.observe(duration)
|
| 259 |
+
request_count.inc()
|
| 260 |
+
return response
|
| 261 |
+
```
|
| 262 |
+
|
| 263 |
+
## 📊 Health Checks
|
| 264 |
+
|
| 265 |
+
All platforms support health checks:
|
| 266 |
+
|
| 267 |
+
```bash
|
| 268 |
+
# Test health endpoint
|
| 269 |
+
curl https://your-api-url.com/health
|
| 270 |
+
|
| 271 |
+
# Expected response
|
| 272 |
+
{
|
| 273 |
+
"status": "healthy",
|
| 274 |
+
"model_loaded": true,
|
| 275 |
+
"model_version": "1.0.0"
|
| 276 |
+
}
|
| 277 |
+
```
|
| 278 |
+
|
| 279 |
+
## 🧪 Testing Deployment
|
| 280 |
+
|
| 281 |
+
### 1. Health Check
|
| 282 |
+
|
| 283 |
+
```bash
|
| 284 |
+
curl https://your-api-url.com/health
|
| 285 |
+
```
|
| 286 |
+
|
| 287 |
+
### 2. Prediction Test
|
| 288 |
+
|
| 289 |
+
```bash
|
| 290 |
+
curl -X POST https://your-api-url.com/predict \
|
| 291 |
+
-H "Content-Type: application/json" \
|
| 292 |
+
-d '{
|
| 293 |
+
"transaction": {
|
| 294 |
+
"step": 1,
|
| 295 |
+
"type": "TRANSFER",
|
| 296 |
+
"amount": 5000.00,
|
| 297 |
+
"nameOrig": "C123456789",
|
| 298 |
+
"oldBalanceOrig": 50000.00,
|
| 299 |
+
"newBalanceOrig": 45000.00,
|
| 300 |
+
"nameDest": "C234567890",
|
| 301 |
+
"oldBalanceDest": 25000.00,
|
| 302 |
+
"newBalanceDest": 30000.00
|
| 303 |
+
}
|
| 304 |
+
}'
|
| 305 |
+
```
|
| 306 |
+
|
| 307 |
+
### 3. Load Testing
|
| 308 |
+
|
| 309 |
+
Use `ab` (Apache Bench) or `wrk`:
|
| 310 |
+
|
| 311 |
+
```bash
|
| 312 |
+
# Install
|
| 313 |
+
brew install wrk # macOS
|
| 314 |
+
# or
|
| 315 |
+
apt-get install apache2-utils # Linux
|
| 316 |
+
|
| 317 |
+
# Test
|
| 318 |
+
wrk -t4 -c100 -d30s --script=test.lua https://your-api-url.com/predict
|
| 319 |
+
```
|
| 320 |
+
|
| 321 |
+
## 🐛 Troubleshooting
|
| 322 |
+
|
| 323 |
+
### Model Not Loading
|
| 324 |
+
|
| 325 |
+
**Symptoms**: `503 Model not loaded` error
|
| 326 |
+
|
| 327 |
+
**Solutions**:
|
| 328 |
+
1. Check `MODEL_PATH` environment variable
|
| 329 |
+
2. Verify model file exists in container
|
| 330 |
+
3. Check file permissions
|
| 331 |
+
4. Review container logs
|
| 332 |
+
|
| 333 |
+
### Out of Memory
|
| 334 |
+
|
| 335 |
+
**Symptoms**: Container crashes, OOM errors
|
| 336 |
+
|
| 337 |
+
**Solutions**:
|
| 338 |
+
1. Increase container memory limit
|
| 339 |
+
2. Use model quantization
|
| 340 |
+
3. Implement model caching
|
| 341 |
+
4. Scale horizontally
|
| 342 |
+
|
| 343 |
+
### Slow Response Times
|
| 344 |
+
|
| 345 |
+
**Symptoms**: High latency
|
| 346 |
+
|
| 347 |
+
**Solutions**:
|
| 348 |
+
1. Optimize model (quantization, pruning)
|
| 349 |
+
2. Use faster hardware (GPU if available)
|
| 350 |
+
3. Implement caching
|
| 351 |
+
4. Optimize feature engineering
|
| 352 |
+
|
| 353 |
+
### CORS Errors
|
| 354 |
+
|
| 355 |
+
**Symptoms**: Frontend can't call API
|
| 356 |
+
|
| 357 |
+
**Solutions**:
|
| 358 |
+
1. Configure CORS in `main.py`
|
| 359 |
+
2. Add frontend domain to allowed origins
|
| 360 |
+
3. Check platform CORS settings
|
| 361 |
+
|
| 362 |
+
## 📈 Scaling
|
| 363 |
+
|
| 364 |
+
### Horizontal Scaling
|
| 365 |
+
|
| 366 |
+
- Use load balancer
|
| 367 |
+
- Deploy multiple instances
|
| 368 |
+
- Use platform auto-scaling
|
| 369 |
+
|
| 370 |
+
### Vertical Scaling
|
| 371 |
+
|
| 372 |
+
- Increase container resources
|
| 373 |
+
- Use faster CPU/GPU
|
| 374 |
+
- Optimize model
|
| 375 |
+
|
| 376 |
+
## ✅ Deployment Checklist
|
| 377 |
+
|
| 378 |
+
- [ ] Model file accessible
|
| 379 |
+
- [ ] Environment variables set
|
| 380 |
+
- [ ] Health check working
|
| 381 |
+
- [ ] API endpoints responding
|
| 382 |
+
- [ ] CORS configured
|
| 383 |
+
- [ ] Authentication enabled (production)
|
| 384 |
+
- [ ] Rate limiting configured
|
| 385 |
+
- [ ] Monitoring set up
|
| 386 |
+
- [ ] Logs accessible
|
| 387 |
+
- [ ] HTTPS enabled
|
| 388 |
+
- [ ] Backup strategy in place
|
| 389 |
+
|
| 390 |
+
## 🚀 Quick Deploy Commands
|
| 391 |
+
|
| 392 |
+
### Render
|
| 393 |
+
|
| 394 |
+
```bash
|
| 395 |
+
# Using Render CLI
|
| 396 |
+
render deploy
|
| 397 |
+
```
|
| 398 |
+
|
| 399 |
+
### Railway
|
| 400 |
+
|
| 401 |
+
```bash
|
| 402 |
+
# Using Railway CLI
|
| 403 |
+
railway up
|
| 404 |
+
```
|
| 405 |
+
|
| 406 |
+
### Docker Hub + Any Platform
|
| 407 |
+
|
| 408 |
+
```bash
|
| 409 |
+
# Build and push
|
| 410 |
+
docker build -t your-username/clovershield-ml-api .
|
| 411 |
+
docker push your-username/clovershield-ml-api
|
| 412 |
+
|
| 413 |
+
# Deploy to platform using image
|
| 414 |
+
```
|
| 415 |
+
|
| 416 |
+
---
|
| 417 |
+
|
| 418 |
+
**Status**: Ready for deployment! 🚀
|
| 419 |
+
|
Dockerfile
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# CloverShield ML Inference API Dockerfile
|
| 2 |
+
# Optimized for Python 3.9 and Hugging Face Spaces
|
| 3 |
+
|
| 4 |
+
FROM python:3.9-slim
|
| 5 |
+
|
| 6 |
+
# Create a non-root user (user 1000) to comply with HF security context
|
| 7 |
+
RUN useradd -m -u 1000 user
|
| 8 |
+
|
| 9 |
+
# Set working directory
|
| 10 |
+
WORKDIR /app
|
| 11 |
+
|
| 12 |
+
# Set environment variables
|
| 13 |
+
ENV PYTHONUNBUFFERED=1 \
|
| 14 |
+
PYTHONDONTWRITEBYTECODE=1 \
|
| 15 |
+
PIP_NO_CACHE_DIR=1 \
|
| 16 |
+
PIP_DISABLE_PIP_VERSION_CHECK=1 \
|
| 17 |
+
HOME=/home/user \
|
| 18 |
+
PATH=/home/user/.local/bin:$PATH
|
| 19 |
+
|
| 20 |
+
# Install system dependencies
|
| 21 |
+
RUN apt-get update && apt-get install -y \
|
| 22 |
+
gcc \
|
| 23 |
+
g++ \
|
| 24 |
+
&& rm -rf /var/lib/apt/lists/*
|
| 25 |
+
|
| 26 |
+
# Copy requirements first (for better Docker layer caching)
|
| 27 |
+
COPY requirements.txt .
|
| 28 |
+
|
| 29 |
+
# Install Python dependencies as the non-root user
|
| 30 |
+
RUN pip install --upgrade pip && \
|
| 31 |
+
pip install -r requirements.txt
|
| 32 |
+
|
| 33 |
+
# Copy application code
|
| 34 |
+
COPY --chown=user . .
|
| 35 |
+
|
| 36 |
+
# Create directory for model (if not exists) and ensure permissions
|
| 37 |
+
RUN mkdir -p Models && chown -R user:user /app
|
| 38 |
+
|
| 39 |
+
# Copy and set up entrypoint script
|
| 40 |
+
COPY --chown=user entrypoint.sh /entrypoint.sh
|
| 41 |
+
RUN chmod +x /entrypoint.sh
|
| 42 |
+
|
| 43 |
+
# Switch to the non-root user
|
| 44 |
+
USER user
|
| 45 |
+
|
| 46 |
+
# Expose port 7860 (Hugging Face Spaces standard)
|
| 47 |
+
EXPOSE 7860
|
| 48 |
+
|
| 49 |
+
# Health check (uses PORT env var, defaults to 7860)
|
| 50 |
+
HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \
|
| 51 |
+
CMD python -c "import os, urllib.request; port=os.getenv('PORT', '7860'); urllib.request.urlopen(f'http://localhost:{port}/health')" || exit 1
|
| 52 |
+
|
| 53 |
+
# Run the application using entrypoint script
|
| 54 |
+
ENTRYPOINT ["/entrypoint.sh"]
|
| 55 |
+
|
HUGGINGFACE_DEPLOYMENT.md
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Deploying CloverShield ML API to Hugging Face Spaces
|
| 2 |
+
|
| 3 |
+
This guide outlines the steps to migrate the backend ML API from Render to Hugging Face Spaces using the Docker SDK.
|
| 4 |
+
|
| 5 |
+
## Prerequisites
|
| 6 |
+
|
| 7 |
+
- A [Hugging Face account](https://huggingface.co/join).
|
| 8 |
+
- The `ml-api` folder ready with the updated `Dockerfile` and `README.md`.
|
| 9 |
+
|
| 10 |
+
## Step 1: Create a New Space
|
| 11 |
+
|
| 12 |
+
1. Log in to your Hugging Face account.
|
| 13 |
+
2. Click on your profile picture in the top right and select **"New Space"**.
|
| 14 |
+
3. Fill in the details:
|
| 15 |
+
- **Space Name:** `clovershield-ml-api` (or similar).
|
| 16 |
+
- **License:** MIT or Apache 2.0 (matches your project).
|
| 17 |
+
- **Space SDK:** Select **Docker**.
|
| 18 |
+
- **Space Hardware:** **CPU Basic (Free)** is usually sufficient for this model size (2MB). If performance is slow, you can upgrade later.
|
| 19 |
+
- **Visibility:** **Public** or **Private** (Private is safer for internal testing, but you might need a token to access it from the frontend). For the hackathon demo, **Public** is easiest but be careful with keys. **Private** is recommended if you can handle authentication headers in your frontend.
|
| 20 |
+
|
| 21 |
+
4. Click **"Create Space"**.
|
| 22 |
+
|
| 23 |
+
## Step 2: Upload Files
|
| 24 |
+
|
| 25 |
+
You have two options to upload the code:
|
| 26 |
+
|
| 27 |
+
### Option A: Web Interface (Easiest)
|
| 28 |
+
|
| 29 |
+
1. Once the Space is created, you will see the instructions page. Click on the **"Files"** tab.
|
| 30 |
+
2. Click **"Add file"** > **"Upload files"**.
|
| 31 |
+
3. Drag and drop **ALL** files and folders from your local `ml-api` directory into the upload area.
|
| 32 |
+
- **Important:** Ensure you include:
|
| 33 |
+
- `Dockerfile`
|
| 34 |
+
- `requirements.txt`
|
| 35 |
+
- `main.py`
|
| 36 |
+
- `inference.py`
|
| 37 |
+
- `feature_engineering.py`
|
| 38 |
+
- `entrypoint.sh`
|
| 39 |
+
- `README.md`
|
| 40 |
+
- `Models/fraud_pipeline_final.pkl` (The folder structure `Models/` must be preserved).
|
| 41 |
+
- `dataset/test_dataset.csv` (If used).
|
| 42 |
+
4. In the "Commit changes" box, type "Initial commit" and click **"Commit changes to main"**.
|
| 43 |
+
|
| 44 |
+
### Option B: Git Command Line (Manual)
|
| 45 |
+
|
| 46 |
+
1. Clone the Space repository locally.
|
| 47 |
+
2. Copy files and push manually.
|
| 48 |
+
|
| 49 |
+
### Option C: Automatic Sync with GitHub (Recommended)
|
| 50 |
+
|
| 51 |
+
This method automatically updates your Space whenever you push code to GitHub.
|
| 52 |
+
|
| 53 |
+
1. **Get a Hugging Face Access Token:**
|
| 54 |
+
* Go to [Hugging Face Settings > Access Tokens](https://huggingface.co/settings/tokens).
|
| 55 |
+
* Click "New token".
|
| 56 |
+
* Name it `GITHUB_ACTION` (or similar).
|
| 57 |
+
* **Permissions:** Select **"Write"** access.
|
| 58 |
+
* Copy the token (starts with `hf_...`).
|
| 59 |
+
|
| 60 |
+
2. **Add Token to GitHub Secrets:**
|
| 61 |
+
* Go to your GitHub repository.
|
| 62 |
+
* Navigate to **Settings** > **Secrets and variables** > **Actions**.
|
| 63 |
+
* Click **"New repository secret"**.
|
| 64 |
+
* **Name:** `HF_TOKEN`
|
| 65 |
+
* **Value:** (Paste your Hugging Face token here).
|
| 66 |
+
* Click "Add secret".
|
| 67 |
+
|
| 68 |
+
3. **Configure the Workflow:**
|
| 69 |
+
* Open the file `.github/workflows/deploy-ml-api.yml` in your project.
|
| 70 |
+
* Edit the `env` section to match your details:
|
| 71 |
+
```yaml
|
| 72 |
+
HF_SPACE_USERNAME: "your-hf-username"
|
| 73 |
+
HF_SPACE_NAME: "clovershield-ml-api"
|
| 74 |
+
```
|
| 75 |
+
* Commit and push this change to GitHub.
|
| 76 |
+
|
| 77 |
+
Now, whenever you push changes to the `ml-api` folder in GitHub, the Action will run and update your Hugging Face Space automatically.
|
| 78 |
+
|
| 79 |
+
## Step 3: Configure Environment Variables (Secrets)
|
| 80 |
+
|
| 81 |
+
Your API needs the `GROQ_API_KEY` to function if LLM explanations are enabled.
|
| 82 |
+
|
| 83 |
+
1. Go to the **"Settings"** tab of your Space.
|
| 84 |
+
2. Scroll down to the **"Variables and secrets"** section.
|
| 85 |
+
3. Click **"New secret"**.
|
| 86 |
+
- **Name:** `GROQ_API_KEY`
|
| 87 |
+
- **Value:** (Paste your Groq API key here)
|
| 88 |
+
4. (Optional) Add other variables if you changed defaults:
|
| 89 |
+
- `MODEL_THRESHOLD`
|
| 90 |
+
- `PAGERANK_LIMIT`
|
| 91 |
+
|
| 92 |
+
**Note:** Hugging Face automatically sets `PORT` to `7860`. Our updated `Dockerfile` and `entrypoint.sh` are configured to respect this.
|
| 93 |
+
|
| 94 |
+
## Step 4: Build and Verify
|
| 95 |
+
|
| 96 |
+
1. Go to the **"App"** tab.
|
| 97 |
+
2. You will see a "Building" status. Click "Logs" to watch the build process.
|
| 98 |
+
3. Once "Running", you should see the Swagger UI or the JSON response from the root endpoint in the preview window.
|
| 99 |
+
4. Your API URL will be: `https://<your-username>-clovershield-ml-api.hf.space`
|
| 100 |
+
|
| 101 |
+
## Step 5: Update Frontend Connection (Vercel)
|
| 102 |
+
|
| 103 |
+
Now that your backend is on HF Spaces, you need to update your Vercel deployment to point to the new URL.
|
| 104 |
+
|
| 105 |
+
1. **Go to Vercel Dashboard:**
|
| 106 |
+
* Navigate to your project on [Vercel](https://vercel.com).
|
| 107 |
+
* Go to **Settings** > **Environment Variables**.
|
| 108 |
+
|
| 109 |
+
2. **Update the Variable:**
|
| 110 |
+
* Find the existing `NEXT_PUBLIC_ML_API_URL` or create a new one.
|
| 111 |
+
* **Value:** `https://<your-username>-<your-space-name>.hf.space`
|
| 112 |
+
* *Important:* Do NOT include a trailing slash (/) at the end.
|
| 113 |
+
|
| 114 |
+
3. **Redeploy:**
|
| 115 |
+
* Go to the **Deployments** tab.
|
| 116 |
+
* Click on the three dots (...) of your latest deployment and select **Redeploy**. This is necessary for Vercel to pick up the new environment variable value in the client-side code.
|
| 117 |
+
|
| 118 |
+
## Troubleshooting
|
| 119 |
+
|
| 120 |
+
- **Build Fails:** Check the "Logs". Common issues are missing dependencies in `requirements.txt` or file path errors.
|
| 121 |
+
- **"403 Forbidden":** If your Space is **Private**, you cannot access it directly from a public frontend without passing an authorization header (Bearer token). For this hackathon/demo, switch the Space Visibility to **Public** in Settings > General.
|
| 122 |
+
- **Model Load Error:** Ensure `Models/fraud_pipeline_final.pkl` was actually uploaded. Check the "Files" tab.
|
| 123 |
+
|
Models/fraud_pipeline_final.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:817193f2758c331d61f6749771a821ad1eb790c7aace69a3982b7282aa7f7e27
|
| 3 |
+
size 2259837
|
QUICK_START.md
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ML API Quick Start
|
| 2 |
+
|
| 3 |
+
Quick reference for deploying CloverShield ML Inference API.
|
| 4 |
+
|
| 5 |
+
## ✅ What's Included
|
| 6 |
+
|
| 7 |
+
- ✅ FastAPI service with `/predict` endpoint
|
| 8 |
+
- ✅ Docker containerization
|
| 9 |
+
- ✅ Deployment configs for Render/Railway/Vercel
|
| 10 |
+
- ✅ Complete documentation
|
| 11 |
+
|
| 12 |
+
## 🚀 Deploy in 5 Minutes
|
| 13 |
+
|
| 14 |
+
### Option 1: Render.com (Recommended)
|
| 15 |
+
|
| 16 |
+
1. **Sign up** at [render.com](https://render.com)
|
| 17 |
+
2. **Create Web Service** → Connect GitHub repo
|
| 18 |
+
3. **Configure**:
|
| 19 |
+
- Environment: Docker
|
| 20 |
+
- Build Command: (auto-detected)
|
| 21 |
+
- Start Command: (auto-detected)
|
| 22 |
+
4. **Set Environment Variables**:
|
| 23 |
+
```
|
| 24 |
+
MODEL_PATH=Models/fraud_pipeline_final.pkl
|
| 25 |
+
GROQ_API_KEY=your-key (optional)
|
| 26 |
+
```
|
| 27 |
+
5. **Deploy** → Get URL: `https://your-app.onrender.com`
|
| 28 |
+
|
| 29 |
+
### Option 2: Railway.app
|
| 30 |
+
|
| 31 |
+
1. **Sign up** at [railway.app](https://railway.app)
|
| 32 |
+
2. **New Project** → Connect GitHub
|
| 33 |
+
3. **Railway auto-detects** `railway.json`
|
| 34 |
+
4. **Set environment variables** in dashboard
|
| 35 |
+
5. **Deploy** → Get URL automatically
|
| 36 |
+
|
| 37 |
+
### Option 3: Local Docker
|
| 38 |
+
|
| 39 |
+
```bash
|
| 40 |
+
# Build
|
| 41 |
+
docker build -t clovershield-ml-api .
|
| 42 |
+
|
| 43 |
+
# Run
|
| 44 |
+
docker run -p 8000:8000 \
|
| 45 |
+
-v $(pwd)/Models:/app/Models \
|
| 46 |
+
clovershield-ml-api
|
| 47 |
+
|
| 48 |
+
# Test
|
| 49 |
+
curl http://localhost:8000/health
|
| 50 |
+
```
|
| 51 |
+
|
| 52 |
+
## 📝 API Usage
|
| 53 |
+
|
| 54 |
+
### Health Check
|
| 55 |
+
```bash
|
| 56 |
+
curl https://your-api-url.com/health
|
| 57 |
+
```
|
| 58 |
+
|
| 59 |
+
### Predict Fraud
|
| 60 |
+
```bash
|
| 61 |
+
curl -X POST https://your-api-url.com/predict \
|
| 62 |
+
-H "Content-Type: application/json" \
|
| 63 |
+
-d '{
|
| 64 |
+
"transaction": {
|
| 65 |
+
"step": 1,
|
| 66 |
+
"type": "TRANSFER",
|
| 67 |
+
"amount": 5000.00,
|
| 68 |
+
"nameOrig": "C123456789",
|
| 69 |
+
"oldBalanceOrig": 50000.00,
|
| 70 |
+
"newBalanceOrig": 45000.00,
|
| 71 |
+
"nameDest": "C234567890",
|
| 72 |
+
"oldBalanceDest": 25000.00,
|
| 73 |
+
"newBalanceDest": 30000.00
|
| 74 |
+
}
|
| 75 |
+
}'
|
| 76 |
+
```
|
| 77 |
+
|
| 78 |
+
## 📚 Full Documentation
|
| 79 |
+
|
| 80 |
+
- [README.md](./README.md) - Complete API documentation
|
| 81 |
+
- [DEPLOYMENT_GUIDE.md](./DEPLOYMENT_GUIDE.md) - Detailed deployment guide
|
| 82 |
+
|
| 83 |
+
## ✅ Checklist
|
| 84 |
+
|
| 85 |
+
- [ ] Model file available (`Models/fraud_pipeline_final.pkl`)
|
| 86 |
+
- [ ] Environment variables configured
|
| 87 |
+
- [ ] API deployed and accessible
|
| 88 |
+
- [ ] Health check passing
|
| 89 |
+
- [ ] Test prediction working
|
| 90 |
+
|
| 91 |
+
**Ready to deploy!** 🚀
|
| 92 |
+
|
README.md
CHANGED
|
@@ -1,11 +1,310 @@
|
|
| 1 |
---
|
| 2 |
-
title:
|
| 3 |
-
emoji:
|
| 4 |
-
colorFrom:
|
| 5 |
-
colorTo:
|
| 6 |
sdk: docker
|
| 7 |
-
|
| 8 |
-
license: apache-2.0
|
| 9 |
---
|
| 10 |
|
| 11 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: CloverShield ML API
|
| 3 |
+
emoji: 🛡️
|
| 4 |
+
colorFrom: green
|
| 5 |
+
colorTo: blue
|
| 6 |
sdk: docker
|
| 7 |
+
app_port: 7860
|
|
|
|
| 8 |
---
|
| 9 |
|
| 10 |
+
# CloverShield ML Inference API
|
| 11 |
+
|
| 12 |
+
FastAPI microservice for fraud detection predictions. This service replaces the local model loading mechanism in the Streamlit app.
|
| 13 |
+
|
| 14 |
+
## 🚀 Quick Start
|
| 15 |
+
|
| 16 |
+
### Local Development
|
| 17 |
+
|
| 18 |
+
1. **Install dependencies:**
|
| 19 |
+
```bash
|
| 20 |
+
pip install -r requirements.txt
|
| 21 |
+
```
|
| 22 |
+
|
| 23 |
+
2. **Copy model file:**
|
| 24 |
+
```bash
|
| 25 |
+
cp ../Models/fraud_pipeline_final.pkl Models/
|
| 26 |
+
```
|
| 27 |
+
|
| 28 |
+
3. **Set environment variables:**
|
| 29 |
+
```bash
|
| 30 |
+
cp env.template .env
|
| 31 |
+
# Edit .env with your values
|
| 32 |
+
```
|
| 33 |
+
|
| 34 |
+
4. **Run the API:**
|
| 35 |
+
```bash
|
| 36 |
+
python main.py
|
| 37 |
+
# Or
|
| 38 |
+
uvicorn main:app --reload
|
| 39 |
+
```
|
| 40 |
+
|
| 41 |
+
5. **Test the API:**
|
| 42 |
+
```bash
|
| 43 |
+
curl http://localhost:8000/health
|
| 44 |
+
```
|
| 45 |
+
|
| 46 |
+
### Docker
|
| 47 |
+
|
| 48 |
+
1. **Build the image:**
|
| 49 |
+
```bash
|
| 50 |
+
docker build -t clovershield-ml-api .
|
| 51 |
+
```
|
| 52 |
+
|
| 53 |
+
2. **Run the container:**
|
| 54 |
+
```bash
|
| 55 |
+
docker run -p 8000:8000 \
|
| 56 |
+
-v $(pwd)/Models:/app/Models \
|
| 57 |
+
-e GROQ_API_KEY=your-key \
|
| 58 |
+
clovershield-ml-api
|
| 59 |
+
```
|
| 60 |
+
|
| 61 |
+
## 📚 API Endpoints
|
| 62 |
+
|
| 63 |
+
### `GET /`
|
| 64 |
+
Root endpoint - service information
|
| 65 |
+
|
| 66 |
+
### `GET /health`
|
| 67 |
+
Health check endpoint
|
| 68 |
+
|
| 69 |
+
**Response:**
|
| 70 |
+
```json
|
| 71 |
+
{
|
| 72 |
+
"status": "healthy",
|
| 73 |
+
"model_loaded": true,
|
| 74 |
+
"model_version": "1.0.0",
|
| 75 |
+
"shap_available": true,
|
| 76 |
+
"llm_available": true
|
| 77 |
+
}
|
| 78 |
+
```
|
| 79 |
+
|
| 80 |
+
### `GET /model/info`
|
| 81 |
+
Get model metadata
|
| 82 |
+
|
| 83 |
+
### `POST /predict`
|
| 84 |
+
Predict fraud probability for a single transaction
|
| 85 |
+
|
| 86 |
+
**Request:**
|
| 87 |
+
```json
|
| 88 |
+
{
|
| 89 |
+
"transaction": {
|
| 90 |
+
"step": 1,
|
| 91 |
+
"type": "CASH_OUT",
|
| 92 |
+
"amount": 5000.00,
|
| 93 |
+
"nameOrig": "C123456789",
|
| 94 |
+
"oldBalanceOrig": 50000.00,
|
| 95 |
+
"newBalanceOrig": 45000.00,
|
| 96 |
+
"nameDest": "C234567890",
|
| 97 |
+
"oldBalanceDest": 25000.00,
|
| 98 |
+
"newBalanceDest": 30000.00
|
| 99 |
+
},
|
| 100 |
+
"options": {
|
| 101 |
+
"include_shap": true,
|
| 102 |
+
"include_llm_explanation": false,
|
| 103 |
+
"language": "en",
|
| 104 |
+
"topk": 10
|
| 105 |
+
}
|
| 106 |
+
}
|
| 107 |
+
```
|
| 108 |
+
|
| 109 |
+
**Response:**
|
| 110 |
+
```json
|
| 111 |
+
{
|
| 112 |
+
"transaction_id": "550e8400-e29b-41d4-a716-446655440000",
|
| 113 |
+
"prediction": {
|
| 114 |
+
"fraud_probability": 0.2345,
|
| 115 |
+
"decision": "pass",
|
| 116 |
+
"risk_level": "low",
|
| 117 |
+
"confidence": 0.85
|
| 118 |
+
},
|
| 119 |
+
"shap_explanations": [
|
| 120 |
+
{
|
| 121 |
+
"feature": "amount_over_oldBalanceOrig",
|
| 122 |
+
"value": 0.1,
|
| 123 |
+
"shap": 0.0234,
|
| 124 |
+
"shap_abs": 0.0234,
|
| 125 |
+
"rank": 1
|
| 126 |
+
}
|
| 127 |
+
],
|
| 128 |
+
"llm_explanation": null,
|
| 129 |
+
"processing_time_ms": 145,
|
| 130 |
+
"model_version": "1.0.0",
|
| 131 |
+
"timestamp": "2024-01-15T10:30:00Z"
|
| 132 |
+
}
|
| 133 |
+
```
|
| 134 |
+
|
| 135 |
+
### `POST /predict/batch`
|
| 136 |
+
Predict fraud probability for multiple transactions
|
| 137 |
+
|
| 138 |
+
## 🐳 Docker Deployment
|
| 139 |
+
|
| 140 |
+
### Build and Push to Registry
|
| 141 |
+
|
| 142 |
+
```bash
|
| 143 |
+
# Build
|
| 144 |
+
docker build -t clovershield-ml-api:latest .
|
| 145 |
+
|
| 146 |
+
# Tag for registry
|
| 147 |
+
docker tag clovershield-ml-api:latest your-registry/clovershield-ml-api:latest
|
| 148 |
+
|
| 149 |
+
# Push
|
| 150 |
+
docker push your-registry/clovershield-ml-api:latest
|
| 151 |
+
```
|
| 152 |
+
|
| 153 |
+
## ☁️ Platform Deployment
|
| 154 |
+
|
| 155 |
+
### Hugging Face Spaces (Recommended)
|
| 156 |
+
|
| 157 |
+
1. Create a new Space on Hugging Face.
|
| 158 |
+
2. Select "Docker" as the SDK.
|
| 159 |
+
3. Upload the contents of this directory to the Space.
|
| 160 |
+
4. Set the `GROQ_API_KEY` secret in the Space settings.
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
### Vercel
|
| 164 |
+
|
| 165 |
+
1. Install Vercel CLI: `npm i -g vercel`
|
| 166 |
+
2. Deploy: `vercel`
|
| 167 |
+
3. Set environment variables in Vercel dashboard
|
| 168 |
+
|
| 169 |
+
**Note:** Vercel has limitations for ML models. Consider Render or Railway for better performance.
|
| 170 |
+
|
| 171 |
+
### Render
|
| 172 |
+
|
| 173 |
+
1. Connect your GitHub repository
|
| 174 |
+
2. Create new Web Service
|
| 175 |
+
3. Use `render.yaml` configuration
|
| 176 |
+
4. Set environment variables in dashboard
|
| 177 |
+
5. Deploy
|
| 178 |
+
|
| 179 |
+
### Railway
|
| 180 |
+
|
| 181 |
+
1. Connect your GitHub repository
|
| 182 |
+
2. Create new project
|
| 183 |
+
3. Railway will auto-detect `railway.json`
|
| 184 |
+
4. Set environment variables
|
| 185 |
+
5. Deploy
|
| 186 |
+
|
| 187 |
+
## 🔧 Configuration
|
| 188 |
+
|
| 189 |
+
### Environment Variables
|
| 190 |
+
|
| 191 |
+
- `MODEL_PATH` - Path to model file (default: `Models/fraud_pipeline_final.pkl`)
|
| 192 |
+
- `TEST_DATASET_PATH` - Path to test dataset CSV for feature engineering (optional, auto-detected if not provided)
|
| 193 |
+
- `MODEL_THRESHOLD` - Decision threshold (default: `0.00754482` - optimized for 99% recall)
|
| 194 |
+
- `PORT` - Server port (default: `8000`)
|
| 195 |
+
- `HOST` - Server host (default: `0.0.0.0`)
|
| 196 |
+
- `GROQ_API_KEY` - Optional Groq API key for LLM explanations
|
| 197 |
+
- `PAGERANK_LIMIT` - Optional limit on nodes for PageRank computation (for memory optimization)
|
| 198 |
+
- `MAX_FIT_ROWS` - Maximum rows to use for feature engineering fitting (default: `50000`)
|
| 199 |
+
|
| 200 |
+
### Model File
|
| 201 |
+
|
| 202 |
+
The model file (`fraud_pipeline_final.pkl`) must be available at runtime. Options:
|
| 203 |
+
|
| 204 |
+
1. **Include in Docker image** (not recommended for large files)
|
| 205 |
+
2. **Mount as volume** (Docker)
|
| 206 |
+
3. **Download from cloud storage** (S3, GCS, etc.)
|
| 207 |
+
4. **Use environment variable** to specify path
|
| 208 |
+
|
| 209 |
+
## 📊 API Documentation
|
| 210 |
+
|
| 211 |
+
Interactive API documentation available at:
|
| 212 |
+
- Swagger UI: `http://localhost:8000/docs`
|
| 213 |
+
- ReDoc: `http://localhost:8000/redoc`
|
| 214 |
+
|
| 215 |
+
## 🧪 Testing
|
| 216 |
+
|
| 217 |
+
### Test with curl
|
| 218 |
+
|
| 219 |
+
```bash
|
| 220 |
+
# Health check
|
| 221 |
+
curl http://localhost:8000/health
|
| 222 |
+
|
| 223 |
+
# Prediction
|
| 224 |
+
curl -X POST http://localhost:8000/predict \
|
| 225 |
+
-H "Content-Type: application/json" \
|
| 226 |
+
-d '{
|
| 227 |
+
"transaction": {
|
| 228 |
+
"step": 1,
|
| 229 |
+
"type": "TRANSFER",
|
| 230 |
+
"amount": 5000.00,
|
| 231 |
+
"nameOrig": "C123456789",
|
| 232 |
+
"oldBalanceOrig": 50000.00,
|
| 233 |
+
"newBalanceOrig": 45000.00,
|
| 234 |
+
"nameDest": "C234567890",
|
| 235 |
+
"oldBalanceDest": 25000.00,
|
| 236 |
+
"newBalanceDest": 30000.00
|
| 237 |
+
}
|
| 238 |
+
}'
|
| 239 |
+
```
|
| 240 |
+
|
| 241 |
+
### Test with Python
|
| 242 |
+
|
| 243 |
+
```python
|
| 244 |
+
import requests
|
| 245 |
+
|
| 246 |
+
response = requests.post(
|
| 247 |
+
"http://localhost:8000/predict",
|
| 248 |
+
json={
|
| 249 |
+
"transaction": {
|
| 250 |
+
"step": 1,
|
| 251 |
+
"type": "TRANSFER",
|
| 252 |
+
"amount": 5000.00,
|
| 253 |
+
"nameOrig": "C123456789",
|
| 254 |
+
"oldBalanceOrig": 50000.00,
|
| 255 |
+
"newBalanceOrig": 45000.00,
|
| 256 |
+
"nameDest": "C234567890",
|
| 257 |
+
"oldBalanceDest": 25000.00,
|
| 258 |
+
"newBalanceDest": 30000.00
|
| 259 |
+
}
|
| 260 |
+
}
|
| 261 |
+
)
|
| 262 |
+
|
| 263 |
+
print(response.json())
|
| 264 |
+
```
|
| 265 |
+
|
| 266 |
+
## 🔒 Security
|
| 267 |
+
|
| 268 |
+
**Production Recommendations:**
|
| 269 |
+
- Add API key authentication for production
|
| 270 |
+
- Use HTTPS in production
|
| 271 |
+
- Implement rate limiting
|
| 272 |
+
- Validate all inputs
|
| 273 |
+
- Monitor for abuse
|
| 274 |
+
- Restrict CORS origins (currently set to `["*"` for development)
|
| 275 |
+
- Use environment variables for sensitive data
|
| 276 |
+
|
| 277 |
+
## 📈 Performance
|
| 278 |
+
|
| 279 |
+
**Optimization Tips:**
|
| 280 |
+
- Model loads once on startup (lazy loading supported for serverless)
|
| 281 |
+
- Single worker recommended (ML models are not thread-safe)
|
| 282 |
+
- Consider caching for frequently accessed data
|
| 283 |
+
- Monitor memory usage (XGBoost models can be large)
|
| 284 |
+
- Feature engineering uses sampled dataset (configurable via `MAX_FIT_ROWS`)
|
| 285 |
+
- SHAP background uses minimal sample size (100 rows) for efficiency
|
| 286 |
+
|
| 287 |
+
## 🐛 Troubleshooting
|
| 288 |
+
|
| 289 |
+
### Model not loading
|
| 290 |
+
|
| 291 |
+
- Check `MODEL_PATH` environment variable
|
| 292 |
+
- Verify model file exists
|
| 293 |
+
- Check file permissions
|
| 294 |
+
- Review logs for error messages
|
| 295 |
+
|
| 296 |
+
### SHAP not working
|
| 297 |
+
|
| 298 |
+
- Ensure XGBoost is installed
|
| 299 |
+
- Check model type compatibility
|
| 300 |
+
- Review SHAP initialization logs
|
| 301 |
+
|
| 302 |
+
### LLM explanations failing
|
| 303 |
+
|
| 304 |
+
- Verify `GROQ_API_KEY` is set
|
| 305 |
+
- Check API key validity
|
| 306 |
+
- Review Groq API status
|
| 307 |
+
|
| 308 |
+
## 📝 License
|
| 309 |
+
|
| 310 |
+
See main project LICENSE file.
|
dataset/test_dataset.csv.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5ce99ce839f2f1f427668dd8e4fd7d74b975395222082356e38c89c326bfba36
|
| 3 |
+
size 4033802
|
entrypoint.sh
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/bin/bash
|
| 2 |
+
# Entrypoint script for CloverShield ML API
|
| 3 |
+
# Reads PORT from environment variable (required by Render)
|
| 4 |
+
|
| 5 |
+
PORT=${PORT:-8000}
|
| 6 |
+
HOST=${HOST:-0.0.0.0}
|
| 7 |
+
|
| 8 |
+
echo "🚀 Starting CloverShield ML API"
|
| 9 |
+
echo "📋 Configuration: HOST=$HOST, PORT=$PORT"
|
| 10 |
+
echo "⏳ Server will start, model will load on startup..."
|
| 11 |
+
echo "💡 Note: First request may take longer if model is still loading"
|
| 12 |
+
|
| 13 |
+
# Start uvicorn with the PORT from environment
|
| 14 |
+
# Use --log-level info for better visibility
|
| 15 |
+
exec uvicorn main:app --host "$HOST" --port "$PORT" --workers 1 --log-level info
|
| 16 |
+
|
env.template
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# CloverShield ML API Environment Variables
|
| 2 |
+
# Copy this file to .env and fill in your values
|
| 3 |
+
|
| 4 |
+
# Model Configuration
|
| 5 |
+
MODEL_PATH=Models/fraud_pipeline_final.pkl
|
| 6 |
+
MODEL_THRESHOLD=0.00754482
|
| 7 |
+
|
| 8 |
+
# Server Configuration
|
| 9 |
+
PORT=8000
|
| 10 |
+
HOST=0.0.0.0
|
| 11 |
+
|
| 12 |
+
# Memory Optimization (for limited RAM environments like Render free tier)
|
| 13 |
+
# Maximum number of rows to use for fitting feature engineer (default: 50000)
|
| 14 |
+
# Reduce this if you're running out of RAM (e.g., 20000 or 10000)
|
| 15 |
+
MAX_FIT_ROWS=50000
|
| 16 |
+
# Limit PageRank computation to top N nodes (default: 10000, set to 0 to disable limit)
|
| 17 |
+
# Reduce this for very large graphs to save memory (e.g., 5000)
|
| 18 |
+
PAGERANK_LIMIT=10000
|
| 19 |
+
|
| 20 |
+
# LLM Explanations (Optional)
|
| 21 |
+
GROQ_API_KEY=your-groq-api-key-here
|
| 22 |
+
|
| 23 |
+
# API Authentication (Optional - add if implementing auth)
|
| 24 |
+
# API_KEY=your-api-key-here
|
| 25 |
+
|
feature_engineering.py
ADDED
|
@@ -0,0 +1,166 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Feature Engineering Module for Fraud Detection
|
| 3 |
+
Defines FraudFeatureEngineer class for model pipeline compatibility
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
import pandas as pd
|
| 7 |
+
import numpy as np
|
| 8 |
+
from sklearn.base import BaseEstimator, TransformerMixin
|
| 9 |
+
from typing import Optional
|
| 10 |
+
import networkx as nx
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class FraudFeatureEngineer(BaseEstimator, TransformerMixin):
|
| 14 |
+
"""
|
| 15 |
+
Vectorized, deterministic feature transformer.
|
| 16 |
+
- Builds weighted directed graph (aggregated by (origin,dest) counts)
|
| 17 |
+
- Creates frequency, ratio, log and graph features
|
| 18 |
+
"""
|
| 19 |
+
|
| 20 |
+
def __init__(self, pagerank_limit=None):
|
| 21 |
+
"""
|
| 22 |
+
Initialize the feature engineer
|
| 23 |
+
|
| 24 |
+
Args:
|
| 25 |
+
pagerank_limit: Optional limit on number of nodes for PageRank computation
|
| 26 |
+
"""
|
| 27 |
+
self.pagerank_limit = pagerank_limit
|
| 28 |
+
self.stats = {}
|
| 29 |
+
self.graph_meta = {}
|
| 30 |
+
self.type_map = {'TRANSFER': 0, 'CASH_OUT': 1}
|
| 31 |
+
self.global_mean = 0.0
|
| 32 |
+
self.global_median = 0.0
|
| 33 |
+
|
| 34 |
+
def fit(self, X, y=None):
|
| 35 |
+
"""
|
| 36 |
+
Fit the feature engineer on training/test data (memory-optimized)
|
| 37 |
+
|
| 38 |
+
Args:
|
| 39 |
+
X: Input DataFrame with raw transaction data
|
| 40 |
+
y: Optional target variable (not used)
|
| 41 |
+
|
| 42 |
+
Returns:
|
| 43 |
+
self
|
| 44 |
+
"""
|
| 45 |
+
# Memory optimization: work with views where possible, avoid unnecessary copies
|
| 46 |
+
# Only copy if we need to modify (sorting)
|
| 47 |
+
if 'step' in X.columns:
|
| 48 |
+
# Sort in-place if possible, or use a view
|
| 49 |
+
X_sorted = X.sort_values('step', kind='mergesort') # Stable sort
|
| 50 |
+
else:
|
| 51 |
+
X_sorted = X
|
| 52 |
+
|
| 53 |
+
# Basic global stats (compute once, reuse)
|
| 54 |
+
self.global_mean = float(X_sorted['amount'].mean())
|
| 55 |
+
self.global_median = float(X_sorted['amount'].median())
|
| 56 |
+
|
| 57 |
+
# Frequency & mean - use memory-efficient operations
|
| 58 |
+
self.stats['orig_counts'] = X_sorted['nameOrig'].value_counts().to_dict()
|
| 59 |
+
self.stats['dest_counts'] = X_sorted['nameDest'].value_counts().to_dict()
|
| 60 |
+
|
| 61 |
+
# Groupby operations - compute once, store as dicts
|
| 62 |
+
orig_groups = X_sorted.groupby('nameOrig')['amount']
|
| 63 |
+
self.stats['orig_mean_amt'] = orig_groups.mean().to_dict()
|
| 64 |
+
self.stats['orig_median_amt'] = orig_groups.median().to_dict()
|
| 65 |
+
self.stats['orig_log_median_amt'] = orig_groups.apply(lambda s: np.log1p(s).median()).to_dict()
|
| 66 |
+
|
| 67 |
+
if 'step' in X_sorted.columns:
|
| 68 |
+
self.stats['last_step'] = X_sorted.groupby('nameOrig')['step'].last().to_dict()
|
| 69 |
+
else:
|
| 70 |
+
self.stats['last_step'] = {}
|
| 71 |
+
|
| 72 |
+
# Weighted graph: count transactions per (origin,dest) - memory efficient
|
| 73 |
+
# Use value_counts on tuples for edge weights (more memory efficient)
|
| 74 |
+
edge_counts = X_sorted.groupby(['nameOrig', 'nameDest']).size()
|
| 75 |
+
|
| 76 |
+
# Build graph directly from edge counts (avoid intermediate DataFrame)
|
| 77 |
+
G = nx.DiGraph()
|
| 78 |
+
if len(edge_counts) > 0:
|
| 79 |
+
# Add edges directly from the Series (more memory efficient)
|
| 80 |
+
for (orig, dest), weight in edge_counts.items():
|
| 81 |
+
G.add_edge(orig, dest, weight=float(weight))
|
| 82 |
+
|
| 83 |
+
# Store degree dictionaries (memory efficient)
|
| 84 |
+
self.graph_meta['in_degree'] = dict(G.in_degree(weight='weight'))
|
| 85 |
+
self.graph_meta['out_degree'] = dict(G.out_degree(weight='weight'))
|
| 86 |
+
|
| 87 |
+
# Pagerank: limit nodes if requested to save time/memory
|
| 88 |
+
# Default to limiting if graph is large
|
| 89 |
+
default_pagerank_limit = 10000 # Limit to top 10k nodes by default
|
| 90 |
+
try:
|
| 91 |
+
if G.number_of_nodes() == 0:
|
| 92 |
+
self.graph_meta['pagerank'] = {}
|
| 93 |
+
else:
|
| 94 |
+
# Use pagerank_limit if set, otherwise use default limit for large graphs
|
| 95 |
+
effective_limit = self.pagerank_limit if self.pagerank_limit else default_pagerank_limit
|
| 96 |
+
|
| 97 |
+
if effective_limit < G.number_of_nodes():
|
| 98 |
+
# Only compute PageRank on top nodes by degree
|
| 99 |
+
top_nodes = sorted(G.degree(weight='weight'), key=lambda x: x[1], reverse=True)[:effective_limit]
|
| 100 |
+
keep = set(n for n, _ in top_nodes)
|
| 101 |
+
sub = G.subgraph(keep).copy()
|
| 102 |
+
self.graph_meta['pagerank'] = nx.pagerank(sub, alpha=0.85, weight='weight', tol=1e-4)
|
| 103 |
+
print(f"📊 PageRank computed on {len(keep):,} top nodes (graph has {G.number_of_nodes():,} total nodes)")
|
| 104 |
+
else:
|
| 105 |
+
self.graph_meta['pagerank'] = nx.pagerank(G, alpha=0.85, weight='weight', tol=1e-4)
|
| 106 |
+
except Exception as e:
|
| 107 |
+
# Pagerank failure should not break training
|
| 108 |
+
print(f"⚠️ PageRank computation failed: {str(e)}, using empty pagerank")
|
| 109 |
+
self.graph_meta['pagerank'] = {}
|
| 110 |
+
|
| 111 |
+
return self
|
| 112 |
+
|
| 113 |
+
def transform(self, X):
|
| 114 |
+
"""
|
| 115 |
+
Transform input data by engineering features
|
| 116 |
+
|
| 117 |
+
Args:
|
| 118 |
+
X: Input DataFrame with raw transaction data
|
| 119 |
+
|
| 120 |
+
Returns:
|
| 121 |
+
DataFrame with engineered features matching the training feature set
|
| 122 |
+
"""
|
| 123 |
+
X = X.copy()
|
| 124 |
+
|
| 125 |
+
# Time features
|
| 126 |
+
X['hour'] = X['step'] % 24 if 'step' in X.columns else 0
|
| 127 |
+
|
| 128 |
+
# Frequency mapping
|
| 129 |
+
X['orig_txn_count'] = X['nameOrig'].map(self.stats.get('orig_counts', {})).fillna(0).astype(int)
|
| 130 |
+
X['dest_txn_count'] = X['nameDest'].map(self.stats.get('dest_counts', {})).fillna(0).astype(int)
|
| 131 |
+
|
| 132 |
+
# Ratio features
|
| 133 |
+
user_mean = X['nameOrig'].map(self.stats.get('orig_mean_amt', {})).fillna(self.global_mean)
|
| 134 |
+
X['amt_ratio_to_user_mean'] = X['amount'] / (user_mean + 1.0)
|
| 135 |
+
X['amount_log1p'] = np.log1p(X['amount'])
|
| 136 |
+
X['amount_over_oldBalanceOrig'] = X['amount'] / (X['oldBalanceOrig'].replace(0, np.nan).fillna(1.0))
|
| 137 |
+
|
| 138 |
+
user_median = X['nameOrig'].map(self.stats.get('orig_median_amt', {})).fillna(self.global_median)
|
| 139 |
+
# Apply fallback to global median for users with too few transactions
|
| 140 |
+
MIN_TXNS = 3
|
| 141 |
+
user_median = np.where(X['orig_txn_count'] >= MIN_TXNS, user_median, self.global_median)
|
| 142 |
+
X['amt_ratio_to_user_median'] = (X['amount'] / (user_median + 1.0))
|
| 143 |
+
|
| 144 |
+
user_log_median = X['nameOrig'].map(self.stats.get('orig_log_median_amt', {})).fillna(np.log1p(self.global_median))
|
| 145 |
+
X['amt_log_ratio_to_user_median'] = (np.log1p(X['amount']) / (user_log_median + 1e-6))
|
| 146 |
+
|
| 147 |
+
# Graph features
|
| 148 |
+
X['in_degree'] = X['nameDest'].map(self.graph_meta.get('in_degree', {})).fillna(0).astype(float)
|
| 149 |
+
X['out_degree'] = X['nameOrig'].map(self.graph_meta.get('out_degree', {})).fillna(0).astype(float)
|
| 150 |
+
X['network_trust'] = X['nameOrig'].map(self.graph_meta.get('pagerank', {})).fillna(0.0).astype(float)
|
| 151 |
+
|
| 152 |
+
# New/novelty flags
|
| 153 |
+
X['is_new_origin'] = (X['orig_txn_count'] == 0).astype(int)
|
| 154 |
+
X['is_new_dest'] = (X['dest_txn_count'] == 0).astype(int)
|
| 155 |
+
|
| 156 |
+
# Type encoding (fast & vectorized)
|
| 157 |
+
X['type_encoded'] = X['type'].map(self.type_map).fillna(-1).astype(int)
|
| 158 |
+
|
| 159 |
+
# Drop identifiers and non-numeric columns
|
| 160 |
+
for c in ['nameOrig', 'nameDest', 'type', 'isFlaggedFraud']:
|
| 161 |
+
if c in X.columns:
|
| 162 |
+
X.drop(columns=c, inplace=True)
|
| 163 |
+
|
| 164 |
+
# Return numeric-only DataFrame expected by XGBoost & SHAP
|
| 165 |
+
return X.select_dtypes(include=[np.number])
|
| 166 |
+
|
inference.py
ADDED
|
@@ -0,0 +1,590 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Inference Module for Fraud Detection Pipeline
|
| 3 |
+
Adapted for FastAPI microservice
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
import os
|
| 7 |
+
import gc
|
| 8 |
+
import numpy as np
|
| 9 |
+
import pandas as pd
|
| 10 |
+
import joblib
|
| 11 |
+
import warnings
|
| 12 |
+
from typing import Dict, Optional, Tuple
|
| 13 |
+
|
| 14 |
+
# Suppress warnings
|
| 15 |
+
warnings.filterwarnings('ignore', message='.*is_sparse.*', category=FutureWarning)
|
| 16 |
+
warnings.filterwarnings('ignore', message='is_sparse is deprecated')
|
| 17 |
+
|
| 18 |
+
# Core imports
|
| 19 |
+
try:
|
| 20 |
+
from feature_engineering import FraudFeatureEngineer
|
| 21 |
+
except ImportError:
|
| 22 |
+
from sklearn.base import BaseEstimator, TransformerMixin
|
| 23 |
+
|
| 24 |
+
class FraudFeatureEngineer(BaseEstimator, TransformerMixin):
|
| 25 |
+
"""Fallback feature engineer if import fails"""
|
| 26 |
+
def fit(self, X, y=None):
|
| 27 |
+
return self
|
| 28 |
+
def transform(self, X):
|
| 29 |
+
return X
|
| 30 |
+
|
| 31 |
+
# Optional imports
|
| 32 |
+
try:
|
| 33 |
+
import shap
|
| 34 |
+
SHAP_AVAILABLE = True
|
| 35 |
+
except ImportError:
|
| 36 |
+
SHAP_AVAILABLE = False
|
| 37 |
+
|
| 38 |
+
try:
|
| 39 |
+
from groq import Groq
|
| 40 |
+
GROQ_AVAILABLE = True
|
| 41 |
+
except ImportError:
|
| 42 |
+
GROQ_AVAILABLE = False
|
| 43 |
+
|
| 44 |
+
try:
|
| 45 |
+
import xgboost as xgb
|
| 46 |
+
XGBOOST_AVAILABLE = True
|
| 47 |
+
except ImportError:
|
| 48 |
+
XGBOOST_AVAILABLE = False
|
| 49 |
+
|
| 50 |
+
# Load environment variables
|
| 51 |
+
try:
|
| 52 |
+
from dotenv import load_dotenv
|
| 53 |
+
load_dotenv()
|
| 54 |
+
except ImportError:
|
| 55 |
+
pass
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
class FraudInference:
|
| 59 |
+
"""
|
| 60 |
+
Inference class for fraud detection pipeline with explainability
|
| 61 |
+
Uses fitted feature engineer and XGBoost model separately
|
| 62 |
+
"""
|
| 63 |
+
|
| 64 |
+
def __init__(
|
| 65 |
+
self,
|
| 66 |
+
model_path: str,
|
| 67 |
+
test_dataset_path: Optional[str] = None,
|
| 68 |
+
threshold: float = 0.0793,
|
| 69 |
+
groq_api_key: Optional[str] = None,
|
| 70 |
+
pagerank_limit: Optional[int] = None
|
| 71 |
+
):
|
| 72 |
+
"""
|
| 73 |
+
Initialize inference engine
|
| 74 |
+
|
| 75 |
+
Args:
|
| 76 |
+
model_path: Path to the saved XGBoost model pkl file
|
| 77 |
+
test_dataset_path: Path to test dataset CSV for fitting feature engineer
|
| 78 |
+
threshold: Decision threshold for fraud classification
|
| 79 |
+
groq_api_key: Optional Groq API key for LLM explanations
|
| 80 |
+
pagerank_limit: Optional limit on nodes for PageRank computation
|
| 81 |
+
"""
|
| 82 |
+
self.model_path = model_path
|
| 83 |
+
self.test_dataset_path = test_dataset_path
|
| 84 |
+
self.threshold = threshold
|
| 85 |
+
self.groq_api_key = groq_api_key
|
| 86 |
+
self.pagerank_limit = pagerank_limit
|
| 87 |
+
self.model = None
|
| 88 |
+
self.feature_engineer = None
|
| 89 |
+
self.shap_background = None
|
| 90 |
+
self.shap_explainer = None
|
| 91 |
+
|
| 92 |
+
# Load model and fit feature engineer
|
| 93 |
+
self.load_model()
|
| 94 |
+
self.fit_feature_engineer()
|
| 95 |
+
|
| 96 |
+
def load_model(self):
|
| 97 |
+
"""Load the trained XGBoost model"""
|
| 98 |
+
try:
|
| 99 |
+
if not os.path.exists(self.model_path):
|
| 100 |
+
raise FileNotFoundError(f"Model file not found: {self.model_path}")
|
| 101 |
+
|
| 102 |
+
# Load model - could be XGBoost directly or pipeline with XGBoost
|
| 103 |
+
loaded_obj = joblib.load(self.model_path)
|
| 104 |
+
|
| 105 |
+
# Check if it's a pipeline or just XGBoost
|
| 106 |
+
if hasattr(loaded_obj, 'named_steps') and 'clf' in loaded_obj.named_steps:
|
| 107 |
+
# It's a pipeline, extract the classifier
|
| 108 |
+
self.model = loaded_obj.named_steps['clf']
|
| 109 |
+
print(f"✅ Model loaded from pipeline at {self.model_path}")
|
| 110 |
+
elif XGBOOST_AVAILABLE and isinstance(loaded_obj, xgb.XGBClassifier):
|
| 111 |
+
# It's XGBoost directly
|
| 112 |
+
self.model = loaded_obj
|
| 113 |
+
print(f"✅ XGBoost model loaded successfully from {self.model_path}")
|
| 114 |
+
else:
|
| 115 |
+
# Try to use it as-is (might be XGBoost wrapped)
|
| 116 |
+
self.model = loaded_obj
|
| 117 |
+
print(f"✅ Model loaded from {self.model_path} (assuming XGBoost)")
|
| 118 |
+
|
| 119 |
+
except Exception as e:
|
| 120 |
+
print(f"❌ Error loading model: {str(e)}")
|
| 121 |
+
raise
|
| 122 |
+
|
| 123 |
+
def fit_feature_engineer(self):
|
| 124 |
+
"""Load test dataset and fit feature engineer"""
|
| 125 |
+
try:
|
| 126 |
+
# Initialize feature engineer
|
| 127 |
+
from feature_engineering import FraudFeatureEngineer
|
| 128 |
+
self.feature_engineer = FraudFeatureEngineer(pagerank_limit=self.pagerank_limit)
|
| 129 |
+
|
| 130 |
+
# Find test dataset path - use script directory as base
|
| 131 |
+
script_dir = os.path.dirname(os.path.abspath(__file__))
|
| 132 |
+
|
| 133 |
+
# Find test dataset path
|
| 134 |
+
test_paths = []
|
| 135 |
+
if self.test_dataset_path:
|
| 136 |
+
test_paths.append(self.test_dataset_path)
|
| 137 |
+
# Also try as absolute path if relative
|
| 138 |
+
if not os.path.isabs(self.test_dataset_path):
|
| 139 |
+
test_paths.append(os.path.join(script_dir, self.test_dataset_path))
|
| 140 |
+
|
| 141 |
+
# Try common locations - prioritize dataset folder in ml-api
|
| 142 |
+
test_paths.extend([
|
| 143 |
+
# Compressed versions (preferred for size)
|
| 144 |
+
os.path.join(script_dir, "dataset", "test_dataset.csv.gz"),
|
| 145 |
+
"dataset/test_dataset.csv.gz",
|
| 146 |
+
"../dataset/test_dataset.csv.gz",
|
| 147 |
+
"/app/dataset/test_dataset.csv.gz",
|
| 148 |
+
# Uncompressed versions
|
| 149 |
+
os.path.join(script_dir, "dataset", "test_dataset.csv"), # ml-api/dataset/test_dataset.csv
|
| 150 |
+
os.path.join(script_dir, "dataset", "test_dataset_woIDX.csv"), # ml-api/dataset/test_dataset_woIDX.csv
|
| 151 |
+
"dataset/test_dataset.csv", # Relative to current working directory
|
| 152 |
+
"dataset/test_dataset_woIDX.csv",
|
| 153 |
+
"../dataset/test_dataset.csv", # Fallback
|
| 154 |
+
"../dataset/test_dataset_woIDX.csv",
|
| 155 |
+
"/app/dataset/test_dataset.csv", # For Docker deployment
|
| 156 |
+
"/app/dataset/test_dataset_woIDX.csv",
|
| 157 |
+
# Legacy paths for backward compatibility
|
| 158 |
+
"assets/test_dataset.csv",
|
| 159 |
+
"../assets/test_dataset.csv",
|
| 160 |
+
"/app/assets/test_dataset.csv"
|
| 161 |
+
])
|
| 162 |
+
|
| 163 |
+
test_df = None
|
| 164 |
+
dataset_path = None
|
| 165 |
+
for path in test_paths:
|
| 166 |
+
if os.path.exists(path):
|
| 167 |
+
dataset_path = path
|
| 168 |
+
print(f"📊 Found test dataset at {path}")
|
| 169 |
+
break
|
| 170 |
+
|
| 171 |
+
if dataset_path is None:
|
| 172 |
+
raise FileNotFoundError(
|
| 173 |
+
f"Test dataset not found. Tried: {test_paths}\n"
|
| 174 |
+
"Please ensure test dataset CSV is available for fitting feature engineer."
|
| 175 |
+
)
|
| 176 |
+
|
| 177 |
+
# Memory-efficient loading: sample dataset for fitting to reduce RAM usage
|
| 178 |
+
# Read a sample of the dataset instead of the full file
|
| 179 |
+
max_rows_for_fitting = int(os.getenv("MAX_FIT_ROWS", "50000")) # Default 50k rows
|
| 180 |
+
print(f"💾 Loading sample of dataset (max {max_rows_for_fitting:,} rows) for memory efficiency...")
|
| 181 |
+
|
| 182 |
+
try:
|
| 183 |
+
# Use chunked reading with sampling for memory efficiency
|
| 184 |
+
# Read in chunks and sample from each chunk
|
| 185 |
+
chunk_size = 10000
|
| 186 |
+
chunks = []
|
| 187 |
+
total_read = 0
|
| 188 |
+
|
| 189 |
+
for chunk in pd.read_csv(dataset_path, chunksize=chunk_size):
|
| 190 |
+
# Sample from chunk if we're getting close to limit
|
| 191 |
+
if total_read + len(chunk) > max_rows_for_fitting:
|
| 192 |
+
remaining = max_rows_for_fitting - total_read
|
| 193 |
+
if remaining > 0:
|
| 194 |
+
chunk = chunk.head(remaining)
|
| 195 |
+
chunks.append(chunk)
|
| 196 |
+
break
|
| 197 |
+
chunks.append(chunk)
|
| 198 |
+
total_read += len(chunk)
|
| 199 |
+
if total_read >= max_rows_for_fitting:
|
| 200 |
+
break
|
| 201 |
+
|
| 202 |
+
# Combine chunks
|
| 203 |
+
if chunks:
|
| 204 |
+
test_df = pd.concat(chunks, ignore_index=True)
|
| 205 |
+
print(f"✅ Loaded {len(test_df):,} rows for feature engineering fitting (sampled from dataset)")
|
| 206 |
+
else:
|
| 207 |
+
# Fallback: load with direct limit
|
| 208 |
+
test_df = pd.read_csv(dataset_path, nrows=max_rows_for_fitting)
|
| 209 |
+
print(f"✅ Loaded {len(test_df):,} rows (direct read with limit)")
|
| 210 |
+
|
| 211 |
+
except Exception as e:
|
| 212 |
+
print(f"⚠️ Failed to load with chunking, trying direct read: {str(e)}")
|
| 213 |
+
# Fallback: load with limit
|
| 214 |
+
test_df = pd.read_csv(dataset_path, nrows=max_rows_for_fitting)
|
| 215 |
+
print(f"✅ Loaded {len(test_df):,} rows (fallback method)")
|
| 216 |
+
|
| 217 |
+
# Ensure required columns exist
|
| 218 |
+
required_cols = ['step', 'type', 'amount', 'nameOrig', 'oldBalanceOrig',
|
| 219 |
+
'newBalanceOrig', 'nameDest', 'oldBalanceDest', 'newBalanceDest']
|
| 220 |
+
missing_cols = [col for col in required_cols if col not in test_df.columns]
|
| 221 |
+
if missing_cols:
|
| 222 |
+
raise ValueError(f"Test dataset missing required columns: {missing_cols}")
|
| 223 |
+
|
| 224 |
+
# Add isFlaggedFraud if missing
|
| 225 |
+
if 'isFlaggedFraud' not in test_df.columns:
|
| 226 |
+
test_df['isFlaggedFraud'] = 0
|
| 227 |
+
|
| 228 |
+
# Fit feature engineer on sampled dataset
|
| 229 |
+
print("🔧 Fitting feature engineer on sampled dataset...")
|
| 230 |
+
self.feature_engineer.fit(test_df)
|
| 231 |
+
print("✅ Feature engineer fitted successfully")
|
| 232 |
+
|
| 233 |
+
# Clear the full dataset from memory
|
| 234 |
+
del test_df
|
| 235 |
+
gc.collect()
|
| 236 |
+
print("🧹 Cleared dataset from memory")
|
| 237 |
+
|
| 238 |
+
# Prepare SHAP background from a small sample (reload minimal data)
|
| 239 |
+
print("📊 Preparing SHAP background data (small sample)...")
|
| 240 |
+
shap_sample_size = min(100, max_rows_for_fitting) # Reduced from 200 to 100
|
| 241 |
+
# Reload just a tiny sample for SHAP background
|
| 242 |
+
shap_df = pd.read_csv(dataset_path, nrows=shap_sample_size * 2) # Get more to sample from
|
| 243 |
+
shap_sample = shap_df.sample(n=min(shap_sample_size, len(shap_df)), random_state=42)
|
| 244 |
+
# Ensure required columns
|
| 245 |
+
if 'isFlaggedFraud' not in shap_sample.columns:
|
| 246 |
+
shap_sample['isFlaggedFraud'] = 0
|
| 247 |
+
self.shap_background = self.feature_engineer.transform(shap_sample)
|
| 248 |
+
del shap_df, shap_sample
|
| 249 |
+
gc.collect()
|
| 250 |
+
print(f"✅ SHAP background prepared ({len(self.shap_background)} samples)")
|
| 251 |
+
|
| 252 |
+
# Initialize SHAP explainer
|
| 253 |
+
if SHAP_AVAILABLE and XGBOOST_AVAILABLE and isinstance(self.model, xgb.XGBClassifier):
|
| 254 |
+
self.shap_explainer = shap.TreeExplainer(self.model)
|
| 255 |
+
print("✅ SHAP explainer initialized")
|
| 256 |
+
else:
|
| 257 |
+
if not SHAP_AVAILABLE:
|
| 258 |
+
print("⚠️ SHAP explainer not initialized (SHAP library not available)")
|
| 259 |
+
else:
|
| 260 |
+
print("⚠️ SHAP explainer not initialized (XGBoost not available or model type unknown)")
|
| 261 |
+
|
| 262 |
+
except Exception as e:
|
| 263 |
+
print(f"❌ Error fitting feature engineer: {str(e)}")
|
| 264 |
+
raise
|
| 265 |
+
|
| 266 |
+
def predict(self, transaction_df: pd.DataFrame) -> Tuple[np.ndarray, np.ndarray]:
|
| 267 |
+
"""
|
| 268 |
+
Predict fraud probability for transactions
|
| 269 |
+
|
| 270 |
+
Args:
|
| 271 |
+
transaction_df: DataFrame with raw transaction data
|
| 272 |
+
|
| 273 |
+
Returns:
|
| 274 |
+
probabilities: Array of fraud probabilities
|
| 275 |
+
decisions: Array of binary decisions (0/1)
|
| 276 |
+
"""
|
| 277 |
+
if self.model is None:
|
| 278 |
+
raise ValueError("Model not loaded. Call load_model() first.")
|
| 279 |
+
|
| 280 |
+
if self.feature_engineer is None:
|
| 281 |
+
raise ValueError("Feature engineer not fitted. Call fit_feature_engineer() first.")
|
| 282 |
+
|
| 283 |
+
# Transform transaction using fitted feature engineer
|
| 284 |
+
X_transformed = self.feature_engineer.transform(transaction_df)
|
| 285 |
+
|
| 286 |
+
# Get probabilities from XGBoost model
|
| 287 |
+
probabilities = self.model.predict_proba(X_transformed)[:, 1]
|
| 288 |
+
|
| 289 |
+
# Make decisions based on threshold
|
| 290 |
+
decisions = (probabilities >= self.threshold).astype(int)
|
| 291 |
+
|
| 292 |
+
return probabilities, decisions
|
| 293 |
+
|
| 294 |
+
def explain_shap(self, transaction_df: pd.DataFrame, topk: int = 10) -> pd.DataFrame:
|
| 295 |
+
"""
|
| 296 |
+
Generate SHAP explanations for a transaction
|
| 297 |
+
|
| 298 |
+
Args:
|
| 299 |
+
transaction_df: Single transaction DataFrame (raw format)
|
| 300 |
+
topk: Number of top features to return
|
| 301 |
+
|
| 302 |
+
Returns:
|
| 303 |
+
DataFrame with feature contributions sorted by importance
|
| 304 |
+
"""
|
| 305 |
+
if not SHAP_AVAILABLE:
|
| 306 |
+
raise ValueError("SHAP library not available")
|
| 307 |
+
|
| 308 |
+
if self.model is None:
|
| 309 |
+
raise ValueError("Model not loaded")
|
| 310 |
+
|
| 311 |
+
if self.feature_engineer is None:
|
| 312 |
+
raise ValueError("Feature engineer not fitted")
|
| 313 |
+
|
| 314 |
+
# Transform transaction using fitted feature engineer
|
| 315 |
+
X_trans = self.feature_engineer.transform(transaction_df)
|
| 316 |
+
feature_names = X_trans.columns.tolist()
|
| 317 |
+
|
| 318 |
+
# Compute SHAP values
|
| 319 |
+
try:
|
| 320 |
+
if self.shap_explainer is not None:
|
| 321 |
+
shap_values = self._compute_shap_values(self.shap_explainer, X_trans)
|
| 322 |
+
elif XGBOOST_AVAILABLE and isinstance(self.model, xgb.XGBClassifier):
|
| 323 |
+
# Fallback: create explainer on the fly
|
| 324 |
+
explainer = shap.TreeExplainer(self.model)
|
| 325 |
+
shap_values = self._compute_shap_values(explainer, X_trans)
|
| 326 |
+
else:
|
| 327 |
+
explainer = shap.Explainer(self.model, X_trans.iloc[[0]], feature_names=feature_names)
|
| 328 |
+
shap_exp = explainer(X_trans)
|
| 329 |
+
shap_values = shap_exp.values[0] if shap_exp.values.ndim == 2 else shap_exp.values
|
| 330 |
+
except Exception as e:
|
| 331 |
+
print(f"⚠️ SHAP computation failed: {str(e)}")
|
| 332 |
+
shap_values = np.zeros(X_trans.shape[1])
|
| 333 |
+
|
| 334 |
+
# Ensure shap_values is 1D
|
| 335 |
+
if shap_values.ndim > 1:
|
| 336 |
+
shap_values = shap_values[0]
|
| 337 |
+
|
| 338 |
+
# Build feature contribution DataFrame
|
| 339 |
+
feat_df = pd.DataFrame({
|
| 340 |
+
'feature': feature_names,
|
| 341 |
+
'value': X_trans.iloc[0].values,
|
| 342 |
+
'shap_abs': np.abs(shap_values),
|
| 343 |
+
'shap': shap_values
|
| 344 |
+
})
|
| 345 |
+
|
| 346 |
+
# Sort by absolute SHAP value
|
| 347 |
+
feat_df = feat_df.sort_values('shap_abs', ascending=False).reset_index(drop=True)
|
| 348 |
+
|
| 349 |
+
return feat_df.head(topk)
|
| 350 |
+
|
| 351 |
+
def _compute_shap_values(self, explainer, X_trans: pd.DataFrame) -> np.ndarray:
|
| 352 |
+
"""Helper method to compute SHAP values"""
|
| 353 |
+
if isinstance(explainer, shap.TreeExplainer):
|
| 354 |
+
shap_values = explainer.shap_values(X_trans)
|
| 355 |
+
if isinstance(shap_values, list):
|
| 356 |
+
shap_values = shap_values[1] # Get positive class values
|
| 357 |
+
else:
|
| 358 |
+
shap_exp = explainer(X_trans)
|
| 359 |
+
shap_values = shap_exp.values[0] if shap_exp.values.ndim == 2 else shap_exp.values
|
| 360 |
+
return shap_values
|
| 361 |
+
|
| 362 |
+
def explain_llm(self, probability: float, shap_table: pd.DataFrame, transaction_df: Optional[pd.DataFrame] = None, topk: int = 6, language: str = 'en') -> Optional[str]:
|
| 363 |
+
"""
|
| 364 |
+
Generate human-readable explanation using Groq LLM
|
| 365 |
+
|
| 366 |
+
Args:
|
| 367 |
+
probability: Fraud probability
|
| 368 |
+
shap_table: DataFrame with SHAP contributions
|
| 369 |
+
topk: Number of top features to include
|
| 370 |
+
language: Language code ('en' for English, 'bn' for Bangla)
|
| 371 |
+
|
| 372 |
+
Returns:
|
| 373 |
+
LLM-generated explanation text or None
|
| 374 |
+
"""
|
| 375 |
+
if not GROQ_AVAILABLE:
|
| 376 |
+
return None
|
| 377 |
+
|
| 378 |
+
if self.groq_api_key is None:
|
| 379 |
+
self.groq_api_key = os.getenv('GROQ_API_KEY')
|
| 380 |
+
|
| 381 |
+
if self.groq_api_key is None:
|
| 382 |
+
return None
|
| 383 |
+
|
| 384 |
+
try:
|
| 385 |
+
client = Groq(api_key=self.groq_api_key)
|
| 386 |
+
|
| 387 |
+
# Determine if fraud is detected
|
| 388 |
+
is_fraud = probability >= self.threshold
|
| 389 |
+
decision = "block" if is_fraud else ("warn" if probability >= self.threshold * 0.5 else "pass")
|
| 390 |
+
|
| 391 |
+
# Get transaction details if available
|
| 392 |
+
amount = None
|
| 393 |
+
tx_type = None
|
| 394 |
+
old_balance_orig = None
|
| 395 |
+
new_balance_orig = None
|
| 396 |
+
old_balance_dest = None
|
| 397 |
+
new_balance_dest = None
|
| 398 |
+
|
| 399 |
+
if transaction_df is not None and len(transaction_df) > 0:
|
| 400 |
+
row = transaction_df.iloc[0]
|
| 401 |
+
amount = row.get('amount', None)
|
| 402 |
+
tx_type = row.get('type', None)
|
| 403 |
+
old_balance_orig = row.get('oldBalanceOrig', None)
|
| 404 |
+
new_balance_orig = row.get('newBalanceOrig', None)
|
| 405 |
+
old_balance_dest = row.get('oldBalanceDest', None)
|
| 406 |
+
new_balance_dest = row.get('newBalanceDest', None)
|
| 407 |
+
|
| 408 |
+
if language == 'bn':
|
| 409 |
+
system_prompt = (
|
| 410 |
+
"আপনি একজন ব্যবহারকারী-বান্ধব মোবাইল ব্যাংকিং জালিয়াতি সতর্কতা সহায়ক। "
|
| 411 |
+
"আপনার কাজ হল সাধারণ ব্যবহারকারীদের জন্য সহজ ভাষায় ব্যাখ্যা করা, কোন লেনদেন কেন নিরাপদ বা ঝুঁকিপূর্ণ। "
|
| 412 |
+
"কোনও প্রযুক্তিগত শব্দ (যেমন SHAP, বৈশিষ্ট্য মান,technical detail, values ইত্যাদি) ব্যবহার করবেন না। "
|
| 413 |
+
"পরিবর্তে, ব্যবহারকারীকে বলুন: "
|
| 414 |
+
"- এই লেনদেনে কোন লাল সংকেত আছে কিনা "
|
| 415 |
+
"- তারা কী সতর্ক থাকতে হবে "
|
| 416 |
+
"- কেন এই লেনদেন নিরাপদ বা ঝুঁকিপূর্ণ "
|
| 417 |
+
"- যদি ফ্রড সনাক্ত হয়, তাহলে কেন এটি ফ্রড হতে পারে "
|
| 418 |
+
"- তারা কী করতে পারে বা এড়াতে পারে "
|
| 419 |
+
"ব্যাখ্যাটি সহজ, বন্ধুত্বপূর্ণ এবং ব্যবহারকারীর জন্য কার্যকর হতে হবে। "
|
| 420 |
+
"সমস্ত উত্তর বাংলায় লিখুন।"
|
| 421 |
+
)
|
| 422 |
+
|
| 423 |
+
tx_info = ""
|
| 424 |
+
if amount is not None:
|
| 425 |
+
tx_info += f"- লেনদেনের পরিমাণ: ৳ {amount:,.2f}\n"
|
| 426 |
+
if tx_type is not None:
|
| 427 |
+
tx_type_bn = "ক্যাশ আউট" if tx_type == "CASH_OUT" else "স্থানান্তর" if tx_type == "TRANSFER" else tx_type
|
| 428 |
+
tx_info += f"- লেনদেনের ধরন: {tx_type_bn}\n"
|
| 429 |
+
if old_balance_orig is not None and new_balance_orig is not None:
|
| 430 |
+
balance_change = new_balance_orig - old_balance_orig
|
| 431 |
+
tx_info += f"- প্রেরকের ব্যালেন্স পরিবর্তন: ৳ {balance_change:,.2f}\n"
|
| 432 |
+
if old_balance_dest is not None and new_balance_dest is not None:
|
| 433 |
+
balance_change = new_balance_dest - old_balance_dest
|
| 434 |
+
tx_info += f"- গ্রহীতার ব্যালেন্স পরিবর্তন: ৳ {balance_change:,.2f}\n"
|
| 435 |
+
|
| 436 |
+
user_prompt = (
|
| 437 |
+
f"লেনদেনের ফ্রড সম্ভাবনা: {probability*100:.2f}%\n"
|
| 438 |
+
f"সিদ্ধান্ত: {'ফ্রড সনাক্ত হয়েছে - লেনদেন ব্লক করা হয়েছে' if is_fraud else ('সতর্কতা - ম্যানুয়াল পর্যালোচনা প্রয়োজন' if decision == 'warn' else 'লেনদেন নিরাপদ - অনুমোদন করা যেতে পারে')}\n"
|
| 439 |
+
f"লেনদেনের তথ্য:\n{tx_info}"
|
| 440 |
+
f"\nএকটি সহজ, ব্যবহারকারী-বান্ধব ব্যাখ্যা লিখুন যা ব্যবহারকারীকে বুঝতে সাহায্য করবে কেন এই লেনদেন নিরাপদ বা ঝুঁকিপূর্ণ, এবং তাদের কী জানা উচিত বা সতর্ক থাকতে হবে।"
|
| 441 |
+
)
|
| 442 |
+
else:
|
| 443 |
+
system_prompt = (
|
| 444 |
+
"You are a user-friendly mobile banking fraud alert assistant. "
|
| 445 |
+
"Your job is to explain in simple language why a transaction is safe or risky for regular users. "
|
| 446 |
+
"Do NOT use any technical terms (like SHAP, feature values, technical detail, values etc.). "
|
| 447 |
+
"Instead, tell the user: "
|
| 448 |
+
"- What red flags exist in this transaction (if any) "
|
| 449 |
+
"- What they should be aware of or cautious about "
|
| 450 |
+
"- Why this transaction is safe or risky "
|
| 451 |
+
"- If fraud is detected, explain why it might be fraud "
|
| 452 |
+
"- What they can do or should avoid "
|
| 453 |
+
"The explanation should be simple, friendly, and actionable for the user. "
|
| 454 |
+
"Focus on what matters to them, not technical details."
|
| 455 |
+
)
|
| 456 |
+
|
| 457 |
+
tx_info = ""
|
| 458 |
+
if amount is not None:
|
| 459 |
+
tx_info += f"- Transaction amount: ৳ {amount:,.2f}\n"
|
| 460 |
+
if tx_type is not None:
|
| 461 |
+
tx_info += f"- Transaction type: {tx_type}\n"
|
| 462 |
+
if old_balance_orig is not None and new_balance_orig is not None:
|
| 463 |
+
balance_change = new_balance_orig - old_balance_orig
|
| 464 |
+
tx_info += f"- Sender balance change: ৳ {balance_change:,.2f}\n"
|
| 465 |
+
if old_balance_dest is not None and new_balance_dest is not None:
|
| 466 |
+
balance_change = new_balance_dest - old_balance_dest
|
| 467 |
+
tx_info += f"- Receiver balance change: ৳ {balance_change:,.2f}\n"
|
| 468 |
+
|
| 469 |
+
user_prompt = (
|
| 470 |
+
f"Transaction fraud probability: {probability*100:.2f}%\n"
|
| 471 |
+
f"Decision: {'Fraud detected - Transaction blocked' if is_fraud else ('Warning - Manual review required' if decision == 'warn' else 'Transaction safe - Can be approved')}\n"
|
| 472 |
+
f"Transaction details:\n{tx_info}"
|
| 473 |
+
f"\nWrite a simple, user-friendly explanation that helps the user understand why this transaction is safe or risky, and what they should know or be cautious about."
|
| 474 |
+
)
|
| 475 |
+
|
| 476 |
+
chat_completion = client.chat.completions.create(
|
| 477 |
+
messages=[
|
| 478 |
+
{"role": "system", "content": system_prompt},
|
| 479 |
+
{"role": "user", "content": user_prompt}
|
| 480 |
+
],
|
| 481 |
+
model="llama-3.1-8b-instant",
|
| 482 |
+
temperature=0.3,
|
| 483 |
+
max_tokens=500
|
| 484 |
+
)
|
| 485 |
+
|
| 486 |
+
explanation = chat_completion.choices[0].message.content.strip()
|
| 487 |
+
return explanation
|
| 488 |
+
|
| 489 |
+
except Exception as e:
|
| 490 |
+
error_msg = f"(LLM generation failed: {str(e)})"
|
| 491 |
+
if language == 'bn':
|
| 492 |
+
error_msg = f"(LLM তৈরি করতে ব্যর্থ: {str(e)})"
|
| 493 |
+
return error_msg
|
| 494 |
+
|
| 495 |
+
def predict_and_explain(
|
| 496 |
+
self,
|
| 497 |
+
transaction_df: pd.DataFrame,
|
| 498 |
+
shap_background: Optional[pd.DataFrame] = None,
|
| 499 |
+
topk: int = 6,
|
| 500 |
+
use_llm: bool = True,
|
| 501 |
+
language: str = 'en'
|
| 502 |
+
) -> Dict:
|
| 503 |
+
"""
|
| 504 |
+
Complete prediction and explanation pipeline
|
| 505 |
+
|
| 506 |
+
Args:
|
| 507 |
+
transaction_df: Raw transaction DataFrame
|
| 508 |
+
shap_background: Optional background data for SHAP
|
| 509 |
+
topk: Number of top features to explain
|
| 510 |
+
use_llm: Whether to generate LLM explanation
|
| 511 |
+
language: Language code ('en' for English, 'bn' for Bangla)
|
| 512 |
+
|
| 513 |
+
Returns:
|
| 514 |
+
Dictionary with:
|
| 515 |
+
- probabilities: Fraud probabilities
|
| 516 |
+
- decisions: Binary decisions
|
| 517 |
+
- shap_table: Feature contributions DataFrame
|
| 518 |
+
- llm_explanation: Optional LLM explanation text
|
| 519 |
+
"""
|
| 520 |
+
# Predict
|
| 521 |
+
probabilities, decisions = self.predict(transaction_df)
|
| 522 |
+
|
| 523 |
+
# Prepare SHAP background if needed
|
| 524 |
+
if shap_background is not None:
|
| 525 |
+
self.prepare_shap_background(shap_background)
|
| 526 |
+
|
| 527 |
+
# Generate SHAP explanations
|
| 528 |
+
shap_table = self.explain_shap(transaction_df, topk=topk)
|
| 529 |
+
|
| 530 |
+
# Generate LLM explanation if requested
|
| 531 |
+
llm_explanation = None
|
| 532 |
+
if use_llm and GROQ_AVAILABLE:
|
| 533 |
+
llm_explanation = self.explain_llm(probabilities[0], shap_table, transaction_df=transaction_df, topk=topk, language=language)
|
| 534 |
+
|
| 535 |
+
return {
|
| 536 |
+
'probabilities': probabilities,
|
| 537 |
+
'decisions': decisions,
|
| 538 |
+
'shap_table': shap_table,
|
| 539 |
+
'llm_explanation': llm_explanation
|
| 540 |
+
}
|
| 541 |
+
|
| 542 |
+
|
| 543 |
+
def load_inference_engine(
|
| 544 |
+
model_path: str = "Models/fraud_pipeline_final.pkl",
|
| 545 |
+
test_dataset_path: Optional[str] = None,
|
| 546 |
+
threshold: float = 0.0793,
|
| 547 |
+
groq_api_key: Optional[str] = None,
|
| 548 |
+
pagerank_limit: Optional[int] = None
|
| 549 |
+
) -> FraudInference:
|
| 550 |
+
"""
|
| 551 |
+
Convenience function to load inference engine
|
| 552 |
+
|
| 553 |
+
Args:
|
| 554 |
+
model_path: Path to model file
|
| 555 |
+
test_dataset_path: Path to test dataset CSV (optional, will search common locations)
|
| 556 |
+
threshold: Decision threshold
|
| 557 |
+
groq_api_key: Optional Groq API key
|
| 558 |
+
pagerank_limit: Optional limit on nodes for PageRank computation
|
| 559 |
+
|
| 560 |
+
Returns:
|
| 561 |
+
Initialized FraudInference instance
|
| 562 |
+
"""
|
| 563 |
+
possible_paths = [
|
| 564 |
+
model_path,
|
| 565 |
+
f"/app/{model_path}",
|
| 566 |
+
f"/app/Models/fraud_pipeline_final.pkl",
|
| 567 |
+
"Models/fraud_pipeline_final.pkl",
|
| 568 |
+
"../Models/fraud_pipeline_final.pkl"
|
| 569 |
+
]
|
| 570 |
+
|
| 571 |
+
actual_path = None
|
| 572 |
+
for path in possible_paths:
|
| 573 |
+
if os.path.exists(path):
|
| 574 |
+
actual_path = path
|
| 575 |
+
break
|
| 576 |
+
|
| 577 |
+
if actual_path is None:
|
| 578 |
+
raise FileNotFoundError(
|
| 579 |
+
f"Model file not found. Tried: {possible_paths}\n"
|
| 580 |
+
"Please ensure the model file is in one of these locations."
|
| 581 |
+
)
|
| 582 |
+
|
| 583 |
+
return FraudInference(
|
| 584 |
+
actual_path,
|
| 585 |
+
test_dataset_path=test_dataset_path,
|
| 586 |
+
threshold=threshold,
|
| 587 |
+
groq_api_key=groq_api_key,
|
| 588 |
+
pagerank_limit=pagerank_limit
|
| 589 |
+
)
|
| 590 |
+
|
main.py
ADDED
|
@@ -0,0 +1,530 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
CloverShield ML Inference API
|
| 3 |
+
FastAPI microservice for fraud detection predictions
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
import os
|
| 7 |
+
import sys
|
| 8 |
+
import time
|
| 9 |
+
import uuid
|
| 10 |
+
from typing import Optional, Dict, List
|
| 11 |
+
from datetime import datetime
|
| 12 |
+
|
| 13 |
+
import pandas as pd
|
| 14 |
+
import numpy as np
|
| 15 |
+
from fastapi import FastAPI, HTTPException, Header
|
| 16 |
+
from fastapi.middleware.cors import CORSMiddleware
|
| 17 |
+
from fastapi.responses import JSONResponse
|
| 18 |
+
from pydantic import BaseModel, Field, validator
|
| 19 |
+
import uvicorn
|
| 20 |
+
|
| 21 |
+
# Add current directory to path for imports
|
| 22 |
+
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
| 23 |
+
|
| 24 |
+
from inference import FraudInference, load_inference_engine
|
| 25 |
+
import warnings
|
| 26 |
+
|
| 27 |
+
# Suppress warnings
|
| 28 |
+
warnings.filterwarnings('ignore')
|
| 29 |
+
|
| 30 |
+
# ============================================================================
|
| 31 |
+
# PYDANTIC MODELS
|
| 32 |
+
# ============================================================================
|
| 33 |
+
|
| 34 |
+
class TransactionInput(BaseModel):
|
| 35 |
+
"""Transaction input model"""
|
| 36 |
+
step: Optional[int] = Field(default=1, ge=0, description="Time step")
|
| 37 |
+
type: str = Field(..., description="Transaction type")
|
| 38 |
+
amount: float = Field(..., gt=0, description="Transaction amount")
|
| 39 |
+
nameOrig: str = Field(..., description="Sender user ID")
|
| 40 |
+
oldBalanceOrig: float = Field(..., ge=0, description="Sender balance before")
|
| 41 |
+
newBalanceOrig: float = Field(..., ge=0, description="Sender balance after")
|
| 42 |
+
nameDest: str = Field(..., description="Receiver user ID")
|
| 43 |
+
oldBalanceDest: float = Field(..., ge=0, description="Receiver balance before")
|
| 44 |
+
newBalanceDest: float = Field(..., ge=0, description="Receiver balance after")
|
| 45 |
+
|
| 46 |
+
@validator('type')
|
| 47 |
+
def validate_type(cls, v):
|
| 48 |
+
allowed = ['CASH_OUT', 'TRANSFER', 'CASH_IN', 'PAYMENT', 'DEBIT']
|
| 49 |
+
if v not in allowed:
|
| 50 |
+
raise ValueError(f"Transaction type must be one of {allowed}")
|
| 51 |
+
return v
|
| 52 |
+
|
| 53 |
+
@validator('nameDest')
|
| 54 |
+
def validate_different_accounts(cls, v, values):
|
| 55 |
+
if 'nameOrig' in values and v == values['nameOrig']:
|
| 56 |
+
raise ValueError("Sender and receiver cannot be the same")
|
| 57 |
+
return v
|
| 58 |
+
|
| 59 |
+
class PredictionOptions(BaseModel):
|
| 60 |
+
"""Prediction options"""
|
| 61 |
+
include_shap: bool = Field(default=True, description="Include SHAP explanations")
|
| 62 |
+
include_llm_explanation: bool = Field(default=False, description="Include LLM explanation")
|
| 63 |
+
language: str = Field(default='en', description="Language for explanations")
|
| 64 |
+
topk: int = Field(default=10, ge=1, le=20, description="Number of top features")
|
| 65 |
+
|
| 66 |
+
@validator('language')
|
| 67 |
+
def validate_language(cls, v):
|
| 68 |
+
if v not in ['en', 'bn']:
|
| 69 |
+
raise ValueError("Language must be 'en' or 'bn'")
|
| 70 |
+
return v
|
| 71 |
+
|
| 72 |
+
class PredictRequest(BaseModel):
|
| 73 |
+
"""Request model for /predict endpoint"""
|
| 74 |
+
transaction: TransactionInput
|
| 75 |
+
options: Optional[PredictionOptions] = Field(default_factory=PredictionOptions)
|
| 76 |
+
|
| 77 |
+
class SHAPExplanation(BaseModel):
|
| 78 |
+
"""SHAP explanation model"""
|
| 79 |
+
feature: str
|
| 80 |
+
value: float
|
| 81 |
+
shap: float
|
| 82 |
+
shap_abs: float
|
| 83 |
+
rank: int
|
| 84 |
+
|
| 85 |
+
class PredictionResult(BaseModel):
|
| 86 |
+
"""Prediction result model"""
|
| 87 |
+
fraud_probability: float = Field(..., ge=0, le=1)
|
| 88 |
+
decision: str = Field(..., description="pass, warn, or block")
|
| 89 |
+
risk_level: str = Field(..., description="low, medium, or high")
|
| 90 |
+
confidence: float = Field(..., ge=0, le=1)
|
| 91 |
+
|
| 92 |
+
class PredictResponse(BaseModel):
|
| 93 |
+
"""Response model for /predict endpoint"""
|
| 94 |
+
transaction_id: str
|
| 95 |
+
prediction: PredictionResult
|
| 96 |
+
shap_explanations: Optional[List[SHAPExplanation]] = None
|
| 97 |
+
llm_explanation: Optional[Dict[str, str]] = None
|
| 98 |
+
processing_time_ms: int
|
| 99 |
+
model_version: str
|
| 100 |
+
timestamp: str
|
| 101 |
+
|
| 102 |
+
class BatchPredictRequest(BaseModel):
|
| 103 |
+
"""Request model for batch prediction"""
|
| 104 |
+
transactions: List[TransactionInput]
|
| 105 |
+
options: Optional[PredictionOptions] = Field(default_factory=PredictionOptions)
|
| 106 |
+
|
| 107 |
+
class BatchPredictResponse(BaseModel):
|
| 108 |
+
"""Response model for batch prediction"""
|
| 109 |
+
results: List[Dict]
|
| 110 |
+
processing_time_ms: int
|
| 111 |
+
total_transactions: int
|
| 112 |
+
|
| 113 |
+
# ============================================================================
|
| 114 |
+
# FASTAPI APP
|
| 115 |
+
# ============================================================================
|
| 116 |
+
|
| 117 |
+
app = FastAPI(
|
| 118 |
+
title="CloverShield ML Inference API",
|
| 119 |
+
description="Fraud detection ML service for CloverShield platform",
|
| 120 |
+
version="1.0.0",
|
| 121 |
+
docs_url="/docs",
|
| 122 |
+
redoc_url="/redoc"
|
| 123 |
+
)
|
| 124 |
+
|
| 125 |
+
# CORS middleware - configure allowed origins for production
|
| 126 |
+
app.add_middleware(
|
| 127 |
+
CORSMiddleware,
|
| 128 |
+
allow_origins=["*"], # TODO: Restrict to specific origins in production
|
| 129 |
+
allow_credentials=True,
|
| 130 |
+
allow_methods=["*"],
|
| 131 |
+
allow_headers=["*"],
|
| 132 |
+
)
|
| 133 |
+
|
| 134 |
+
# ============================================================================
|
| 135 |
+
# GLOBAL STATE
|
| 136 |
+
# ============================================================================
|
| 137 |
+
|
| 138 |
+
inference_engine: Optional[FraudInference] = None
|
| 139 |
+
model_loading_lock = False
|
| 140 |
+
MODEL_VERSION = "1.0.0"
|
| 141 |
+
MODEL_THRESHOLD = float(os.getenv("MODEL_THRESHOLD", "0.0793"))
|
| 142 |
+
|
| 143 |
+
# Risk thresholds (matching config.py)
|
| 144 |
+
RISK_THRESHOLDS = {
|
| 145 |
+
"pass": 0.30,
|
| 146 |
+
"warn": 0.30,
|
| 147 |
+
"block": 0.70
|
| 148 |
+
}
|
| 149 |
+
|
| 150 |
+
# ============================================================================
|
| 151 |
+
# HELPER FUNCTIONS
|
| 152 |
+
# ============================================================================
|
| 153 |
+
|
| 154 |
+
def load_model():
|
| 155 |
+
"""Load the ML model on startup"""
|
| 156 |
+
global inference_engine, model_loading_lock
|
| 157 |
+
|
| 158 |
+
# Prevent concurrent loading attempts
|
| 159 |
+
if model_loading_lock:
|
| 160 |
+
return
|
| 161 |
+
model_loading_lock = True
|
| 162 |
+
|
| 163 |
+
model_path = os.getenv("MODEL_PATH", "Models/fraud_pipeline_final.pkl")
|
| 164 |
+
test_dataset_path = os.getenv("TEST_DATASET_PATH", None)
|
| 165 |
+
groq_api_key = os.getenv("GROQ_API_KEY")
|
| 166 |
+
pagerank_limit = os.getenv("PAGERANK_LIMIT", None)
|
| 167 |
+
|
| 168 |
+
# Convert pagerank_limit to int if provided
|
| 169 |
+
if pagerank_limit:
|
| 170 |
+
try:
|
| 171 |
+
pagerank_limit = int(pagerank_limit)
|
| 172 |
+
except ValueError:
|
| 173 |
+
pagerank_limit = None
|
| 174 |
+
|
| 175 |
+
# Try multiple paths for model
|
| 176 |
+
possible_paths = [
|
| 177 |
+
model_path,
|
| 178 |
+
f"/app/{model_path}",
|
| 179 |
+
f"/app/Models/fraud_pipeline_final.pkl",
|
| 180 |
+
"Models/fraud_pipeline_final.pkl",
|
| 181 |
+
"../Models/fraud_pipeline_final.pkl"
|
| 182 |
+
]
|
| 183 |
+
|
| 184 |
+
actual_path = None
|
| 185 |
+
for path in possible_paths:
|
| 186 |
+
if os.path.exists(path):
|
| 187 |
+
actual_path = path
|
| 188 |
+
break
|
| 189 |
+
|
| 190 |
+
if actual_path is None:
|
| 191 |
+
raise FileNotFoundError(
|
| 192 |
+
f"Model file not found. Tried: {possible_paths}\n"
|
| 193 |
+
"Please ensure the model file is available."
|
| 194 |
+
)
|
| 195 |
+
|
| 196 |
+
try:
|
| 197 |
+
# Validate file exists and is readable
|
| 198 |
+
if not os.path.exists(actual_path):
|
| 199 |
+
raise FileNotFoundError(f"Model file not found at {actual_path}")
|
| 200 |
+
|
| 201 |
+
# Check file size (should be at least 1MB for a valid model)
|
| 202 |
+
file_size = os.path.getsize(actual_path) / (1024 * 1024)
|
| 203 |
+
if file_size < 1:
|
| 204 |
+
raise ValueError(
|
| 205 |
+
f"Model file is too small ({file_size:.1f} MB). "
|
| 206 |
+
f"Expected at least 1MB. The file may be corrupted or incomplete."
|
| 207 |
+
)
|
| 208 |
+
|
| 209 |
+
# Try to load the model with test dataset for feature engineering
|
| 210 |
+
inference_engine = load_inference_engine(
|
| 211 |
+
model_path=actual_path,
|
| 212 |
+
test_dataset_path=test_dataset_path,
|
| 213 |
+
threshold=MODEL_THRESHOLD,
|
| 214 |
+
groq_api_key=groq_api_key,
|
| 215 |
+
pagerank_limit=pagerank_limit
|
| 216 |
+
)
|
| 217 |
+
print(f"✅ Model loaded successfully from {actual_path} ({file_size:.1f} MB)")
|
| 218 |
+
except FileNotFoundError as e:
|
| 219 |
+
error_msg = f"❌ Model file not found: {str(e)}"
|
| 220 |
+
print(error_msg)
|
| 221 |
+
raise FileNotFoundError(error_msg) from e
|
| 222 |
+
except ValueError as e:
|
| 223 |
+
error_msg = f"❌ Invalid model file: {str(e)}"
|
| 224 |
+
print(error_msg)
|
| 225 |
+
raise ValueError(error_msg) from e
|
| 226 |
+
except Exception as e:
|
| 227 |
+
error_msg = f"❌ Error loading model: {str(e)}"
|
| 228 |
+
print(error_msg)
|
| 229 |
+
print(f" Model path: {actual_path}")
|
| 230 |
+
print(f" File exists: {os.path.exists(actual_path) if actual_path else False}")
|
| 231 |
+
if actual_path and os.path.exists(actual_path):
|
| 232 |
+
print(f" File size: {os.path.getsize(actual_path) / (1024 * 1024):.1f} MB")
|
| 233 |
+
raise Exception(error_msg) from e
|
| 234 |
+
finally:
|
| 235 |
+
model_loading_lock = False
|
| 236 |
+
|
| 237 |
+
def ensure_model_loaded():
|
| 238 |
+
"""Ensure model is loaded, try to load if not loaded (lazy loading for serverless)"""
|
| 239 |
+
global inference_engine, model_loading_lock
|
| 240 |
+
|
| 241 |
+
if inference_engine is not None:
|
| 242 |
+
return True
|
| 243 |
+
|
| 244 |
+
# Try to load if not already loading
|
| 245 |
+
if not model_loading_lock:
|
| 246 |
+
try:
|
| 247 |
+
print("🔄 Model not loaded, attempting to load now...")
|
| 248 |
+
load_model()
|
| 249 |
+
return inference_engine is not None
|
| 250 |
+
except Exception as e:
|
| 251 |
+
print(f"⚠️ Failed to load model on demand: {str(e)}")
|
| 252 |
+
return False
|
| 253 |
+
return False
|
| 254 |
+
|
| 255 |
+
def calculate_decision(probability: float) -> tuple[str, str]:
|
| 256 |
+
"""Calculate decision and risk level from probability"""
|
| 257 |
+
if probability >= RISK_THRESHOLDS['block']:
|
| 258 |
+
return 'block', 'high'
|
| 259 |
+
elif probability >= RISK_THRESHOLDS['warn']:
|
| 260 |
+
return 'warn', 'medium'
|
| 261 |
+
else:
|
| 262 |
+
return 'pass', 'low'
|
| 263 |
+
|
| 264 |
+
def calculate_confidence(probability: float) -> float:
|
| 265 |
+
"""Calculate confidence level based on probability"""
|
| 266 |
+
if probability < 0.1 or probability > 0.9:
|
| 267 |
+
return 0.9
|
| 268 |
+
elif probability < 0.2 or probability > 0.8:
|
| 269 |
+
return 0.75
|
| 270 |
+
elif probability < 0.3 or probability > 0.7:
|
| 271 |
+
return 0.6
|
| 272 |
+
else:
|
| 273 |
+
return 0.4
|
| 274 |
+
|
| 275 |
+
def transaction_to_dataframe(transaction: TransactionInput) -> pd.DataFrame:
|
| 276 |
+
"""Convert transaction input to DataFrame"""
|
| 277 |
+
data = {
|
| 278 |
+
'step': transaction.step or 1,
|
| 279 |
+
'type': transaction.type,
|
| 280 |
+
'amount': float(transaction.amount),
|
| 281 |
+
'nameOrig': transaction.nameOrig,
|
| 282 |
+
'oldBalanceOrig': float(transaction.oldBalanceOrig),
|
| 283 |
+
'newBalanceOrig': float(transaction.newBalanceOrig),
|
| 284 |
+
'nameDest': transaction.nameDest,
|
| 285 |
+
'oldBalanceDest': float(transaction.oldBalanceDest),
|
| 286 |
+
'newBalanceDest': float(transaction.newBalanceDest),
|
| 287 |
+
'isFlaggedFraud': 0
|
| 288 |
+
}
|
| 289 |
+
return pd.DataFrame([data])
|
| 290 |
+
|
| 291 |
+
# ============================================================================
|
| 292 |
+
# API ENDPOINTS
|
| 293 |
+
# ============================================================================
|
| 294 |
+
|
| 295 |
+
@app.on_event("startup")
|
| 296 |
+
async def startup_event():
|
| 297 |
+
"""Load model on startup"""
|
| 298 |
+
port = os.getenv("PORT", "8000")
|
| 299 |
+
print(f"🚀 Server starting on port {port}")
|
| 300 |
+
print("📦 Loading model...")
|
| 301 |
+
|
| 302 |
+
try:
|
| 303 |
+
load_model()
|
| 304 |
+
if inference_engine is not None:
|
| 305 |
+
print("✅ Model loaded successfully - API is ready!")
|
| 306 |
+
else:
|
| 307 |
+
print("⚠️ Warning: Model loading completed but inference_engine is None")
|
| 308 |
+
print("⚠️ API will attempt lazy loading on first request")
|
| 309 |
+
except Exception as e:
|
| 310 |
+
print(f"⚠️ Warning: Model not loaded on startup: {str(e)}")
|
| 311 |
+
print("⚠️ API will attempt lazy loading on first request")
|
| 312 |
+
print("⚠️ This is normal for serverless environments (e.g., Vercel)")
|
| 313 |
+
|
| 314 |
+
@app.get("/")
|
| 315 |
+
async def root():
|
| 316 |
+
"""Root endpoint"""
|
| 317 |
+
return {
|
| 318 |
+
"service": "CloverShield ML Inference API",
|
| 319 |
+
"version": MODEL_VERSION,
|
| 320 |
+
"status": "running",
|
| 321 |
+
"model_loaded": inference_engine is not None
|
| 322 |
+
}
|
| 323 |
+
|
| 324 |
+
@app.get("/health")
|
| 325 |
+
async def health_check():
|
| 326 |
+
"""Health check endpoint"""
|
| 327 |
+
# Lazy loading for serverless environments
|
| 328 |
+
if inference_engine is None:
|
| 329 |
+
ensure_model_loaded()
|
| 330 |
+
|
| 331 |
+
model_loaded = inference_engine is not None
|
| 332 |
+
shap_available = model_loaded and inference_engine.shap_explainer is not None
|
| 333 |
+
|
| 334 |
+
return {
|
| 335 |
+
"status": "healthy" if model_loaded else "degraded",
|
| 336 |
+
"model_loaded": model_loaded,
|
| 337 |
+
"model_version": MODEL_VERSION,
|
| 338 |
+
"shap_available": shap_available,
|
| 339 |
+
"llm_available": os.getenv("GROQ_API_KEY") is not None,
|
| 340 |
+
"message": "Model loaded and ready" if model_loaded else "Model is loading or unavailable"
|
| 341 |
+
}
|
| 342 |
+
|
| 343 |
+
@app.get("/model/info")
|
| 344 |
+
async def model_info():
|
| 345 |
+
"""Get model information"""
|
| 346 |
+
if inference_engine is None:
|
| 347 |
+
raise HTTPException(status_code=503, detail="Model not loaded")
|
| 348 |
+
|
| 349 |
+
return {
|
| 350 |
+
"model_version": MODEL_VERSION,
|
| 351 |
+
"model_type": "XGBoost",
|
| 352 |
+
"threshold": MODEL_THRESHOLD,
|
| 353 |
+
"features": [
|
| 354 |
+
{"name": "step", "type": "integer", "description": "Time step"},
|
| 355 |
+
{"name": "amount", "type": "float", "description": "Transaction amount"},
|
| 356 |
+
{"name": "oldBalanceOrig", "type": "float", "description": "Sender balance before"},
|
| 357 |
+
{"name": "newBalanceOrig", "type": "float", "description": "Sender balance after"},
|
| 358 |
+
{"name": "oldBalanceDest", "type": "float", "description": "Receiver balance before"},
|
| 359 |
+
{"name": "newBalanceDest", "type": "float", "description": "Receiver balance after"},
|
| 360 |
+
{"name": "type", "type": "string", "description": "Transaction type"},
|
| 361 |
+
]
|
| 362 |
+
}
|
| 363 |
+
|
| 364 |
+
@app.post("/predict", response_model=PredictResponse)
|
| 365 |
+
async def predict(request: PredictRequest):
|
| 366 |
+
"""
|
| 367 |
+
Predict fraud probability for a single transaction
|
| 368 |
+
|
| 369 |
+
Returns fraud probability, decision, risk level, and SHAP explanations
|
| 370 |
+
"""
|
| 371 |
+
# Lazy loading for serverless environments
|
| 372 |
+
if inference_engine is None:
|
| 373 |
+
if not ensure_model_loaded():
|
| 374 |
+
raise HTTPException(
|
| 375 |
+
status_code=503,
|
| 376 |
+
detail="Model not loaded. The model is still loading or failed to load. Please try again in a moment or check /health endpoint."
|
| 377 |
+
)
|
| 378 |
+
|
| 379 |
+
start_time = time.time()
|
| 380 |
+
transaction_id = str(uuid.uuid4())
|
| 381 |
+
|
| 382 |
+
try:
|
| 383 |
+
# Convert transaction to DataFrame
|
| 384 |
+
transaction_df = transaction_to_dataframe(request.transaction)
|
| 385 |
+
|
| 386 |
+
# Get options
|
| 387 |
+
options = request.options or PredictionOptions()
|
| 388 |
+
|
| 389 |
+
# Predict
|
| 390 |
+
result = inference_engine.predict_and_explain(
|
| 391 |
+
transaction_df,
|
| 392 |
+
shap_background=None,
|
| 393 |
+
topk=options.topk,
|
| 394 |
+
use_llm=options.include_llm_explanation,
|
| 395 |
+
language=options.language
|
| 396 |
+
)
|
| 397 |
+
|
| 398 |
+
probability = float(result['probabilities'][0])
|
| 399 |
+
decision, risk_level = calculate_decision(probability)
|
| 400 |
+
confidence = calculate_confidence(probability)
|
| 401 |
+
|
| 402 |
+
# Format SHAP explanations
|
| 403 |
+
shap_explanations = None
|
| 404 |
+
if options.include_shap and result.get('shap_table') is not None:
|
| 405 |
+
shap_df = result['shap_table']
|
| 406 |
+
shap_explanations = [
|
| 407 |
+
SHAPExplanation(
|
| 408 |
+
feature=row['feature'],
|
| 409 |
+
value=float(row['value']),
|
| 410 |
+
shap=float(row['shap']),
|
| 411 |
+
shap_abs=float(row['shap_abs']),
|
| 412 |
+
rank=idx + 1
|
| 413 |
+
)
|
| 414 |
+
for idx, (_, row) in enumerate(shap_df.iterrows())
|
| 415 |
+
]
|
| 416 |
+
|
| 417 |
+
# Format LLM explanation
|
| 418 |
+
llm_explanation = None
|
| 419 |
+
if result.get('llm_explanation'):
|
| 420 |
+
llm_explanation = {
|
| 421 |
+
"text": result['llm_explanation'],
|
| 422 |
+
"language": options.language
|
| 423 |
+
}
|
| 424 |
+
|
| 425 |
+
processing_time = int((time.time() - start_time) * 1000)
|
| 426 |
+
|
| 427 |
+
return PredictResponse(
|
| 428 |
+
transaction_id=transaction_id,
|
| 429 |
+
prediction=PredictionResult(
|
| 430 |
+
fraud_probability=probability,
|
| 431 |
+
decision=decision,
|
| 432 |
+
risk_level=risk_level,
|
| 433 |
+
confidence=confidence
|
| 434 |
+
),
|
| 435 |
+
shap_explanations=shap_explanations,
|
| 436 |
+
llm_explanation=llm_explanation,
|
| 437 |
+
processing_time_ms=processing_time,
|
| 438 |
+
model_version=MODEL_VERSION,
|
| 439 |
+
timestamp=datetime.utcnow().isoformat() + "Z"
|
| 440 |
+
)
|
| 441 |
+
|
| 442 |
+
except ValueError as e:
|
| 443 |
+
raise HTTPException(status_code=400, detail=str(e))
|
| 444 |
+
except Exception as e:
|
| 445 |
+
print(f"❌ Prediction error: {str(e)}")
|
| 446 |
+
raise HTTPException(status_code=500, detail=f"Prediction failed: {str(e)}")
|
| 447 |
+
|
| 448 |
+
@app.post("/predict/batch", response_model=BatchPredictResponse)
|
| 449 |
+
async def predict_batch(request: BatchPredictRequest):
|
| 450 |
+
"""
|
| 451 |
+
Predict fraud probability for multiple transactions
|
| 452 |
+
|
| 453 |
+
Returns predictions for all transactions in batch
|
| 454 |
+
"""
|
| 455 |
+
# Lazy loading for serverless environments
|
| 456 |
+
if inference_engine is None:
|
| 457 |
+
if not ensure_model_loaded():
|
| 458 |
+
raise HTTPException(
|
| 459 |
+
status_code=503,
|
| 460 |
+
detail="Model not loaded. The model is still loading or failed to load. Please try again in a moment."
|
| 461 |
+
)
|
| 462 |
+
|
| 463 |
+
start_time = time.time()
|
| 464 |
+
options = request.options or PredictionOptions()
|
| 465 |
+
|
| 466 |
+
results = []
|
| 467 |
+
|
| 468 |
+
try:
|
| 469 |
+
for transaction in request.transactions:
|
| 470 |
+
transaction_df = transaction_to_dataframe(transaction)
|
| 471 |
+
|
| 472 |
+
# Predict (without SHAP for batch to speed up)
|
| 473 |
+
probabilities, decisions = inference_engine.predict(transaction_df)
|
| 474 |
+
probability = float(probabilities[0])
|
| 475 |
+
decision, risk_level = calculate_decision(probability)
|
| 476 |
+
|
| 477 |
+
results.append({
|
| 478 |
+
"transaction_id": str(uuid.uuid4()),
|
| 479 |
+
"prediction": {
|
| 480 |
+
"fraud_probability": probability,
|
| 481 |
+
"decision": decision,
|
| 482 |
+
"risk_level": risk_level
|
| 483 |
+
}
|
| 484 |
+
})
|
| 485 |
+
|
| 486 |
+
processing_time = int((time.time() - start_time) * 1000)
|
| 487 |
+
|
| 488 |
+
return BatchPredictResponse(
|
| 489 |
+
results=results,
|
| 490 |
+
processing_time_ms=processing_time,
|
| 491 |
+
total_transactions=len(results)
|
| 492 |
+
)
|
| 493 |
+
|
| 494 |
+
except Exception as e:
|
| 495 |
+
raise HTTPException(status_code=500, detail=f"Batch prediction failed: {str(e)}")
|
| 496 |
+
|
| 497 |
+
# ============================================================================
|
| 498 |
+
# ERROR HANDLERS
|
| 499 |
+
# ============================================================================
|
| 500 |
+
|
| 501 |
+
@app.exception_handler(HTTPException)
|
| 502 |
+
async def http_exception_handler(request, exc):
|
| 503 |
+
return JSONResponse(
|
| 504 |
+
status_code=exc.status_code,
|
| 505 |
+
content={
|
| 506 |
+
"error": exc.detail,
|
| 507 |
+
"status_code": exc.status_code
|
| 508 |
+
}
|
| 509 |
+
)
|
| 510 |
+
|
| 511 |
+
# ============================================================================
|
| 512 |
+
# MAIN
|
| 513 |
+
# ============================================================================
|
| 514 |
+
|
| 515 |
+
if __name__ == "__main__":
|
| 516 |
+
# Get port from environment (required for Render, Railway, etc.)
|
| 517 |
+
port = int(os.getenv("PORT", 8000))
|
| 518 |
+
host = os.getenv("HOST", "0.0.0.0")
|
| 519 |
+
|
| 520 |
+
print(f"🌐 Binding to {host}:{port}")
|
| 521 |
+
|
| 522 |
+
uvicorn.run(
|
| 523 |
+
"main:app",
|
| 524 |
+
host=host,
|
| 525 |
+
port=port,
|
| 526 |
+
reload=False,
|
| 527 |
+
workers=1, # Single worker for ML model
|
| 528 |
+
log_level="info"
|
| 529 |
+
)
|
| 530 |
+
|
railway.json
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"$schema": "https://railway.app/railway.schema.json",
|
| 3 |
+
"build": {
|
| 4 |
+
"builder": "DOCKERFILE",
|
| 5 |
+
"dockerfilePath": "Dockerfile"
|
| 6 |
+
},
|
| 7 |
+
"deploy": {
|
| 8 |
+
"startCommand": "uvicorn main:app --host 0.0.0.0 --port $PORT",
|
| 9 |
+
"restartPolicyType": "ON_FAILURE",
|
| 10 |
+
"restartPolicyMaxRetries": 10
|
| 11 |
+
}
|
| 12 |
+
}
|
| 13 |
+
|
render.yaml
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Render.com deployment configuration for CloverShield ML API
|
| 2 |
+
|
| 3 |
+
services:
|
| 4 |
+
- type: web
|
| 5 |
+
name: clovershield-ml-api
|
| 6 |
+
runtime: docker
|
| 7 |
+
dockerfilePath: ./Dockerfile
|
| 8 |
+
dockerContext: .
|
| 9 |
+
envVars:
|
| 10 |
+
- key: MODEL_PATH
|
| 11 |
+
value: Models/fraud_pipeline_final.pkl
|
| 12 |
+
- key: MODEL_THRESHOLD
|
| 13 |
+
value: 0.0793
|
| 14 |
+
- key: PORT
|
| 15 |
+
value: 8000
|
| 16 |
+
- key: HOST
|
| 17 |
+
value: 0.0.0.0
|
| 18 |
+
- key: MAX_FIT_ROWS
|
| 19 |
+
value: "30000" # Reduced for Render free tier (512MB RAM)
|
| 20 |
+
- key: PAGERANK_LIMIT
|
| 21 |
+
value: "5000" # Reduced for memory efficiency
|
| 22 |
+
- key: GROQ_API_KEY
|
| 23 |
+
sync: false # Set in Render dashboard
|
| 24 |
+
healthCheckPath: /health
|
| 25 |
+
plan: starter # Upgrade for production
|
| 26 |
+
|
requirements.txt
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# CloverShield ML Inference API - Dependencies
|
| 2 |
+
# Python 3.9+
|
| 3 |
+
|
| 4 |
+
# FastAPI and server
|
| 5 |
+
fastapi>=0.104.0
|
| 6 |
+
uvicorn[standard]>=0.24.0
|
| 7 |
+
pydantic>=2.0.0
|
| 8 |
+
|
| 9 |
+
# Data processing
|
| 10 |
+
pandas>=1.5.0
|
| 11 |
+
numpy>=1.23.0
|
| 12 |
+
|
| 13 |
+
# Model loading and ML dependencies
|
| 14 |
+
joblib>=1.2.0
|
| 15 |
+
scikit-learn>=1.2.0
|
| 16 |
+
xgboost>=1.7.0
|
| 17 |
+
|
| 18 |
+
# Explainability
|
| 19 |
+
shap>=0.44.0
|
| 20 |
+
|
| 21 |
+
# LLM explanations (optional)
|
| 22 |
+
groq>=1.0.0
|
| 23 |
+
|
| 24 |
+
# Environment variable management
|
| 25 |
+
python-dotenv>=1.0.0
|
| 26 |
+
|
| 27 |
+
# Graph features (for model pipeline)
|
| 28 |
+
networkx>=3.0
|
| 29 |
+
|
| 30 |
+
# Optional: For better performance
|
| 31 |
+
scipy>=1.10.0
|
| 32 |
+
|
| 33 |
+
# CORS support (included in FastAPI)
|
| 34 |
+
# python-multipart for file uploads (if needed)
|
| 35 |
+
|
vercel.json
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"version": 2,
|
| 3 |
+
"builds": [
|
| 4 |
+
{
|
| 5 |
+
"src": "main.py",
|
| 6 |
+
"use": "@vercel/python"
|
| 7 |
+
}
|
| 8 |
+
],
|
| 9 |
+
"routes": [
|
| 10 |
+
{
|
| 11 |
+
"src": "/(.*)",
|
| 12 |
+
"dest": "main.py"
|
| 13 |
+
}
|
| 14 |
+
],
|
| 15 |
+
"env": {
|
| 16 |
+
"MODEL_PATH": "Models/fraud_pipeline_final.pkl",
|
| 17 |
+
"MODEL_THRESHOLD": "0.0793"
|
| 18 |
+
}
|
| 19 |
+
}
|
| 20 |
+
|