File size: 6,020 Bytes
44c2f50 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 | """Unit tests for parsers, normalizers, and the JSONL writer.
No network calls — tests use synthetic dataset records that mirror the shape
of SROIE and CORD.
"""
from __future__ import annotations
from datetime import date
from pathlib import Path
import pytest
from src.data_prep.cord import normalize_cord_ground_truth
from src.data_prep.parsers import clean_text, parse_date, parse_money
from src.data_prep.sroie import normalize_sroie_record
from src.data_prep.writer import read_jsonl, write_jsonl
from src.schemas import Receipt
# --- Money parsing ---------------------------------------------------------
@pytest.mark.parametrize(
"raw,expected",
[
("$12.99", 12.99),
("12.99", 12.99),
(12.999, 13.00),
("SGD 45.00", 45.00),
("1,299.50", 1299.50),
("1.299,50", 1299.50), # EU thousands + comma decimal
("12,99", 12.99), # EU decimal
("£10", 10.00),
(None, None),
("", None),
("total:", None),
("abc", None),
(1299, 1299.00),
],
)
def test_parse_money(raw, expected):
assert parse_money(raw) == expected
# --- Date parsing ---------------------------------------------------------
@pytest.mark.parametrize(
"raw,expected",
[
("2026-06-15", date(2026, 6, 15)),
("15/06/2026", date(2026, 6, 15)),
("06-15-2026", date(2026, 6, 15)),
("Jun 15, 2026", date(2026, 6, 15)),
("15 June 2026", date(2026, 6, 15)),
("20260615", date(2026, 6, 15)),
(None, None),
("", None),
("nonsense", None),
],
)
def test_parse_date(raw, expected):
assert parse_date(raw) == expected
def test_clean_text_collapses_whitespace():
assert clean_text(" hello world\n\t") == "hello world"
assert clean_text("") is None
assert clean_text(None) is None
# --- SROIE normalization ---------------------------------------------------
def test_normalize_sroie_flat_shape():
"""SROIE flat-format record → Receipt."""
rec = {
"company": "TAN WOON YANN",
"address": "789 KING STREET, TAMAN DAYA, 81100 JOHOR BAHRU",
"date": "25/06/2018",
"total": "$72.00",
}
receipt = normalize_sroie_record(rec)
assert receipt is not None
assert receipt.merchant == "TAN WOON YANN"
assert receipt.total == 72.00
assert receipt.transaction_date == date(2018, 6, 25)
assert receipt.currency == "SGD"
def test_normalize_sroie_nested_shape():
"""SROIE with parsed_data wrapper (mychen76 dataset variant)."""
rec = {
"id": "X001",
"image": None,
"parsed_data": {
"company": "AEON CO",
"date": "2018-11-30",
"address": "AEON MALURI",
"total": "39.20",
},
}
receipt = normalize_sroie_record(rec)
assert receipt is not None
assert receipt.merchant == "AEON CO"
assert receipt.total == 39.20
def test_normalize_sroie_missing_total_returns_none():
rec = {"company": "Some Shop", "date": "2020-01-01", "address": "X"}
assert normalize_sroie_record(rec) is None
def test_normalize_sroie_missing_merchant_returns_none():
rec = {"date": "2020-01-01", "total": "10.00"}
assert normalize_sroie_record(rec) is None
# --- CORD normalization ----------------------------------------------------
def test_normalize_cord_with_menu_and_totals():
gt = {
"menu": [
{"nm": "Iced Americano", "cnt": "1", "price": "4500", "unitprice": "4500"},
{"nm": "Muffin", "cnt": "2", "price": "6000", "unitprice": "3000"},
],
"sub_total": {"subtotal_price": "10500", "tax_price": "1050"},
"total": {"total_price": "11550"},
}
receipt = normalize_cord_ground_truth(gt)
assert receipt is not None
assert receipt.total == 11550.00
assert receipt.subtotal == 10500.00
assert receipt.tax == 1050.00
assert len(receipt.line_items) == 2
assert receipt.line_items[0].description == "Iced Americano"
assert receipt.line_items[1].quantity == 2.0
assert receipt.currency == "KRW"
def test_normalize_cord_missing_total_returns_none():
gt = {"menu": [{"nm": "X", "cnt": "1", "price": "1000"}], "sub_total": {}, "total": {}}
assert normalize_cord_ground_truth(gt) is None
def test_normalize_cord_wrapped_in_gt_parse():
"""CORD often ships ground truth under a `gt_parse` key."""
gt = {
"gt_parse": {
"menu": [{"nm": "Coffee", "cnt": "1", "price": "3000"}],
"sub_total": {"subtotal_price": "3000"},
"total": {"total_price": "3300"},
}
}
receipt = normalize_cord_ground_truth(gt)
assert receipt is not None
assert receipt.total == 3300.00
# --- JSONL round-trip ------------------------------------------------------
def test_jsonl_write_and_read(tmp_path: Path):
r1 = Receipt(merchant="A", total=1.00, currency="USD")
r2 = Receipt(merchant="B", total=2.00, currency="USD")
out = tmp_path / "sample.jsonl"
n = write_jsonl([("id1", r1), ("id2", r2)], out, source="test")
assert n == 2
records = read_jsonl(out)
assert len(records) == 2
assert records[0]["id"] == "id1"
assert records[0]["source"] == "test"
assert records[0]["ground_truth"]["merchant"] == "A"
assert records[1]["ground_truth"]["total"] == 2.00
# --- Sample data present ---------------------------------------------------
def test_sample_data_exists_and_parses():
"""The committed sample JSONL should always be readable and valid."""
for name in ("sroie_sample.jsonl", "cord_sample.jsonl"):
p = Path("data/samples") / name
assert p.exists(), f"Missing sample dataset: {p}"
records = read_jsonl(p)
assert len(records) >= 5, f"{p} should have at least 5 samples"
# Every record must round-trip through Receipt validation.
for rec in records:
Receipt.model_validate(rec["ground_truth"])
|