Sagar Patel commited on
Commit
3ee3ddb
·
1 Parent(s): 3a42655

Add bulk notes import

Browse files
README.md CHANGED
@@ -6,6 +6,7 @@ The current version focuses on a clean, deterministic foundation:
6
 
7
  - Record a transaction with your microphone and transcribe it with faster-whisper.
8
  - Type or paste a transaction note.
 
9
  - Parse it with simple rules.
10
  - Save the structured transaction to SQLite.
11
  - View the ledger, customer credit book, and inventory in a Gradio interface.
@@ -20,7 +21,9 @@ LLM parsing is intentionally not implemented yet.
20
  | Input | Parsed result |
21
  | --- | --- |
22
  | `Sold 12 mangoes, 20 each` | Sale, quantity `12`, item `mangoes`, unit price `20`, amount `240` |
 
23
  | `Paid 500 for supplies` | Expense, amount `500`, item `supplies` |
 
24
  | `Bought 50 mangoes` | Inventory purchase, quantity `50`, item `mangoes` |
25
  | `Amit owes 100` | Customer credit, customer `Amit`, amount `100` |
26
  | `Amit paid 50` | Customer payment, customer `Amit`, amount `50` |
@@ -74,3 +77,4 @@ pytest
74
  - Parser architecture supports rule-based and Hugging Face Inference API compatible LLM parsers, with rule fallback on LLM failure.
75
  - The dashboard shows daily sales, expenses, profit, outstanding credit, top sellers, and low-stock alerts from saved data.
76
  - WhatsApp summaries provide a short copyable daily recap for sharing.
 
 
6
 
7
  - Record a transaction with your microphone and transcribe it with faster-whisper.
8
  - Type or paste a transaction note.
9
+ - Bulk import multiple pasted notes for review and editing.
10
  - Parse it with simple rules.
11
  - Save the structured transaction to SQLite.
12
  - View the ledger, customer credit book, and inventory in a Gradio interface.
 
21
  | Input | Parsed result |
22
  | --- | --- |
23
  | `Sold 12 mangoes, 20 each` | Sale, quantity `12`, item `mangoes`, unit price `20`, amount `240` |
24
+ | `mango 12 x 20` | Sale, quantity `12`, item `mango`, unit price `20`, amount `240` |
25
  | `Paid 500 for supplies` | Expense, amount `500`, item `supplies` |
26
+ | `rent 300` | Expense, amount `300`, item `rent` |
27
  | `Bought 50 mangoes` | Inventory purchase, quantity `50`, item `mangoes` |
28
  | `Amit owes 100` | Customer credit, customer `Amit`, amount `100` |
29
  | `Amit paid 50` | Customer payment, customer `Amit`, amount `50` |
 
77
  - Parser architecture supports rule-based and Hugging Face Inference API compatible LLM parsers, with rule fallback on LLM failure.
78
  - The dashboard shows daily sales, expenses, profit, outstanding credit, top sellers, and low-stock alerts from saved data.
79
  - WhatsApp summaries provide a short copyable daily recap for sharing.
80
+ - Bulk import splits pasted notes by line, parses each line, supports review edits, and saves all reviewed transactions.
tests/test_bulk_import.py ADDED
@@ -0,0 +1,57 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from pathlib import Path
2
+
3
+ from voiceledger.ledger.customers import get_customer_balances
4
+ from voiceledger.ledger.database import add_transaction, get_transactions
5
+ from voiceledger.parser.bulk import parse_bulk_notes, review_table_to_transactions
6
+
7
+
8
+ def test_parse_bulk_notes_splits_lines_and_parses_each_transaction() -> None:
9
+ notes = """
10
+ mango 12 x 20
11
+ rent 300
12
+ Amit owes 100
13
+ Ramesh paid 50
14
+ """
15
+
16
+ review_table = parse_bulk_notes(notes)
17
+
18
+ assert len(review_table) == 4
19
+ assert review_table.iloc[0]["transaction_type"] == "sale"
20
+ assert review_table.iloc[0]["item"] == "mango"
21
+ assert review_table.iloc[0]["quantity"] == 12
22
+ assert review_table.iloc[0]["unit_price"] == 20
23
+ assert review_table.iloc[0]["amount"] == 240
24
+ assert review_table.iloc[1]["transaction_type"] == "expense"
25
+ assert review_table.iloc[1]["item"] == "rent"
26
+ assert review_table.iloc[1]["amount"] == 300
27
+ assert review_table.iloc[2]["transaction_type"] == "customer_credit"
28
+ assert review_table.iloc[2]["customer"] == "Amit"
29
+ assert review_table.iloc[3]["transaction_type"] == "customer_payment"
30
+ assert review_table.iloc[3]["customer"] == "Ramesh"
31
+
32
+
33
+ def test_review_table_to_transactions_uses_edited_values() -> None:
34
+ review_table = parse_bulk_notes("mango 12 x 20")
35
+ review_table.loc[0, "quantity"] = 10
36
+ review_table.loc[0, "amount"] = 200
37
+
38
+ transactions = review_table_to_transactions(review_table)
39
+
40
+ assert len(transactions) == 1
41
+ assert transactions[0].quantity == 10
42
+ assert transactions[0].amount == 200
43
+
44
+
45
+ def test_bulk_transactions_save_through_ledger_path(tmp_path: Path) -> None:
46
+ db_path = tmp_path / "voiceledger.sqlite3"
47
+ review_table = parse_bulk_notes("mango 12 x 20\nAmit owes 100")
48
+
49
+ for transaction in review_table_to_transactions(review_table):
50
+ add_transaction(transaction, db_path)
51
+
52
+ ledger = get_transactions(db_path)
53
+ balances = get_customer_balances(db_path)
54
+
55
+ assert len(ledger) == 2
56
+ assert balances.iloc[0]["customer"] == "Amit"
57
+ assert balances.iloc[0]["outstanding_balance"] == 100
tests/test_parser.py CHANGED
@@ -12,6 +12,16 @@ def test_parse_sale_with_quantity_and_unit_price() -> None:
12
  assert transaction.payment_status == "paid"
13
 
14
 
 
 
 
 
 
 
 
 
 
 
15
  def test_parse_expense() -> None:
16
  transaction = parse_transaction("Paid 500 for supplies")
17
 
@@ -21,6 +31,14 @@ def test_parse_expense() -> None:
21
  assert transaction.payment_status == "paid"
22
 
23
 
 
 
 
 
 
 
 
 
24
  def test_parse_inventory_purchase() -> None:
25
  transaction = parse_transaction("Bought 50 mangoes")
26
 
 
12
  assert transaction.payment_status == "paid"
13
 
14
 
15
+ def test_parse_shorthand_sale() -> None:
16
+ transaction = parse_transaction("mango 12 x 20")
17
+
18
+ assert transaction.transaction_type == "sale"
19
+ assert transaction.item == "mango"
20
+ assert transaction.quantity == 12
21
+ assert transaction.unit_price == 20
22
+ assert transaction.amount == 240
23
+
24
+
25
  def test_parse_expense() -> None:
26
  transaction = parse_transaction("Paid 500 for supplies")
27
 
 
31
  assert transaction.payment_status == "paid"
32
 
33
 
34
+ def test_parse_shorthand_expense() -> None:
35
+ transaction = parse_transaction("rent 300")
36
+
37
+ assert transaction.transaction_type == "expense"
38
+ assert transaction.item == "rent"
39
+ assert transaction.amount == 300
40
+
41
+
42
  def test_parse_inventory_purchase() -> None:
43
  transaction = parse_transaction("Bought 50 mangoes")
44
 
voiceledger/parser/__init__.py CHANGED
@@ -1,9 +1,18 @@
1
  """Parsing utilities for transaction notes."""
2
 
3
  from voiceledger.parser.base import Parser
 
4
  from voiceledger.parser.llm_parser import LLMParser
5
  from voiceledger.parser.rules import RuleParser
6
  from voiceledger.parser.rules import parse_transaction
7
  from voiceledger.parser.schema import Transaction
8
 
9
- __all__ = ["Transaction", "Parser", "RuleParser", "LLMParser", "parse_transaction"]
 
 
 
 
 
 
 
 
 
1
  """Parsing utilities for transaction notes."""
2
 
3
  from voiceledger.parser.base import Parser
4
+ from voiceledger.parser.bulk import parse_bulk_notes, review_table_to_transactions
5
  from voiceledger.parser.llm_parser import LLMParser
6
  from voiceledger.parser.rules import RuleParser
7
  from voiceledger.parser.rules import parse_transaction
8
  from voiceledger.parser.schema import Transaction
9
 
10
+ __all__ = [
11
+ "Transaction",
12
+ "Parser",
13
+ "RuleParser",
14
+ "LLMParser",
15
+ "parse_transaction",
16
+ "parse_bulk_notes",
17
+ "review_table_to_transactions",
18
+ ]
voiceledger/parser/bulk.py ADDED
@@ -0,0 +1,116 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Bulk note parsing helpers."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Any
6
+
7
+ import pandas as pd
8
+
9
+ from voiceledger.parser.rules import parse_transaction
10
+ from voiceledger.parser.schema import Transaction
11
+
12
+
13
+ REVIEW_COLUMNS = [
14
+ "transaction_type",
15
+ "item",
16
+ "quantity",
17
+ "unit_price",
18
+ "amount",
19
+ "customer",
20
+ "payment_status",
21
+ "notes",
22
+ "confidence",
23
+ ]
24
+
25
+
26
+ def parse_bulk_notes(notes: str) -> pd.DataFrame:
27
+ """Split pasted notes into lines and parse each line for review."""
28
+ transactions = [parse_transaction(line) for line in _split_note_lines(notes)]
29
+ return transactions_to_dataframe(transactions)
30
+
31
+
32
+ def transactions_to_dataframe(transactions: list[Transaction]) -> pd.DataFrame:
33
+ """Convert transactions into an editable review DataFrame."""
34
+ records = [transaction.model_dump() for transaction in transactions]
35
+ return pd.DataFrame.from_records(records, columns=REVIEW_COLUMNS)
36
+
37
+
38
+ def review_table_to_transactions(review_table: Any) -> list[Transaction]:
39
+ """Convert an edited review table back into validated transactions."""
40
+ frame = _coerce_review_table(review_table)
41
+ transactions: list[Transaction] = []
42
+
43
+ for record in frame.to_dict(orient="records"):
44
+ cleaned = _clean_record(record)
45
+ if _is_empty_record(cleaned):
46
+ continue
47
+ transactions.append(Transaction(**cleaned))
48
+
49
+ return transactions
50
+
51
+
52
+ def _split_note_lines(notes: str) -> list[str]:
53
+ """Return non-empty stripped lines from pasted notes."""
54
+ return [line.strip() for line in (notes or "").splitlines() if line.strip()]
55
+
56
+
57
+ def _coerce_review_table(review_table: Any) -> pd.DataFrame:
58
+ """Coerce common Gradio Dataframe values into a Pandas DataFrame."""
59
+ if review_table is None:
60
+ return pd.DataFrame(columns=REVIEW_COLUMNS)
61
+ if isinstance(review_table, pd.DataFrame):
62
+ return review_table.reindex(columns=REVIEW_COLUMNS)
63
+ if isinstance(review_table, list):
64
+ return pd.DataFrame(review_table, columns=REVIEW_COLUMNS)
65
+ if isinstance(review_table, dict) and "data" in review_table:
66
+ headers = review_table.get("headers") or REVIEW_COLUMNS
67
+ return pd.DataFrame(review_table["data"], columns=headers).reindex(columns=REVIEW_COLUMNS)
68
+ return pd.DataFrame(review_table).reindex(columns=REVIEW_COLUMNS)
69
+
70
+
71
+ def _clean_record(record: dict[str, Any]) -> dict[str, Any]:
72
+ """Normalize editable table values before Pydantic validation."""
73
+ cleaned = dict(record)
74
+ for field in ["item", "customer", "notes"]:
75
+ cleaned[field] = _none_if_blank(cleaned.get(field))
76
+
77
+ for field in ["quantity", "unit_price", "amount", "confidence"]:
78
+ cleaned[field] = _number_or_none(cleaned.get(field))
79
+
80
+ cleaned["transaction_type"] = _none_if_blank(cleaned.get("transaction_type")) or "unknown"
81
+ cleaned["payment_status"] = _none_if_blank(cleaned.get("payment_status")) or "unknown"
82
+ cleaned["notes"] = cleaned["notes"] or ""
83
+ cleaned["confidence"] = cleaned["confidence"] if cleaned["confidence"] is not None else 0.0
84
+ return cleaned
85
+
86
+
87
+ def _none_if_blank(value: Any) -> Any:
88
+ """Return None for blank, null, or NaN values."""
89
+ if value is None:
90
+ return None
91
+ if pd.isna(value):
92
+ return None
93
+ if isinstance(value, str) and not value.strip():
94
+ return None
95
+ if isinstance(value, str):
96
+ return value.strip()
97
+ return value
98
+
99
+
100
+ def _number_or_none(value: Any) -> float | None:
101
+ """Return a float for numeric values, otherwise None."""
102
+ normalized = _none_if_blank(value)
103
+ if normalized is None:
104
+ return None
105
+ return float(normalized)
106
+
107
+
108
+ def _is_empty_record(record: dict[str, Any]) -> bool:
109
+ """Return whether a review row has no meaningful transaction data."""
110
+ return (
111
+ record["transaction_type"] == "unknown"
112
+ and record["item"] is None
113
+ and record["amount"] is None
114
+ and record["customer"] is None
115
+ and not record["notes"]
116
+ )
voiceledger/parser/rules.py CHANGED
@@ -26,6 +26,17 @@ EXPENSE_PATTERN = re.compile(
26
  rf"(?P<item>[a-zA-Z][a-zA-Z\s-]*)?",
27
  re.IGNORECASE,
28
  )
 
 
 
 
 
 
 
 
 
 
 
29
  INVENTORY_PURCHASE_PATTERN = re.compile(
30
  rf"\b(?:bought|purchased|buy|stocked|restocked)\s+"
31
  rf"(?P<quantity>\d+(?:\.\d+)?)\s+"
@@ -81,10 +92,18 @@ def parse_transaction(note: str) -> Transaction:
81
  if sale:
82
  return sale
83
 
 
 
 
 
84
  expense = _parse_expense(cleaned_note)
85
  if expense:
86
  return expense
87
 
 
 
 
 
88
  return Transaction(notes=cleaned_note, confidence=0.15)
89
 
90
 
@@ -132,6 +151,42 @@ def _parse_expense(note: str) -> Transaction | None:
132
  )
133
 
134
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
135
  def _parse_inventory_purchase(note: str) -> Transaction | None:
136
  """Parse notes like 'Bought 50 mangoes'."""
137
  match = INVENTORY_PURCHASE_PATTERN.search(note)
 
26
  rf"(?P<item>[a-zA-Z][a-zA-Z\s-]*)?",
27
  re.IGNORECASE,
28
  )
29
+ SHORTHAND_SALE_PATTERN = re.compile(
30
+ rf"^\s*(?P<item>[a-zA-Z][a-zA-Z\s-]*?)\s+"
31
+ rf"(?P<quantity>\d+(?:\.\d+)?)\s*(?:x|@)\s*"
32
+ rf"(?P<unit_price>\d+(?:\.\d+)?)\s*$",
33
+ re.IGNORECASE,
34
+ )
35
+ SHORTHAND_EXPENSE_PATTERN = re.compile(
36
+ rf"^\s*(?P<item>[a-zA-Z][a-zA-Z\s-]*?)\s+"
37
+ rf"(?P<amount>\d+(?:\.\d+)?)\s*$",
38
+ re.IGNORECASE,
39
+ )
40
  INVENTORY_PURCHASE_PATTERN = re.compile(
41
  rf"\b(?:bought|purchased|buy|stocked|restocked)\s+"
42
  rf"(?P<quantity>\d+(?:\.\d+)?)\s+"
 
92
  if sale:
93
  return sale
94
 
95
+ shorthand_sale = _parse_shorthand_sale(cleaned_note)
96
+ if shorthand_sale:
97
+ return shorthand_sale
98
+
99
  expense = _parse_expense(cleaned_note)
100
  if expense:
101
  return expense
102
 
103
+ shorthand_expense = _parse_shorthand_expense(cleaned_note)
104
+ if shorthand_expense:
105
+ return shorthand_expense
106
+
107
  return Transaction(notes=cleaned_note, confidence=0.15)
108
 
109
 
 
151
  )
152
 
153
 
154
+ def _parse_shorthand_sale(note: str) -> Transaction | None:
155
+ """Parse notes like 'mango 12 x 20'."""
156
+ match = SHORTHAND_SALE_PATTERN.search(note)
157
+ if not match:
158
+ return None
159
+
160
+ quantity = _to_float(match.group("quantity"))
161
+ unit_price = _to_float(match.group("unit_price"))
162
+ return Transaction(
163
+ transaction_type="sale",
164
+ item=_clean_item(match.group("item")),
165
+ quantity=quantity,
166
+ unit_price=unit_price,
167
+ amount=round(quantity * unit_price, 2) if quantity is not None and unit_price is not None else None,
168
+ payment_status="paid",
169
+ notes=note,
170
+ confidence=0.82,
171
+ )
172
+
173
+
174
+ def _parse_shorthand_expense(note: str) -> Transaction | None:
175
+ """Parse notes like 'rent 300'."""
176
+ match = SHORTHAND_EXPENSE_PATTERN.search(note)
177
+ if not match:
178
+ return None
179
+
180
+ return Transaction(
181
+ transaction_type="expense",
182
+ item=_clean_item(match.group("item")),
183
+ amount=_to_float(match.group("amount")),
184
+ payment_status="paid",
185
+ notes=note,
186
+ confidence=0.68,
187
+ )
188
+
189
+
190
  def _parse_inventory_purchase(note: str) -> Transaction | None:
191
  """Parse notes like 'Bought 50 mangoes'."""
192
  match = INVENTORY_PURCHASE_PATTERN.search(note)
voiceledger/ui/gradio_app.py CHANGED
@@ -19,6 +19,7 @@ from voiceledger.ledger.analytics import (
19
  from voiceledger.ledger.customers import get_customer_balances
20
  from voiceledger.ledger.database import add_transaction, get_transactions, initialize_database
21
  from voiceledger.ledger.inventory import get_inventory
 
22
  from voiceledger.parser.rules import parse_transaction
23
  from voiceledger.parser.schema import Transaction
24
  from voiceledger.reports.pdf_report import generate_daily_summary_pdf
@@ -82,6 +83,33 @@ def create_app(db_path: str | Path | None = None) -> gr.Blocks:
82
  outputs=status_output,
83
  )
84
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
85
  with gr.Tab("Dashboard"):
86
  refresh_dashboard_button = gr.Button("Refresh Dashboard", variant="primary")
87
  with gr.Row():
@@ -261,6 +289,28 @@ def _save_transaction(transaction_payload: dict[str, Any] | None, db_path: str |
261
  return f"Saved transaction #{transaction_id}."
262
 
263
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
264
  def _get_dashboard_data(
265
  db_path: str | Path | None,
266
  ) -> tuple[float, float, float, float, str, pd.DataFrame, pd.DataFrame]:
 
19
  from voiceledger.ledger.customers import get_customer_balances
20
  from voiceledger.ledger.database import add_transaction, get_transactions, initialize_database
21
  from voiceledger.ledger.inventory import get_inventory
22
+ from voiceledger.parser.bulk import REVIEW_COLUMNS, parse_bulk_notes, review_table_to_transactions
23
  from voiceledger.parser.rules import parse_transaction
24
  from voiceledger.parser.schema import Transaction
25
  from voiceledger.reports.pdf_report import generate_daily_summary_pdf
 
83
  outputs=status_output,
84
  )
85
 
86
+ with gr.Tab("Bulk Import"):
87
+ bulk_notes_input = gr.Textbox(
88
+ label="Paste transaction notes",
89
+ placeholder="mango 12 x 20\nrent 300\nAmit owes 100\nRamesh paid 50",
90
+ lines=8,
91
+ )
92
+ with gr.Row():
93
+ bulk_parse_button = gr.Button("Parse Lines", variant="primary")
94
+ bulk_save_button = gr.Button("Save All Transactions")
95
+ bulk_review_output = gr.Dataframe(
96
+ headers=REVIEW_COLUMNS,
97
+ label="Review and edit transactions",
98
+ interactive=True,
99
+ wrap=True,
100
+ )
101
+ bulk_status_output = gr.Markdown()
102
+ bulk_parse_button.click(
103
+ fn=_parse_bulk_notes_for_review,
104
+ inputs=bulk_notes_input,
105
+ outputs=[bulk_review_output, bulk_status_output],
106
+ )
107
+ bulk_save_button.click(
108
+ fn=lambda review_table: _save_bulk_transactions(review_table, db_path),
109
+ inputs=bulk_review_output,
110
+ outputs=bulk_status_output,
111
+ )
112
+
113
  with gr.Tab("Dashboard"):
114
  refresh_dashboard_button = gr.Button("Refresh Dashboard", variant="primary")
115
  with gr.Row():
 
289
  return f"Saved transaction #{transaction_id}."
290
 
291
 
292
+ def _parse_bulk_notes_for_review(notes: str) -> tuple[pd.DataFrame, str]:
293
+ """Parse pasted multiline notes into an editable review table."""
294
+ review_table = parse_bulk_notes(notes)
295
+ if review_table.empty:
296
+ return review_table, "Paste one transaction per line, then parse."
297
+ return review_table, f"Parsed {len(review_table)} transaction lines. Review and edit before saving."
298
+
299
+
300
+ def _save_bulk_transactions(review_table: Any, db_path: str | Path | None) -> str:
301
+ """Save all reviewed bulk import transactions."""
302
+ try:
303
+ transactions = review_table_to_transactions(review_table)
304
+ except Exception as exc:
305
+ return f"Could not save bulk import: {exc}"
306
+
307
+ if not transactions:
308
+ return "No reviewed transactions to save."
309
+
310
+ saved_ids = [add_transaction(transaction, db_path) for transaction in transactions]
311
+ return f"Saved {len(saved_ids)} transactions. Last transaction id: #{saved_ids[-1]}."
312
+
313
+
314
  def _get_dashboard_data(
315
  db_path: str | Path | None,
316
  ) -> tuple[float, float, float, float, str, pd.DataFrame, pd.DataFrame]: