Initial kitchen scaffold — Phase 1 kitchen port (build-verified 2026-07-11)

FastAPI backend (Python 3.11, MSSQL ODBC for SambaPOS, Azure DI OCR),
kitchen_db on central PG. React/TS/Vite frontend with navy sidebar layout.

Backend: auth.py (APP_SLUG=kitchen, SimpleNamespace — archive routes use
.kitchen_id/.is_admin without modification), main.py (51 migrations, scheduler,
internal router for KDS bookings feed), api/internal.py, full archive API
(31 routers: invoices, recipes, menus, sambapos, resos, newbook, disputes,
purchase_orders, etc.), models, migrations, OCR pipeline.
kitchen_id pinned to 1 (B1 — single hotel).

Frontend: AuthGate (app=kitchen, token shim for archive compat — B5b pending),
Layout (navy sidebar, 6 sections, Lucide icons, teal --app-primary),
App.tsx (Outlet pattern, UploadApp outside Layout), index.css (full :root block).
strict: false — archive components have type issues; build clean.

Note: 45 archive components call fetch('/api/...') without /kitchen/ prefix
(B5b). Runtime 404s; deferred until after initial testing.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
jtricerolph 2026-07-12 12:15:39 +00:00
commit 8d688b459d
10003 changed files with 1928395 additions and 0 deletions

4
backend/ocr/__init__.py Normal file
View file

@ -0,0 +1,4 @@
from .extractor import process_invoice_image
from .parser import extract_invoice_fields
__all__ = ["process_invoice_image", "extract_invoice_fields"]

Binary file not shown.

Binary file not shown.

Binary file not shown.

View file

@ -0,0 +1,853 @@
import logging
import json
import asyncio
import re
from datetime import date
from decimal import Decimal
from typing import Optional, Any
from azure.ai.formrecognizer import DocumentAnalysisClient
from azure.core.credentials import AzureKeyCredential
from azure.core.exceptions import HttpResponseError
logger = logging.getLogger(__name__)
# Map written/abbreviated unit names to standard unit_size_type
_UNIT_NORMALIZE = {
# Weight
'g': 'g', 'gm': 'g', 'gms': 'g', 'grm': 'g', 'gram': 'g', 'grams': 'g',
'kg': 'kg', 'kgs': 'kg', 'kilo': 'kg', 'kilos': 'kg',
'kilogram': 'kg', 'kilograms': 'kg', 'kilergram': 'kg',
'oz': 'oz', 'ounce': 'oz', 'ounces': 'oz',
# Volume
'ml': 'ml', 'mls': 'ml', 'millilitre': 'ml', 'millilitres': 'ml',
'milliliter': 'ml', 'milliliters': 'ml',
'cl': 'cl', 'cls': 'cl', 'centilitre': 'cl', 'centilitres': 'cl',
'l': 'ltr', 'ltr': 'ltr', 'ltrs': 'ltr',
'litre': 'ltr', 'litres': 'ltr', 'liter': 'ltr', 'liters': 'ltr',
'pint': 'pint', 'pints': 'pint', 'pt': 'pint',
}
# Build regex alternation from keys, longest first to avoid partial matches
_UNIT_PATTERN = '|'.join(sorted(_UNIT_NORMALIZE.keys(), key=len, reverse=True))
def _normalize_unit(raw_unit: str) -> str:
"""Normalize a matched unit string to standard abbreviation."""
return _UNIT_NORMALIZE.get(raw_unit.lower(), raw_unit.lower())
def parse_pack_size(raw_content: str) -> dict:
"""
Extract pack size info from raw line item content.
For weight ranges (e.g. 140-170GM), uses the larger end.
Examples:
"120x15g" -> pack_quantity=120, unit_size=15, unit_size_type="g"
"12×1ltr" -> pack_quantity=12, unit_size=1, unit_size_type="ltr"
"6x1.5kg" -> pack_quantity=6, unit_size=1.5, unit_size_type="kg"
"Coconut Milk 400ml" -> pack_quantity=1, unit_size=400, unit_size_type="ml"
"Semiskimmed milk 1 Litre" -> pack_quantity=1, unit_size=1, unit_size_type="ltr"
"FILLET 140-170GM SKINNED" -> pack_quantity=1, unit_size=170, unit_size_type="g"
"454GM NETT PACK" -> pack_quantity=1, unit_size=454, unit_size_type="g"
"""
result = {
"pack_quantity": None,
"unit_size": None,
"unit_size_type": None,
}
if not raw_content:
return result
# Pattern 1a: Pack format with range - 10x140-170g (uses larger end)
pack_range = rf'(\d+)\s*[x×]\s*\d+(?:\.\d+)?\s*[-]\s*(\d+(?:\.\d+)?)\s*({_UNIT_PATTERN})\b'
match = re.search(pack_range, raw_content, re.IGNORECASE)
if match:
result["pack_quantity"] = int(match.group(1))
result["unit_size"] = float(match.group(2))
result["unit_size_type"] = _normalize_unit(match.group(3))
return result
# Pattern 1b: Pack format - 120x15g, 12×1ltr, 6x1.5kg, 2x1 Litre
pack_pattern = rf'(\d+)\s*[x×]\s*(\d+(?:\.\d+)?)\s*({_UNIT_PATTERN})\b'
match = re.search(pack_pattern, raw_content, re.IGNORECASE)
if match:
result["pack_quantity"] = int(match.group(1))
result["unit_size"] = float(match.group(2))
result["unit_size_type"] = _normalize_unit(match.group(3))
return result
# Pattern 2a: Standalone range - "140-170GM", "1.25-1.65KG" (uses larger end)
range_pattern = rf'(?<!£)(?<!\$)\d+(?:\.\d+)?\s*[-]\s*(\d+(?:\.\d+)?)\s*({_UNIT_PATTERN})\b'
match = re.search(range_pattern, raw_content, re.IGNORECASE)
if match:
result["pack_quantity"] = 1
result["unit_size"] = float(match.group(1))
result["unit_size_type"] = _normalize_unit(match.group(2))
return result
# Pattern 2b: Standalone size - "400ml", "1 Litre", "250 Grams", "454GM"
standalone_pattern = rf'(?<!£)(?<!\$)(\d+(?:\.\d+)?)\s*({_UNIT_PATTERN})\b'
match = re.search(standalone_pattern, raw_content, re.IGNORECASE)
if match:
result["pack_quantity"] = 1
result["unit_size"] = float(match.group(1))
result["unit_size_type"] = _normalize_unit(match.group(2))
return result
# Retry configuration for rate limiting
MAX_RETRIES = 3
BASE_RETRY_DELAY = 5 # seconds
def extract_field_value(field: Any) -> Any:
"""Extract the actual value from an Azure DocumentField, handling different types."""
if field is None:
return None
# Try the .value attribute first (standard SDK approach)
if hasattr(field, 'value'):
return field.value
# For currency types, check for .amount
if hasattr(field, 'amount'):
return field.amount
# Direct value
return field
def extract_currency_amount(field: Any) -> Optional[Decimal]:
"""Extract a currency amount from an Azure field."""
if field is None:
return None
val = extract_field_value(field)
if val is None:
return None
# If it's a currency object with .amount
if hasattr(val, 'amount'):
return Decimal(str(val.amount))
# Direct numeric value
try:
return Decimal(str(val))
except (ValueError, TypeError):
return None
def serialize_bounding_regions(field: Any) -> list:
"""Extract bounding regions from an Azure DocumentField."""
regions = []
if hasattr(field, 'bounding_regions') and field.bounding_regions:
for region in field.bounding_regions:
region_data = {
'page_number': region.page_number if hasattr(region, 'page_number') else 1,
'polygon': []
}
if hasattr(region, 'polygon') and region.polygon:
# polygon is a list of Point objects with x, y attributes
region_data['polygon'] = [
[p.x, p.y] for p in region.polygon
]
regions.append(region_data)
return regions
def serialize_azure_field(field: Any) -> Any:
"""Serialize an Azure DocumentField to a JSON-compatible dict."""
if field is None:
return None
result = {}
# Get field type
if hasattr(field, 'value_type'):
result['type'] = str(field.value_type)
# Get content (the raw text from the document)
if hasattr(field, 'content'):
result['content'] = field.content
# Get confidence
if hasattr(field, 'confidence'):
result['confidence'] = field.confidence
# Get bounding regions (coordinates for highlighting)
bounding_regions = serialize_bounding_regions(field)
if bounding_regions:
result['bounding_regions'] = bounding_regions
# Get the value based on type
value = extract_field_value(field)
if value is None:
result['value'] = None
elif isinstance(value, (str, int, float, bool)):
result['value'] = value
elif isinstance(value, date):
result['value'] = value.isoformat()
elif isinstance(value, Decimal):
result['value'] = float(value)
elif hasattr(value, 'amount'):
# Currency type
result['value'] = {
'amount': float(value.amount) if value.amount else None,
'symbol': getattr(value, 'symbol', None),
'code': getattr(value, 'code', None),
}
elif isinstance(value, list):
# Array of items
result['value'] = [serialize_azure_field(item) for item in value]
elif isinstance(value, dict):
# Object with nested fields
result['value'] = {k: serialize_azure_field(v) for k, v in value.items()}
else:
# Unknown type - try to convert to string
result['value'] = str(value)
return result
def serialize_azure_result(result: Any) -> dict:
"""Serialize the full Azure AnalyzeResult to a JSON-compatible dict."""
output = {
'content': result.content,
'documents': [],
'pages': []
}
# Capture page dimensions for coordinate scaling
if hasattr(result, 'pages') and result.pages:
for page in result.pages:
page_info = {
'page_number': page.page_number if hasattr(page, 'page_number') else 1,
'width': page.width if hasattr(page, 'width') else None,
'height': page.height if hasattr(page, 'height') else None,
'unit': str(page.unit) if hasattr(page, 'unit') else 'inch',
'angle': page.angle if hasattr(page, 'angle') else 0,
}
output['pages'].append(page_info)
if result.documents:
for doc in result.documents:
doc_dict = {
'doc_type': doc.doc_type if hasattr(doc, 'doc_type') else None,
'confidence': doc.confidence if hasattr(doc, 'confidence') else None,
'fields': {}
}
if doc.fields:
for field_name, field_value in doc.fields.items():
doc_dict['fields'][field_name] = serialize_azure_field(field_value)
output['documents'].append(doc_dict)
return output
def _call_azure_sync(image_path: str, azure_endpoint: str, azure_key: str):
"""
Synchronous Azure API call - runs in thread pool to avoid blocking event loop.
"""
client = DocumentAnalysisClient(
endpoint=azure_endpoint,
credential=AzureKeyCredential(azure_key)
)
with open(image_path, "rb") as f:
poller = client.begin_analyze_document(
"prebuilt-invoice",
document=f
)
return poller.result()
async def process_invoice_with_azure(
image_path: str,
azure_endpoint: str,
azure_key: str,
clean_product_codes: bool = False,
filter_subtotal_rows: bool = False,
use_weight_as_quantity: bool = False
) -> dict:
"""
Process an invoice image using Azure Document Intelligence.
Args:
image_path: Path to the invoice image
azure_endpoint: Azure Document Intelligence endpoint
azure_key: Azure API key
clean_product_codes: Strip section headers (like "CHILL/AMBIENT") from product codes
filter_subtotal_rows: Filter out subtotal/total rows from line items
Returns:
dict with extracted fields:
- invoice_number: str or None
- invoice_date: date or None
- total: Decimal or None
- vendor_name: str or None
- order_number: str or None
- line_items: list of dicts
- raw_text: str
- confidence: float
"""
# Retry loop for rate limiting (429 errors)
last_error = None
loop = asyncio.get_event_loop()
for attempt in range(MAX_RETRIES + 1):
try:
# Run blocking Azure SDK call in thread pool to avoid blocking event loop
result = await loop.run_in_executor(
None, # Use default thread pool
_call_azure_sync,
image_path,
azure_endpoint,
azure_key
)
break # Success - exit retry loop
except HttpResponseError as e:
if e.status_code == 429:
# Rate limited - extract retry-after or use exponential backoff
retry_after = BASE_RETRY_DELAY * (2 ** attempt)
if hasattr(e, 'response') and e.response:
retry_header = e.response.headers.get('Retry-After')
if retry_header:
try:
retry_after = int(retry_header)
except ValueError:
pass
if attempt < MAX_RETRIES:
logger.warning(f"Azure rate limited (429). Retry {attempt + 1}/{MAX_RETRIES} after {retry_after}s")
await asyncio.sleep(retry_after)
last_error = e
continue
else:
logger.error(f"Azure rate limit exceeded after {MAX_RETRIES} retries")
raise Exception(f"Azure rate limit exceeded. Please wait a moment and try again.") from e
elif e.status_code == 403:
# Quota exceeded or access denied
logger.error(f"Azure access denied (403): {str(e)}")
raise Exception(
"Azure quota exceeded or access denied. "
"Please check your Azure subscription and budget limits in the Azure portal."
) from e
elif e.status_code == 401:
# Authentication failed
logger.error(f"Azure authentication failed (401): {str(e)}")
raise Exception(
"Azure authentication failed. Please check your API credentials in Settings."
) from e
else:
# Other HTTP error - don't retry
logger.error(f"Azure HTTP error {e.status_code}: {str(e)}")
raise
except Exception as e:
logger.error(f"Azure OCR error: {e}")
raise
else:
# Exhausted retries without success
raise Exception(f"Azure processing failed after {MAX_RETRIES} retries") from last_error
# Log document count for debugging
doc_count = len(result.documents) if result.documents else 0
logger.info(f"Azure returned {doc_count} document(s)")
# Serialize the full Azure response for storage/debugging
raw_json = serialize_azure_result(result)
if not result.documents:
logger.warning("No invoice detected in document")
return {
"invoice_number": None,
"invoice_date": None,
"total": None,
"vendor_name": None,
"order_number": None,
"line_items": [],
"raw_text": result.content or "",
"raw_json": raw_json,
"confidence": 0.0
}
# For multi-page invoices, we may have multiple documents
# Use first document for header fields, but combine line items from all
invoice = result.documents[0]
fields = invoice.fields
# Log available fields for debugging
field_names = list(fields.keys()) if fields else []
logger.info(f"Available fields in first document: {field_names}")
# Extract invoice number
invoice_number = None
if "InvoiceId" in fields:
val = extract_field_value(fields["InvoiceId"])
if val:
invoice_number = str(val)
logger.debug(f"Extracted InvoiceId: {invoice_number}")
# Extract invoice date
invoice_date = None
if "InvoiceDate" in fields:
val = extract_field_value(fields["InvoiceDate"])
if val and isinstance(val, date):
invoice_date = val
logger.debug(f"Extracted InvoiceDate: {invoice_date}")
# Extract total (gross, inc. VAT)
total = extract_currency_amount(fields.get("InvoiceTotal"))
if total:
logger.debug(f"Extracted InvoiceTotal: {total}")
# Extract subtotal (net, exc. VAT)
net_total = extract_currency_amount(fields.get("SubTotal"))
if net_total:
logger.debug(f"Extracted SubTotal: {net_total}")
# Extract vendor name
vendor_name = None
if "VendorName" in fields:
val = extract_field_value(fields["VendorName"])
if val:
vendor_name = str(val)
logger.debug(f"Extracted VendorName: {vendor_name}")
# Extract purchase order / order number
order_number = None
if "PurchaseOrder" in fields:
val = extract_field_value(fields["PurchaseOrder"])
if val:
order_number = str(val)
logger.debug(f"Extracted PurchaseOrder: {order_number}")
# Extract line items from ALL documents (for multi-page invoices)
line_items = []
for doc_idx, doc in enumerate(result.documents):
doc_fields = doc.fields or {}
if "Items" in doc_fields:
items_field = doc_fields["Items"]
items_value = extract_field_value(items_field)
if items_value and hasattr(items_value, '__iter__'):
logger.info(f"Document {doc_idx}: Found {len(items_value)} line items")
for item_idx, item in enumerate(items_value):
try:
# Get the item's fields - handle various Azure SDK structures
item_fields = None
# Try item.value first (standard DocumentField)
if hasattr(item, 'value') and item.value is not None:
item_fields = item.value
# If value is still None, try to use item directly
if item_fields is None:
item_fields = item
# Skip if we still don't have usable fields
if item_fields is None:
logger.warning(f"Item {item_idx}: fields is None, skipping")
continue
# item_fields should be dict-like (either dict or has __getitem__)
if not (isinstance(item_fields, dict) or hasattr(item_fields, '__getitem__')):
logger.warning(f"Item {item_idx}: Unexpected item_fields type: {type(item_fields)}")
continue
line_item = {
"product_code": None,
"description": None,
"unit": None,
"quantity": None,
"order_quantity": None,
"unit_price": None,
"tax_rate": None,
"tax_amount": None,
"amount": None,
"bounding_regions": [],
"raw_content": None,
"pack_quantity": None,
"unit_size": None,
"unit_size_type": None,
"cost_per_item": None,
"cost_per_portion": None,
}
# Capture bounding regions for this line item
line_item["bounding_regions"] = serialize_bounding_regions(item)
# Capture raw content (contains all text including pack size info)
line_item["raw_content"] = item.content if hasattr(item, 'content') else None
# Helper to safely get field value
def safe_get_field(fields, key):
try:
if key in fields:
return extract_field_value(fields[key])
except (KeyError, TypeError):
pass
return None
# Extract product code
line_item["product_code"] = safe_get_field(item_fields, "ProductCode")
# Clean product code if enabled - strip section headers
# NOTE: May need supplier-specific constraints in future (e.g., only for Brakes/Sysco)
if clean_product_codes and line_item["product_code"]:
if '\n' in line_item["product_code"]:
original_code = line_item["product_code"]
line_item["product_code"] = line_item["product_code"].split('\n')[-1].strip()
logger.debug(f"Cleaned product code: '{original_code}' -> '{line_item['product_code']}'")
# Fallback: extract product code from raw_content if Azure missed it
# Only use if first line looks like a product code (alphanumeric, no decimals, reasonable length)
line_item["product_code_fallback"] = False # Track if fallback was used
if not line_item.get("product_code") and line_item.get("raw_content"):
first_line = line_item["raw_content"].split('\n')[0].strip()
# Valid product code patterns:
# - All digits, 2-6 chars (e.g., "1646", "955")
# - Alphanumeric with optional hyphens, 2-15 chars, must contain at least one digit
# (e.g., "01SAL4K06", "ABC-123") — pure alpha words like "Sausages" are descriptions, not codes
# - Must NOT look like a price (no decimal point with 2 digits after)
is_numeric_code = re.match(r'^\d{2,6}$', first_line)
is_alphanum_code = (
re.match(r'^[A-Z0-9][A-Z0-9\-]{1,14}$', first_line, re.IGNORECASE)
and re.search(r'\d', first_line) # Must contain at least one digit
)
is_price = re.match(r'^\d+\.\d{2}$', first_line) # e.g., "14.64"
if (is_numeric_code or is_alphanum_code) and not is_price:
line_item["product_code"] = first_line
line_item["product_code_fallback"] = True
logger.info(f"Extracted product code from raw_content fallback: '{first_line}'")
# Extract description
line_item["description"] = safe_get_field(item_fields, "Description")
# Check for content vs value mismatch in Description field
# If they differ significantly, store the alternative for user selection
line_item["description_alt"] = None
if "Description" in item_fields:
desc_field = item_fields["Description"]
if hasattr(desc_field, 'content') and hasattr(desc_field, 'value'):
content = desc_field.content or ""
value = str(desc_field.value) if desc_field.value else ""
# Normalize for comparison (strip, lowercase)
content_norm = content.strip().lower()
value_norm = value.strip().lower()
# If first lines differ, store alternative
if content_norm.split('\n')[0] != value_norm.split('\n')[0]:
# Use value as description (current behavior), store content as alt
line_item["description_alt"] = content
logger.debug(f"Description mismatch detected: value='{value[:50]}...' content='{content[:50]}...'")
# Extract unit of measure
line_item["unit"] = safe_get_field(item_fields, "Unit")
# Extract quantity (delivered)
qty = safe_get_field(item_fields, "Quantity")
if qty is not None:
try:
line_item["quantity"] = float(qty)
except (ValueError, TypeError):
pass
# Fallback: parse quantity from leading number in raw_content
# e.g., "2\nHovis Soft White..." -> quantity = 2
if not line_item.get("quantity") and line_item.get("raw_content"):
leading_qty_match = re.match(r'^(\d+)\s*[\n\r]', line_item["raw_content"])
if leading_qty_match:
try:
line_item["quantity"] = float(leading_qty_match.group(1))
logger.debug(f"Extracted quantity {line_item['quantity']} from raw_content leading number")
except (ValueError, TypeError):
pass
# Extract order quantity (if available, e.g., Brakes invoices)
order_qty = safe_get_field(item_fields, "OrderQuantity")
if order_qty is not None:
try:
line_item["order_quantity"] = float(order_qty)
except (ValueError, TypeError):
pass
# Extract unit price
up_val = safe_get_field(item_fields, "UnitPrice")
if up_val is not None:
try:
if hasattr(up_val, 'amount') and up_val.amount is not None:
line_item["unit_price"] = float(up_val.amount)
else:
line_item["unit_price"] = float(up_val)
except (ValueError, TypeError):
pass
# Extract tax rate (e.g., "ZERO", "20.00", "No VAT")
tax_rate = safe_get_field(item_fields, "TaxRate")
if tax_rate is None:
# Fall back to Tax field which sometimes contains rate description
tax_rate = safe_get_field(item_fields, "Tax")
if tax_rate is not None:
line_item["tax_rate"] = str(tax_rate)
# Extract tax amount (if specified separately from rate)
tax_val = safe_get_field(item_fields, "Tax")
if tax_val is not None:
try:
if hasattr(tax_val, 'amount') and tax_val.amount is not None:
line_item["tax_amount"] = float(tax_val.amount)
elif isinstance(tax_val, (int, float)):
line_item["tax_amount"] = float(tax_val)
except (ValueError, TypeError):
pass
# Extract amount (line total - may be gross or net depending on supplier)
amt_val = safe_get_field(item_fields, "Amount")
if amt_val is not None:
try:
if hasattr(amt_val, 'amount') and amt_val.amount is not None:
line_item["amount"] = float(amt_val.amount)
else:
line_item["amount"] = float(amt_val)
except (ValueError, TypeError):
pass
# Check if amount is gross (inc VAT) or net (exc VAT)
# Compare against unit_price * quantity to determine
if line_item.get("amount") and line_item.get("tax_amount") and line_item.get("unit_price") and line_item.get("quantity"):
expected_net = line_item["unit_price"] * line_item["quantity"]
current_amount = line_item["amount"]
adjusted_amount = current_amount - line_item["tax_amount"]
# Check which is closer to expected net: current amount or adjusted amount
diff_current = abs(current_amount - expected_net)
diff_adjusted = abs(adjusted_amount - expected_net)
if diff_adjusted < diff_current and adjusted_amount > 0:
# Amount appears to be gross, adjust to net
line_item["amount"] = round(adjusted_amount, 2)
logger.debug(f"Adjusted line amount from gross {current_amount} to net {adjusted_amount} (expected ~{expected_net:.2f})")
# Parse pack size from raw content or description (e.g., "120x15g", "400ml")
pack_info = parse_pack_size(line_item["raw_content"] or line_item.get("description", ""))
line_item["pack_quantity"] = pack_info["pack_quantity"]
line_item["unit_size"] = pack_info["unit_size"]
line_item["unit_size_type"] = pack_info["unit_size_type"]
# If unit_price is missing, try to extract from description/raw_content
if not line_item.get("unit_price"):
# Look for "£X.XX each" or "£X.XX/each" patterns
price_each_pattern = r'£(\d+\.?\d*)\s*(?:each|/each|per\s*unit|ea\b)'
text_to_search = f"{line_item.get('description', '')} {line_item.get('raw_content', '')}"
price_match = re.search(price_each_pattern, text_to_search, re.IGNORECASE)
if price_match:
try:
line_item["unit_price"] = float(price_match.group(1))
logger.debug(f"Extracted unit_price £{line_item['unit_price']:.2f} from description/raw_content")
except (ValueError, TypeError):
pass
# If still no unit_price, back-calculate from amount and quantity
if not line_item.get("unit_price") and line_item.get("amount") and line_item.get("quantity"):
if line_item["quantity"] > 0:
line_item["unit_price"] = round(line_item["amount"] / line_item["quantity"], 2)
logger.debug(f"Back-calculated unit_price £{line_item['unit_price']:.2f} from amount/quantity")
# If still no quantity, back-calculate from amount and unit_price
if not line_item.get("quantity") and line_item.get("amount") and line_item.get("unit_price"):
if line_item["unit_price"] > 0:
calculated_qty = line_item["amount"] / line_item["unit_price"]
# Only use if result is close to a whole number (receipts usually have integer quantities)
if abs(calculated_qty - round(calculated_qty)) < 0.01:
line_item["quantity"] = round(calculated_qty)
logger.debug(f"Back-calculated quantity {line_item['quantity']} from amount/unit_price")
# Validate numeric values to prevent database overflow
# DECIMAL(10,3) allows up to 9,999,999.999 - cap at reasonable limits
MAX_QUANTITY = 999999.0
MAX_PRICE = 999999.0
MAX_UNIT_SIZE = 99999.0 # 99kg in grams is plenty
MAX_PACK_QTY = 9999
# Collect warnings for values that need capping (OCR misreads)
ocr_warnings = []
if line_item.get("quantity") and line_item["quantity"] > MAX_QUANTITY:
ocr_warnings.append(f"Quantity OCR error: {line_item['quantity']:.0f} → capped to {MAX_QUANTITY:.0f}")
logger.warning(f"Capping quantity {line_item['quantity']} to {MAX_QUANTITY}")
line_item["quantity"] = MAX_QUANTITY
if line_item.get("unit_price") and line_item["unit_price"] > MAX_PRICE:
ocr_warnings.append(f"Unit price OCR error: {line_item['unit_price']:.2f} → capped to {MAX_PRICE:.0f}")
logger.warning(f"Capping unit_price {line_item['unit_price']} to {MAX_PRICE}")
line_item["unit_price"] = MAX_PRICE
if line_item.get("amount") and line_item["amount"] > MAX_PRICE:
ocr_warnings.append(f"Amount OCR error: {line_item['amount']:.2f} → capped to {MAX_PRICE:.0f}")
logger.warning(f"Capping amount {line_item['amount']} to {MAX_PRICE}")
line_item["amount"] = MAX_PRICE
if line_item.get("unit_size") and line_item["unit_size"] > MAX_UNIT_SIZE:
ocr_warnings.append(f"Unit size OCR error: {line_item['unit_size']:.0f} → capped to {MAX_UNIT_SIZE:.0f}")
logger.warning(f"Capping unit_size {line_item['unit_size']} to {MAX_UNIT_SIZE}")
line_item["unit_size"] = MAX_UNIT_SIZE
if line_item.get("pack_quantity") and line_item["pack_quantity"] > MAX_PACK_QTY:
ocr_warnings.append(f"Pack qty OCR error: {line_item['pack_quantity']} → capped to {MAX_PACK_QTY}")
logger.warning(f"Capping pack_quantity {line_item['pack_quantity']} to {MAX_PACK_QTY}")
line_item["pack_quantity"] = MAX_PACK_QTY
# Track product code fallback extraction
if line_item.get("product_code_fallback"):
ocr_warnings.append(f"SKU extracted from raw content: {line_item['product_code']}")
# Store warnings in line item
if ocr_warnings:
line_item["ocr_warnings"] = "; ".join(ocr_warnings)
# Calculate cost per item if we have pack_quantity and unit_price
if pack_info["pack_quantity"] and line_item.get("unit_price"):
line_item["cost_per_item"] = round(
line_item["unit_price"] / pack_info["pack_quantity"], 4
)
# cost_per_portion NOT calculated here - requires portions_per_unit
# which is defined via product_definitions or manual entry
line_item["cost_per_portion"] = None
# Filter subtotal rows if enabled
# Must check ALL THREE conditions: has "Total" in description, no product_code, no quantity
if filter_subtotal_rows:
desc = (line_item.get("description") or "").lower()
has_total_keyword = "sub total" in desc or desc.endswith("total")
no_product_code = not line_item.get("product_code")
no_quantity = line_item.get("quantity") is None
if has_total_keyword and no_product_code and no_quantity:
logger.debug(f"Filtered subtotal row: {line_item.get('description')}")
continue # Skip this item
# Weight-based quantity adjustment for KG items
# When enabled: if unit is KG and qty × price ≠ amount, look for weight in raw_content
if use_weight_as_quantity:
unit = (line_item.get("unit") or "").upper()
qty = line_item.get("quantity")
price = line_item.get("unit_price")
amount = line_item.get("amount")
# Only process KG items with all values present
if unit == "KG" and qty is not None and price is not None and amount is not None:
# Check if current qty × price matches amount (with tolerance)
expected = qty * price
if abs(expected - amount) > 0.02: # Mismatch detected
# Try to parse weight from raw_content
raw = line_item.get("raw_content") or ""
# Pattern: weight on its own line (after newline) with space before KG
# This avoids matching product sizes like "1.25-1.65KG" in descriptions
weight_match = re.search(r'\n(\d+\.\d+)\s+KG', raw, re.IGNORECASE)
# Fallback: try matching at start of content (if weight is first)
if not weight_match:
weight_match = re.search(r'^(\d+\.\d+)\s+KG', raw, re.IGNORECASE)
if weight_match:
weight = float(weight_match.group(1))
# Validate: weight × price should equal amount
weight_expected = weight * price
if abs(weight_expected - amount) <= 0.02: # Match!
logger.info(f"Weight adjustment: qty {qty} -> {weight} KG (validated: {weight} × {price} = {amount})")
# Move original qty to order_quantity, use weight as quantity
line_item["order_quantity"] = qty
line_item["quantity"] = weight
else:
logger.debug(f"Weight {weight} × price {price} = {weight_expected} doesn't match amount {amount}, keeping original qty")
else:
logger.debug(f"No weight pattern found in raw_content for KG item with qty mismatch")
line_item["ocr_index"] = item_idx # Original OCR array index for preview
line_items.append(line_item)
desc_preview = (line_item.get('description') or 'N/A')[:30]
logger.debug(f"Line item {item_idx}: {desc_preview} - "
f"qty={line_item.get('quantity')} amt={line_item.get('amount')}")
except Exception as item_err:
logger.warning(f"Error processing line item {item_idx}: {item_err}")
# Calculate average confidence from extracted fields
# Collect confidence scores from key fields that were actually extracted
confidence_scores = []
if "InvoiceId" in fields and fields["InvoiceId"] and hasattr(fields["InvoiceId"], 'confidence'):
confidence_scores.append(fields["InvoiceId"].confidence)
if "InvoiceDate" in fields and fields["InvoiceDate"] and hasattr(fields["InvoiceDate"], 'confidence'):
confidence_scores.append(fields["InvoiceDate"].confidence)
if "InvoiceTotal" in fields and fields["InvoiceTotal"] and hasattr(fields["InvoiceTotal"], 'confidence'):
confidence_scores.append(fields["InvoiceTotal"].confidence)
if "SubTotal" in fields and fields["SubTotal"] and hasattr(fields["SubTotal"], 'confidence'):
confidence_scores.append(fields["SubTotal"].confidence)
if "VendorName" in fields and fields["VendorName"] and hasattr(fields["VendorName"], 'confidence'):
confidence_scores.append(fields["VendorName"].confidence)
if "PurchaseOrder" in fields and fields["PurchaseOrder"] and hasattr(fields["PurchaseOrder"], 'confidence'):
confidence_scores.append(fields["PurchaseOrder"].confidence)
# Calculate average confidence from extracted fields
if confidence_scores:
confidence = sum(confidence_scores) / len(confidence_scores)
logger.debug(f"Average field confidence: {confidence:.2%} from {len(confidence_scores)} fields")
else:
# Fallback to document-level confidence if no field confidences available
confidence = invoice.confidence if hasattr(invoice, 'confidence') else 0.9
logger.debug(f"Using document confidence: {confidence:.2%} (no field confidences)")
# Detect document type (invoice vs credit note)
document_type = "invoice" # default
# Check Azure's doc_type field
if hasattr(invoice, 'doc_type') and invoice.doc_type:
doc_type_str = str(invoice.doc_type).lower()
if 'credit' in doc_type_str:
document_type = "credit_note"
logger.debug(f"Detected credit note from Azure doc_type: {invoice.doc_type}")
# Check for negative total (strong indicator of credit note)
if total is not None and total < 0:
document_type = "credit_note"
logger.debug(f"Detected credit note from negative total: {total}")
elif net_total is not None and net_total < 0:
document_type = "credit_note"
logger.debug(f"Detected credit note from negative net_total: {net_total}")
# Check for credit note keywords in invoice number
if invoice_number:
inv_num_upper = invoice_number.upper()
credit_keywords = ['CREDIT', 'CR NOTE', 'CN', 'CREDIT NOTE', 'C/N']
if any(keyword in inv_num_upper for keyword in credit_keywords):
document_type = "credit_note"
logger.debug(f"Detected credit note from invoice number keywords: {invoice_number}")
logger.info(f"Azure extracted: invoice_number={invoice_number}, date={invoice_date}, "
f"total={total}, net_total={net_total}, vendor={vendor_name}, order={order_number}, "
f"{len(line_items)} total line items from {doc_count} documents, "
f"document_type={document_type}, confidence={confidence:.2%}")
return {
"invoice_number": invoice_number,
"invoice_date": invoice_date,
"total": total,
"net_total": net_total,
"vendor_name": vendor_name,
"order_number": order_number,
"document_type": document_type,
"line_items": line_items,
"raw_text": result.content or "",
"raw_json": raw_json,
"confidence": confidence
}

172
backend/ocr/extractor.py Normal file
View file

@ -0,0 +1,172 @@
import logging
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select
from .parser import identify_supplier
from .azure_extractor import process_invoice_with_azure
from services.duplicate_detector import detect_document_type
from services.pdf_rotation import rotate_pdf_pages
logger = logging.getLogger(__name__)
async def process_invoice_image(
image_path: str,
kitchen_id: int,
db: AsyncSession
) -> dict:
"""
Process an invoice image through Azure OCR and extract relevant fields.
Args:
image_path: Path to the invoice image
kitchen_id: ID of the kitchen for supplier matching
db: Database session for supplier lookup
Returns:
dict with extracted fields:
- invoice_number: str or None
- invoice_date: date or None
- total: Decimal or None
- supplier_id: int or None
- supplier_match_type: str ("exact", "fuzzy") or None
- vendor_name: str or None
- order_number: str or None
- document_type: str ("invoice" or "delivery_note")
- line_items: list
- raw_text: str
- raw_json: dict (serialized Azure response)
- confidence: float
"""
# Check kitchen settings for Azure credentials
from models.settings import KitchenSettings
settings_result = await db.execute(
select(KitchenSettings).where(KitchenSettings.kitchen_id == kitchen_id)
)
settings = settings_result.scalar_one_or_none()
# Require Azure configuration
if not settings or not settings.azure_endpoint or not settings.azure_key:
logger.error("Azure credentials not configured for kitchen")
return {
"invoice_number": None,
"invoice_date": None,
"total": None,
"net_total": None,
"supplier_id": None,
"supplier_match_type": None,
"vendor_name": None,
"order_number": None,
"document_type": "invoice",
"line_items": [],
"raw_text": "Error: Azure credentials not configured. Please configure in Settings.",
"raw_json": None,
"confidence": 0.0
}
logger.info("Using Azure Document Intelligence for OCR")
try:
result = await process_invoice_with_azure(
image_path,
settings.azure_endpoint,
settings.azure_key,
clean_product_codes=settings.ocr_clean_product_codes,
filter_subtotal_rows=settings.ocr_filter_subtotal_rows,
use_weight_as_quantity=settings.ocr_use_weight_as_quantity
)
# FIRST post-processing step: Rotate PDF pages with non-zero angles
# This must happen BEFORE any other processing extracts data from raw_json
# because coordinates need to be transformed to match the corrected orientation
if image_path.lower().endswith('.pdf') and result.get('raw_json'):
try:
modified, updated_json = rotate_pdf_pages(image_path, result['raw_json'])
if modified:
result['raw_json'] = updated_json
logger.info(f"PDF pages rotated and coordinates transformed for {image_path}")
except Exception as e:
logger.warning(f"PDF rotation failed (non-fatal): {e}")
# Try to identify/match supplier from vendor name
supplier_id = None
supplier_match_type = None
if result.get("vendor_name"):
supplier_id, supplier_match_type = await identify_supplier(result["vendor_name"], kitchen_id, db)
if not supplier_id and result.get("raw_text"):
supplier_id, supplier_match_type = await identify_supplier(result["raw_text"], kitchen_id, db)
# Use document_type from azure_extractor (already detected there)
# Fall back to detect_document_type only if not provided
document_type = result.get("document_type")
if not document_type:
document_type = detect_document_type(
result.get("raw_text", ""),
result
)
# LLM FEATURE — see LLM-MANIFEST.md for removal instructions
# Feature E: LLM fallback when Azure returns null for header fields
invoice_number = result.get("invoice_number")
invoice_date = result.get("invoice_date")
total = result.get("total")
any_null = invoice_number is None or invoice_date is None or total is None
if any_null and result.get("raw_text"):
try:
from services.llm_service import extract_invoice_fields_llm
llm_result = await extract_invoice_fields_llm(
db=db,
kitchen_id=kitchen_id,
raw_text=result["raw_text"],
)
if llm_result["status"] in ("success", "cached") and llm_result.get("fields"):
fields = llm_result["fields"]
if invoice_number is None and fields.get("invoice_number"):
invoice_number = fields["invoice_number"]
logger.info(f"LLM extracted invoice_number: {invoice_number}")
if invoice_date is None and fields.get("invoice_date"):
invoice_date = fields["invoice_date"]
logger.info(f"LLM extracted invoice_date: {invoice_date}")
if total is None and fields.get("total"):
total = fields["total"]
logger.info(f"LLM extracted total: {total}")
except Exception as llm_err:
logger.warning(f"LLM field extraction fallback failed (non-fatal): {llm_err}")
logger.info(f"Processed invoice: number={invoice_number}, "
f"type={document_type}, supplier_id={supplier_id}, match_type={supplier_match_type}")
return {
"invoice_number": invoice_number,
"invoice_date": invoice_date,
"total": total,
"net_total": result.get("net_total"),
"supplier_id": supplier_id,
"supplier_match_type": supplier_match_type,
"vendor_name": result.get("vendor_name"),
"order_number": result.get("order_number"),
"document_type": document_type,
"line_items": result.get("line_items", []),
"raw_text": result.get("raw_text", ""),
"raw_json": result.get("raw_json"),
"confidence": result.get("confidence", 0.0)
}
except Exception as e:
logger.error(f"Azure OCR failed: {e}")
return {
"invoice_number": None,
"invoice_date": None,
"total": None,
"net_total": None,
"supplier_id": None,
"supplier_match_type": None,
"vendor_name": None,
"order_number": None,
"document_type": "invoice",
"line_items": [],
"raw_text": f"Error: Azure OCR failed - {str(e)}",
"raw_json": None,
"confidence": 0.0
}

376
backend/ocr/parser.py Normal file
View file

@ -0,0 +1,376 @@
import re
from datetime import date, datetime
from decimal import Decimal, InvalidOperation
from typing import Optional
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select
# Default patterns for common invoice formats
DEFAULT_PATTERNS = {
"invoice_number": [
# "Invoice No: 12345" or "Invoice Number: ABC-123" or "Invoice #12345"
r"(?:Invoice|Inv)[\s.]*(?:No|Number|#)?[\s.:]*([A-Z0-9][A-Z0-9\-/]+)",
# "Invoice: 12345" - simple format
r"Invoice[:\s]+([A-Z0-9][A-Z0-9\-/]*\d+)",
# Just digits after Invoice keyword
r"(?:Invoice|Inv)[\s#.:]*(\d{3,})",
# Order/Reference numbers
r"(?:Order|Ref|Reference)[\s.]*(?:No|Number|#)?[\s.:]*([A-Z0-9\-/]+)",
# NUMBER: 12345
r"(?:NUMBER|NUMB)[:\s]*([0-9\-]+)",
# Standalone number patterns like "No. 12345" or "No: ABC-123"
r"(?:^|\s)No[.:\s]+([A-Z0-9][A-Z0-9\-/]+)",
],
# Patterns for invoice number on NEXT LINE after label
"invoice_number_multiline": [
# "Invoice\n12345" or "Invoice Number\n12345"
r"(?:Invoice|Inv)(?:\s*(?:No|Number|#))?[\s.:]*\n\s*([A-Z0-9][A-Z0-9\-/]+)",
# "Invoice No.\n12345"
r"Invoice\s*No\.?\s*\n\s*([A-Z0-9][A-Z0-9\-/]+)",
],
"date": [
# DD Mon YYYY (e.g., "15 Jan 2026", "15 January 2026")
r"(\d{1,2}\s*(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:t(?:ember)?)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\s*\d{2,4})",
# DD/MM/YYYY or MM/DD/YYYY
r"(\d{1,2}[/.-]\d{1,2}[/.-]\d{2,4})",
# YYYY-MM-DD
r"(\d{4}[/.-]\d{1,2}[/.-]\d{1,2})",
# Date: or Dated: prefix (same line)
r"(?:Date|Dated)[:\s]*(\d{1,2}[/.\-\s]+(?:\w+|\d{1,2})[/.\-\s]+\d{2,4})",
# Date label followed by date on next line
r"(?:Date|Dated|Invoice Date)[:\s]*\n\s*(\d{1,2}\s*(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:t(?:ember)?)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\s*\d{2,4})",
r"(?:Date|Dated|Invoice Date)[:\s]*\n\s*(\d{1,2}[/.-]\d{1,2}[/.-]\d{2,4})",
],
"total": [
r"(?:Total|Grand\s*Total|Amount\s*Due|Balance\s*Due)[:\s]*[£$€]?\s*([\d,]+\.?\d*)",
r"(?:TOTAL)[:\s]*[£$€]?\s*([\d,]+\.?\d*)",
r"[£$€]\s*([\d,]+\.\d{2})\s*$", # Currency at end of line
]
}
def extract_invoice_fields(raw_text: str, template_config: dict = None) -> dict:
"""
Extract invoice number, date, and total from OCR text.
Args:
raw_text: The raw OCR text
template_config: Optional supplier-specific patterns
Returns:
dict with invoice_number, invoice_date, total
"""
patterns = template_config if template_config else DEFAULT_PATTERNS
result = {
"invoice_number": extract_invoice_number(
raw_text,
patterns.get("invoice_number", DEFAULT_PATTERNS["invoice_number"]),
patterns.get("invoice_number_multiline", DEFAULT_PATTERNS.get("invoice_number_multiline", []))
),
"invoice_date": extract_date(raw_text, patterns.get("date", DEFAULT_PATTERNS["date"])),
"total": extract_total(raw_text, patterns.get("total", DEFAULT_PATTERNS["total"]))
}
return result
def extract_invoice_number(text: str, patterns: list, multiline_patterns: list = None) -> Optional[str]:
"""Extract invoice number using provided patterns"""
# Try single-line patterns first
for pattern in patterns:
match = re.search(pattern, text, re.IGNORECASE | re.MULTILINE)
if match:
result = match.group(1).strip()
# Skip if we just captured the word "Invoice" or similar
if result.lower() not in ('invoice', 'inv', 'number', 'no'):
return result
# Try multiline patterns (number on next line after label)
if multiline_patterns:
for pattern in multiline_patterns:
match = re.search(pattern, text, re.IGNORECASE | re.MULTILINE)
if match:
result = match.group(1).strip()
if result.lower() not in ('invoice', 'inv', 'number', 'no'):
return result
return None
def extract_date(text: str, patterns: list) -> Optional[date]:
"""Extract and parse date from text"""
for pattern in patterns:
match = re.search(pattern, text, re.IGNORECASE)
if match:
date_str = match.group(1)
parsed = parse_date_string(date_str)
if parsed:
return parsed
return None
def parse_date_string(date_str: str) -> Optional[date]:
"""Parse various date formats"""
# Normalize whitespace - OCR sometimes adds extra spaces
date_str = ' '.join(date_str.strip().split())
# Common date formats to try
formats = [
"%d/%m/%Y", "%d-%m-%Y", "%d.%m.%Y", # DD/MM/YYYY
"%m/%d/%Y", "%m-%d-%Y", "%m.%d.%Y", # MM/DD/YYYY
"%Y-%m-%d", "%Y/%m/%d", "%Y.%m.%d", # YYYY-MM-DD
"%d/%m/%y", "%d-%m-%y", "%d.%m.%y", # DD/MM/YY
"%d %b %Y", "%d %B %Y", # DD Mon YYYY
"%d %b %y", "%d %B %y", # DD Mon YY
"%d%b%Y", "%d%B%Y", # DDMonYYYY (no spaces)
"%d%b%y", "%d%B%y", # DDMonYY (no spaces)
]
for fmt in formats:
try:
parsed = datetime.strptime(date_str, fmt)
return parsed.date()
except ValueError:
continue
# Try to handle variations like "15Jan2026" or extra characters
# Remove common noise characters
cleaned = re.sub(r'[,]', ' ', date_str)
cleaned = ' '.join(cleaned.split())
if cleaned != date_str:
for fmt in formats:
try:
parsed = datetime.strptime(cleaned, fmt)
return parsed.date()
except ValueError:
continue
return None
def extract_total(text: str, patterns: list) -> Optional[Decimal]:
"""Extract total amount from text"""
# Try each pattern
for pattern in patterns:
matches = re.findall(pattern, text, re.IGNORECASE | re.MULTILINE)
for match in matches:
amount = parse_amount(match)
if amount and amount > Decimal("0"):
return amount
# Fallback: look for the largest currency amount
currency_pattern = r"[£$€]?\s*([\d,]+\.\d{2})"
amounts = re.findall(currency_pattern, text)
if amounts:
parsed_amounts = [parse_amount(a) for a in amounts]
valid_amounts = [a for a in parsed_amounts if a and a > Decimal("0")]
if valid_amounts:
# Return the largest amount (likely the total)
return max(valid_amounts)
return None
def parse_amount(amount_str: str) -> Optional[Decimal]:
"""Parse amount string to Decimal"""
try:
# Remove currency symbols and commas
cleaned = re.sub(r"[£$€,\s]", "", amount_str)
if cleaned:
return Decimal(cleaned)
except (InvalidOperation, ValueError):
pass
return None
def normalize_text(text: str) -> str:
"""Normalize text for fuzzy matching - lowercase, remove punctuation, extra spaces"""
import string
text = text.lower()
text = text.translate(str.maketrans('', '', string.punctuation))
text = ' '.join(text.split())
return text
def get_words(text: str) -> set:
"""Get set of significant words (3+ chars) from text"""
return {w for w in normalize_text(text).split() if len(w) >= 3}
def fuzzy_match_score(supplier_name: str, text: str) -> float:
"""
Calculate fuzzy match score between supplier name and text.
Returns score from 0.0 to 1.0
Requires strong evidence of match - simple word overlap is not enough.
"""
supplier_norm = normalize_text(supplier_name)
text_norm = normalize_text(text)
# Check if supplier name is contained in text (high confidence)
if supplier_norm in text_norm:
return 0.95
# Get significant words (4+ chars to avoid common short words like "the", "ltd", "and")
supplier_words = {w for w in normalize_text(supplier_name).split() if len(w) >= 4}
text_words = {w for w in normalize_text(text).split() if len(w) >= 4}
if not supplier_words:
# Fall back to 3+ char words if no 4+ char words
supplier_words = get_words(supplier_name)
if not supplier_words:
return 0.0
# Count how many supplier words appear in text
matching_words = supplier_words & text_words
# No matches at all = no fuzzy match
if not matching_words:
return 0.0
# Calculate base score from word overlap
word_score = len(matching_words) / len(supplier_words)
# Bonus: Check if first word (company name) matches - this is most important
supplier_first = supplier_norm.split()[0] if supplier_norm.split() else ""
first_word_matches = False
if supplier_first and len(supplier_first) >= 4:
if supplier_first in text_norm:
first_word_matches = True
word_score = max(word_score, 0.75)
# Require either:
# - First word matching (strong signal), OR
# - Multiple words matching (at least 2)
# Single non-first word matches are not reliable
if not first_word_matches and len(matching_words) < 2:
# Single word match (not first word) - reduce confidence significantly
word_score = word_score * 0.5
return word_score
async def identify_supplier(
text: str,
kitchen_id: int,
db: AsyncSession
) -> tuple[Optional[int], Optional[str]]:
"""
Try to identify the supplier from OCR text.
Match types:
- "exact": vendor_name exactly equals supplier name or alias (case-insensitive)
- "fuzzy": supplier name/alias is contained in text, or fuzzy word matching
Returns:
tuple of (supplier_id, match_type) where match_type is "exact", "fuzzy", or None
"""
from models.supplier import Supplier
result = await db.execute(
select(Supplier).where(Supplier.kitchen_id == kitchen_id)
)
suppliers = result.scalars().all()
text_normalized = normalize_text(text)
text_upper = text.upper()
# First pass: TRUE exact matches (text equals name/alias exactly)
for supplier in suppliers:
supplier_norm = normalize_text(supplier.name)
# Check if text exactly equals supplier name
if text_normalized == supplier_norm:
return (supplier.id, "exact")
# Check if text exactly equals any alias
aliases = supplier.aliases or []
for alias in aliases:
if text_normalized == normalize_text(alias):
return (supplier.id, "exact")
# Second pass: "contains" matches - name/alias found IN text (fuzzy, not exact)
for supplier in suppliers:
# Check if supplier name is contained in text
if supplier.name.upper() in text_upper:
return (supplier.id, "fuzzy")
# Check aliases contained in text
aliases = supplier.aliases or []
for alias in aliases:
if alias.upper() in text_upper:
return (supplier.id, "fuzzy")
# Check identifier_config keywords
identifier_config = supplier.identifier_config or {}
keywords = identifier_config.get("keywords", [])
for keyword in keywords:
if keyword.upper() in text_upper:
return (supplier.id, "fuzzy")
# Third pass: fuzzy word matching
# Only do fuzzy matching on vendor name-like text (short text, not full OCR dump)
# Full OCR text has too many words that could accidentally match
if len(text) > 500:
# Text is too long - likely full OCR text, skip fuzzy matching
return (None, None)
best_match = None
best_score = 0.0
FUZZY_THRESHOLD = 0.6 # Minimum score to consider a fuzzy match
for supplier in suppliers:
# Check supplier name fuzzy match
score = fuzzy_match_score(supplier.name, text)
if score > best_score and score >= FUZZY_THRESHOLD:
best_score = score
best_match = supplier.id
# Check aliases fuzzy match
aliases = supplier.aliases or []
for alias in aliases:
score = fuzzy_match_score(alias, text)
if score > best_score and score >= FUZZY_THRESHOLD:
best_score = score
best_match = supplier.id
if best_match:
return (best_match, "fuzzy")
# LLM FEATURE — see LLM-MANIFEST.md for removal instructions
# Feature F: LLM fallback when all regex/fuzzy passes fail
try:
from services.llm_service import match_supplier_llm
supplier_list = [{"id": s.id, "name": s.name} for s in suppliers]
llm_result = await match_supplier_llm(
db=db,
kitchen_id=kitchen_id,
vendor_text=text[:500],
supplier_list=supplier_list,
)
if llm_result["status"] in ("success", "cached") and llm_result.get("match"):
return (llm_result["match"]["id"], "fuzzy")
except Exception:
pass # Non-fatal — fall through to None
return (None, None)
def build_supplier_template(sample_texts: list[str], extracted_values: list[dict]) -> dict:
"""
Helper to build supplier template from sample invoices.
Args:
sample_texts: List of OCR texts from sample invoices
extracted_values: List of known correct values for each sample
Returns:
Template config with patterns
"""
# This would analyze patterns across samples to build regex
# For now, return default patterns
return DEFAULT_PATTERNS