diff --git a/lending-poc/app/main.py b/lending-poc/app/main.py index dc9c4d0..5b70c89 100644 --- a/lending-poc/app/main.py +++ b/lending-poc/app/main.py @@ -4,10 +4,10 @@ from fastapi import FastAPI from sqlalchemy import text -from app.api.cases import router as cases_router from app.api.health import router as health_router from app.config import logger, settings from app.database import async_session, engine +from cross_document_validation.api import router as cases_router @asynccontextmanager diff --git a/lending-poc/app/matching/__init__.py b/lending-poc/app/models/__init__.py similarity index 100% rename from lending-poc/app/matching/__init__.py rename to lending-poc/app/models/__init__.py diff --git a/lending-poc/app/services/__init__.py b/lending-poc/cross_document_validation/__init__.py similarity index 100% rename from lending-poc/app/services/__init__.py rename to lending-poc/cross_document_validation/__init__.py diff --git a/lending-poc/app/api/cases.py b/lending-poc/cross_document_validation/api.py similarity index 76% rename from lending-poc/app/api/cases.py rename to lending-poc/cross_document_validation/api.py index 4d32522..5b3feea 100644 --- a/lending-poc/app/api/cases.py +++ b/lending-poc/cross_document_validation/api.py @@ -2,11 +2,11 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.database import get_db -from app.schemas.case import CaseCreateRequest, CaseCreateResponse, ValidationResultOut -from app.services.case_parsing import parse_case -from app.services.persistence import save_pipeline_result -from app.services.pipeline import run_pipeline -from app.utils.json_safe import json_safe +from cross_document_validation.schemas.case import CaseCreateRequest, CaseCreateResponse, ValidationResultOut +from cross_document_validation.services.case_parsing import parse_case +from cross_document_validation.services.persistence import save_pipeline_result +from cross_document_validation.services.pipeline import run_pipeline +from cross_document_validation.utils.json_safe import json_safe router = APIRouter(tags=["cases"]) diff --git a/lending-poc/cross_document_validation/matching/__init__.py b/lending-poc/cross_document_validation/matching/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lending-poc/app/matching/embeddings.py b/lending-poc/cross_document_validation/matching/embeddings.py similarity index 100% rename from lending-poc/app/matching/embeddings.py rename to lending-poc/cross_document_validation/matching/embeddings.py diff --git a/lending-poc/app/matching/exact.py b/lending-poc/cross_document_validation/matching/exact.py similarity index 100% rename from lending-poc/app/matching/exact.py rename to lending-poc/cross_document_validation/matching/exact.py diff --git a/lending-poc/app/matching/fuzzy.py b/lending-poc/cross_document_validation/matching/fuzzy.py similarity index 100% rename from lending-poc/app/matching/fuzzy.py rename to lending-poc/cross_document_validation/matching/fuzzy.py diff --git a/lending-poc/cross_document_validation/schemas/__init__.py b/lending-poc/cross_document_validation/schemas/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lending-poc/app/schemas/case.py b/lending-poc/cross_document_validation/schemas/case.py similarity index 100% rename from lending-poc/app/schemas/case.py rename to lending-poc/cross_document_validation/schemas/case.py diff --git a/lending-poc/cross_document_validation/services/__init__.py b/lending-poc/cross_document_validation/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lending-poc/app/services/business_validation.py b/lending-poc/cross_document_validation/services/business_validation.py similarity index 89% rename from lending-poc/app/services/business_validation.py rename to lending-poc/cross_document_validation/services/business_validation.py index ed58b07..c048023 100644 --- a/lending-poc/app/services/business_validation.py +++ b/lending-poc/cross_document_validation/services/business_validation.py @@ -6,9 +6,9 @@ from calendar import monthrange from datetime import date, timedelta -from app.matching.fuzzy import employer_similarity -from app.services import validation_config as cfg -from app.services.dto import ( +from cross_document_validation.matching.fuzzy import employer_similarity +from cross_document_validation.services import validation_config as cfg +from cross_document_validation.services.dto import ( BankStatementDoc, BankTransaction, CaseInput, @@ -135,7 +135,12 @@ def _validate_salary_slip( score=0.0, document_id=slip.doc_id, failure_reason="no_matching_credit_in_window", - evidence={"window": window}, + evidence={ + "source_text": slip.salary_month.strftime("%B %Y"), + "target_text": None, + "match_type": "DATE_MATCH", + "window": window, + }, ) txn, score = selection @@ -145,7 +150,12 @@ def _validate_salary_slip( passed=True, score=score, document_id=slip.doc_id, - evidence={"matched_transaction": txn}, + evidence={ + "source_text": slip.salary_month.strftime("%B %Y"), + "target_text": txn.txn_date.strftime("%Y-%m") if txn.txn_date else None, + "match_type": "DATE_MATCH", + "matched_transaction": txn, + }, ) @@ -187,6 +197,11 @@ def _employer_match_for_slip( score=score, document_id=slip.doc_id, failure_reason=None if passed else "employer_narration_mismatch", + evidence={ + "source_text": slip.employer_name, + "target_text": matched_txn.narration, + "match_type": "FUZZY", + }, ) @@ -206,9 +221,12 @@ def _salary_credit_count( check_type=CheckType.SALARY_CREDIT_COUNT, passed=(matched_slips == total_slips), score=confidence_score, + document_id=bank_statement.doc_id, evidence={ + "source_value": total_slips, + "target_value": matched_slips, + "match_type": "COUNT_MATCH", "stmt_duration": stmt_duration, - "no_of_matches": matched_slips, "total_slips": total_slips, "confidence_score": confidence_score, }, diff --git a/lending-poc/app/services/case_parsing.py b/lending-poc/cross_document_validation/services/case_parsing.py similarity index 98% rename from lending-poc/app/services/case_parsing.py rename to lending-poc/cross_document_validation/services/case_parsing.py index b8a0c33..d9c3a2b 100644 --- a/lending-poc/app/services/case_parsing.py +++ b/lending-poc/cross_document_validation/services/case_parsing.py @@ -5,7 +5,7 @@ from datetime import date, datetime -from app.services.dto import ( +from cross_document_validation.services.dto import ( AadhaarDoc, AddressProofDoc, BankStatementDoc, diff --git a/lending-poc/app/services/decision_engine.py b/lending-poc/cross_document_validation/services/decision_engine.py similarity index 88% rename from lending-poc/app/services/decision_engine.py rename to lending-poc/cross_document_validation/services/decision_engine.py index be8abd5..9b55ffd 100644 --- a/lending-poc/app/services/decision_engine.py +++ b/lending-poc/cross_document_validation/services/decision_engine.py @@ -1,7 +1,7 @@ """Final PASS / FAIL / NEEDS_REVIEW logic.""" -from app.services import validation_config as cfg -from app.services.dto import CheckType, Decision, DecisionResult, ScoreResult, ValidationResult +from cross_document_validation.services import validation_config as cfg +from cross_document_validation.services.dto import CheckType, Decision, DecisionResult, ScoreResult, ValidationResult MANDATORY_CHECK_TYPES = {CheckType.NAME, CheckType.AADHAAR, CheckType.PAN, CheckType.DOB} diff --git a/lending-poc/app/services/dto.py b/lending-poc/cross_document_validation/services/dto.py similarity index 100% rename from lending-poc/app/services/dto.py rename to lending-poc/cross_document_validation/services/dto.py diff --git a/lending-poc/app/services/golden_record.py b/lending-poc/cross_document_validation/services/golden_record.py similarity index 91% rename from lending-poc/app/services/golden_record.py rename to lending-poc/cross_document_validation/services/golden_record.py index c676e81..9e47e2c 100644 --- a/lending-poc/app/services/golden_record.py +++ b/lending-poc/cross_document_validation/services/golden_record.py @@ -11,10 +11,10 @@ rather than silently adopting an unrelated "fuller" name. """ -from app.matching.embeddings import get_address_embedding -from app.matching.fuzzy import name_similarity -from app.services import validation_config as cfg -from app.services.dto import CaseInput, GoldenRecord +from cross_document_validation.matching.embeddings import get_address_embedding +from cross_document_validation.matching.fuzzy import name_similarity +from cross_document_validation.services import validation_config as cfg +from cross_document_validation.services.dto import CaseInput, GoldenRecord FULLER_NAME_RELATEDNESS_THRESHOLD = cfg.NAME_MATCH_THRESHOLD diff --git a/lending-poc/app/services/identity_validation.py b/lending-poc/cross_document_validation/services/identity_validation.py similarity index 79% rename from lending-poc/app/services/identity_validation.py rename to lending-poc/cross_document_validation/services/identity_validation.py index 8b544a9..5e6eb71 100644 --- a/lending-poc/app/services/identity_validation.py +++ b/lending-poc/cross_document_validation/services/identity_validation.py @@ -3,10 +3,10 @@ the Golden Record at all, regardless of why they're missing. """ -from app.matching import exact, fuzzy -from app.matching.embeddings import address_similarity -from app.services import validation_config as cfg -from app.services.dto import CaseInput, CheckType, GoldenRecord, ValidationResult +from cross_document_validation.matching import exact, fuzzy +from cross_document_validation.matching.embeddings import address_similarity +from cross_document_validation.services import validation_config as cfg +from cross_document_validation.services.dto import CaseInput, CheckType, GoldenRecord, ValidationResult MANDATORY_GOLDEN_FIELDS = { CheckType.NAME: "name", @@ -31,7 +31,9 @@ def check_mandatory_presence(golden: GoldenRecord) -> list[ValidationResult]: return results -def _exact_result_to_validation(check_type: CheckType, outcome, document_id: str) -> ValidationResult: +def _exact_result_to_validation( + check_type: CheckType, outcome, document_id: str, source_text, target_text +) -> ValidationResult: passed = outcome.result == exact.MatchResult.MATCH score = 100.0 if passed else (50.0 if outcome.result == exact.MatchResult.INCONCLUSIVE else 0.0) return ValidationResult( @@ -40,6 +42,7 @@ def _exact_result_to_validation(check_type: CheckType, outcome, document_id: str score=score, document_id=document_id, failure_reason=None if passed else outcome.reason, + evidence={"source_text": source_text, "target_text": target_text, "match_type": "EXACT"}, ) @@ -64,6 +67,7 @@ def validate_document_against_golden( score=score, document_id=document_id, failure_reason=None if passed else "name_below_threshold", + evidence={"source_text": golden.name, "target_text": doc_name, "match_type": "FUZZY"}, ) ) @@ -78,20 +82,31 @@ def validate_document_against_golden( score=score, document_id=document_id, failure_reason=None if passed else "address_below_threshold", + evidence={"source_text": golden.address, "target_text": doc_address, "match_type": "SEMANTIC"}, ) ) if doc_aadhaar is not None: outcome = exact.aadhaar_match(golden.aadhaar_number, doc_aadhaar) - results.append(_exact_result_to_validation(CheckType.AADHAAR, outcome, document_id)) + results.append( + _exact_result_to_validation( + CheckType.AADHAAR, outcome, document_id, golden.aadhaar_number, doc_aadhaar + ) + ) if doc_pan is not None: outcome = exact.pan_match(golden.pan_number, doc_pan) - results.append(_exact_result_to_validation(CheckType.PAN, outcome, document_id)) + results.append( + _exact_result_to_validation(CheckType.PAN, outcome, document_id, golden.pan_number, doc_pan) + ) if doc_dob is not None: outcome = exact.dob_match(golden.date_of_birth, doc_dob) - results.append(_exact_result_to_validation(CheckType.DOB, outcome, document_id)) + results.append( + _exact_result_to_validation( + CheckType.DOB, outcome, document_id, golden.date_of_birth, doc_dob + ) + ) return results diff --git a/lending-poc/app/services/persistence.py b/lending-poc/cross_document_validation/services/persistence.py similarity index 91% rename from lending-poc/app/services/persistence.py rename to lending-poc/cross_document_validation/services/persistence.py index 6e41534..011a9a5 100644 --- a/lending-poc/app/services/persistence.py +++ b/lending-poc/cross_document_validation/services/persistence.py @@ -1,9 +1,9 @@ """Persists one pipeline run (CaseInput + PipelineResult) to the database. -The in-memory dataclasses in app.services.dto reference documents by their -string doc_id (e.g. "AADHAAR", "SALARY_SLIP-0"). This module inserts the -Document rows first and keeps a doc_id -> Document.id map so -ValidationResult.document_id (also a doc_id string) can be resolved to the +The in-memory dataclasses in cross_document_validation.services.dto reference +documents by their string doc_id (e.g. "AADHAAR", "SALARY_SLIP-0"). This +module inserts the Document rows first and keeps a doc_id -> Document.id map +so ValidationResult.document_id (also a doc_id string) can be resolved to the real foreign key. """ @@ -11,9 +11,9 @@ from sqlalchemy.ext.asyncio import AsyncSession -from app.services.dto import CaseInput, Decision, DocType -from app.services.dto import PipelineResult as PipelineResultDTO -from app.utils.json_safe import json_safe +from cross_document_validation.services.dto import CaseInput, Decision, DocType +from cross_document_validation.services.dto import PipelineResult as PipelineResultDTO +from cross_document_validation.utils.json_safe import json_safe from db.models.case import Case, CaseStatus from db.models.document import Document from db.models.golden_record import GoldenRecord as GoldenRecordModel diff --git a/lending-poc/app/services/pipeline.py b/lending-poc/cross_document_validation/services/pipeline.py similarity index 83% rename from lending-poc/app/services/pipeline.py rename to lending-poc/cross_document_validation/services/pipeline.py index bdee270..78af125 100644 --- a/lending-poc/app/services/pipeline.py +++ b/lending-poc/cross_document_validation/services/pipeline.py @@ -2,10 +2,10 @@ Decision in one call, with a simple in-memory audit log. """ -from app.services import business_validation, decision_engine, golden_record, scoring -from app.services import validation_config as cfg -from app.services.dto import CaseInput, Decision, DecisionResult, PipelineResult -from app.services.identity_validation import run_identity_validation +from cross_document_validation.services import business_validation, decision_engine, golden_record, scoring +from cross_document_validation.services import validation_config as cfg +from cross_document_validation.services.dto import CaseInput, Decision, DecisionResult, PipelineResult +from cross_document_validation.services.identity_validation import run_identity_validation def run_pipeline(case: CaseInput) -> PipelineResult: diff --git a/lending-poc/app/services/scoring.py b/lending-poc/cross_document_validation/services/scoring.py similarity index 87% rename from lending-poc/app/services/scoring.py rename to lending-poc/cross_document_validation/services/scoring.py index 6f79533..0e3f0eb 100644 --- a/lending-poc/app/services/scoring.py +++ b/lending-poc/cross_document_validation/services/scoring.py @@ -2,8 +2,8 @@ from collections import defaultdict -from app.services import validation_config as cfg -from app.services.dto import ScoreResult, ValidationResult +from cross_document_validation.services import validation_config as cfg +from cross_document_validation.services.dto import ScoreResult, ValidationResult def compute_score(validation_results: list[ValidationResult]) -> ScoreResult: diff --git a/lending-poc/app/services/validation_config.py b/lending-poc/cross_document_validation/services/validation_config.py similarity index 100% rename from lending-poc/app/services/validation_config.py rename to lending-poc/cross_document_validation/services/validation_config.py diff --git a/lending-poc/cross_document_validation/utils/__init__.py b/lending-poc/cross_document_validation/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lending-poc/app/utils/json_safe.py b/lending-poc/cross_document_validation/utils/json_safe.py similarity index 100% rename from lending-poc/app/utils/json_safe.py rename to lending-poc/cross_document_validation/utils/json_safe.py diff --git a/lending-poc/db/migrations/versions/0006_make_document_source_file_ref_nullable.py b/lending-poc/db/migrations/versions/0006_make_document_source_file_ref_nullable.py deleted file mode 100644 index 3189c29..0000000 --- a/lending-poc/db/migrations/versions/0006_make_document_source_file_ref_nullable.py +++ /dev/null @@ -1,25 +0,0 @@ -"""make document source_file_ref nullable - -Revision ID: 0006_doc_source_ref_nullable -Revises: 0005_add_validation_results -Create Date: 2026-08-13 00:00:00.000000 - -""" -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa - -# revision identifiers, used by Alembic. -revision: str = "0006_doc_source_ref_nullable" -down_revision: Union[str, None] = "0005_add_validation_results" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.alter_column("documents", "source_file_ref", existing_type=sa.String(), nullable=True) - - -def downgrade() -> None: - op.alter_column("documents", "source_file_ref", existing_type=sa.String(), nullable=False) diff --git a/lending-poc/docs/cases_api.md b/lending-poc/docs/cases_api.md index 9bca958..78873e1 100644 --- a/lending-poc/docs/cases_api.md +++ b/lending-poc/docs/cases_api.md @@ -92,14 +92,37 @@ Notes: "passed": true, "score": 94.5, "document_id": "PAN", - "evidence": null + "evidence": { + "source_text": "Sneha Sunil Lokhande", + "target_text": "Sneha Lokhande", + "match_type": "FUZZY" + } }, { "check_type": "SALARY_DATE", "passed": false, "score": 0.0, "document_id": "SALARY_SLIP-3", - "evidence": { "window": ["2026-05-27", "2026-07-31"] } + "evidence": { + "source_text": "June 2026", + "target_text": null, + "match_type": "DATE_MATCH", + "window": ["2026-05-27", "2026-07-31"] + } + }, + { + "check_type": "SALARY_CREDIT_COUNT", + "passed": false, + "score": 75.0, + "document_id": "BANK_STATEMENT", + "evidence": { + "source_value": 4, + "target_value": 3, + "match_type": "COUNT_MATCH", + "stmt_duration": { "start": "2026-04-01", "end": "2026-07-04" }, + "total_slips": 4, + "confidence_score": 75.0 + } } ] } diff --git a/lending-poc/docs/features.md b/lending-poc/docs/features.md index 4b5be5f..f7d7166 100644 --- a/lending-poc/docs/features.md +++ b/lending-poc/docs/features.md @@ -4,7 +4,7 @@ This document explains, in detail, what the validation pipeline behind `POST /ca ## 1. Pipeline overview -`app/services/pipeline.py` orchestrates one end-to-end run in this order: +`cross_document_validation/services/pipeline.py` orchestrates one end-to-end run in this order: ``` INGEST -> required-document precheck -> GOLDEN RECORD -> IDENTITY VALIDATION @@ -17,7 +17,7 @@ Every step appends a line to an in-memory `audit_log`, giving a readable trail o Before anything else runs, the pipeline checks that all of `AADHAAR`, `PAN`, `SALARY_SLIP`, `BANK_STATEMENT` are present (`validation_config.REQUIRED_DOCUMENT_TYPES`). If any are missing, the pipeline short-circuits with `decision=FAIL`, `overall_score=0.0`, and reasons like `MISSING_DOCUMENT:PAN` — no golden record or checks are computed. -## 2. Golden Record (`app/services/golden_record.py`) +## 2. Golden Record (`cross_document_validation/services/golden_record.py`) The Golden Record is the single trusted identity profile for the applicant, built by merging the KYC documents: @@ -29,11 +29,11 @@ The Golden Record is the single trusted identity profile for the applicant, buil - If both have a name, and they're recognizably the same person (`fuzzy.name_similarity` >= `NAME_MATCH_THRESHOLD`, 85), the **fuller** name (more tokens) wins — e.g. "Sneha Sunil Lokhande" over "Sneha Lokhande" — because it carries strictly more identity information. - If the two names *aren't* recognizably related, Aadhaar stays authoritative and the mismatch is left for the NAME identity check to flag, rather than silently trusting an unrelated "fuller" name. - The chosen name is split into `first_name` / `middle_name` / `last_name`. -- If an address was resolved, an address embedding is computed (`app.matching.embeddings.get_address_embedding`) and stored for later similarity checks. +- If an address was resolved, an address embedding is computed (`cross_document_validation.matching.embeddings.get_address_embedding`) and stored for later similarity checks. Each golden field also records its `*_source` (which document it came from), useful for traceability. -## 3. Identity Validation (`app/services/identity_validation.py`) +## 3. Identity Validation (`cross_document_validation/services/identity_validation.py`) Two parts: @@ -53,12 +53,12 @@ Every document that carries an identity field is compared against the Golden Rec | Each SALARY_SLIP | name | | BANK_STATEMENT | name | -Matching strategies (`app/matching/`): +Matching strategies (`cross_document_validation/matching/`): - **NAME** — fuzzy string similarity (`fuzzy.name_similarity`), handles reordering (surname-first), initials, and minor spelling differences. Passes at >= 85. - **ADDRESS** — embedding cosine similarity (`embeddings.address_similarity`), tolerant of differently-worded but equivalent addresses (e.g. "Apartment" vs "Flat", "MH" vs "Maharashtra"). Passes at >= 0.55 similarity (scored as similarity × 100). -- **AADHAAR / PAN / DOB** — exact matching (`app.matching.exact`). Result is `MATCH` (score 100), `NO_MATCH` (score 0), or `INCONCLUSIVE` (score 50, e.g. one side missing/unparseable). +- **AADHAAR / PAN / DOB** — exact matching (`cross_document_validation.matching.exact`). Result is `MATCH` (score 100), `NO_MATCH` (score 0), or `INCONCLUSIVE` (score 50, e.g. one side missing/unparseable). -## 4. Business Validation (`app/services/business_validation.py`) +## 4. Business Validation (`cross_document_validation/services/business_validation.py`) Verifies that declared income (salary slips) is corroborated by actual bank activity. Only runs if both salary slips and a bank statement are present. @@ -81,7 +81,7 @@ Each slip's declared `employer_name` is compared — via fuzzy similarity — on An aggregate check: `matched_slips / total_slips × 100`. It passes only if *every* slip matched a transaction, but a partial match (e.g. 3 of 4 months) doesn't hard-fail the case — it only lowers this component's score, which feeds into the weighted overall score. Evidence includes the bank statement's observed date range and match counts. -## 5. Scoring (`app/services/scoring.py`) +## 5. Scoring (`cross_document_validation/services/scoring.py`) Given every `ValidationResult` produced above: 1. Scores are grouped by `check_type` and averaged (e.g. if 4 salary slips each produced a `SALARY_DATE` score, they're averaged into one `SALARY_DATE` component score). @@ -99,7 +99,7 @@ Given every `ValidationResult` produced above: 3. The overall score is the weighted average, **renormalized over only the check types actually observed** in this case (so a case missing an optional check type doesn't get unfairly diluted by a zero for a check that never ran). Note `SALARY_DATE` itself isn't in the weight table — it gates whether a credit was found at all, but the weighted score is driven by `EMPLOYER` and `SALARY_CREDIT_COUNT`. -## 6. Decision Engine (`app/services/decision_engine.py`) +## 6. Decision Engine (`cross_document_validation/services/decision_engine.py`) Final decision logic, in priority order: @@ -108,7 +108,7 @@ Final decision logic, in priority order: 3. **FAIL** — if `overall_score < DECISION_FAIL_THRESHOLD` (60). 4. **NEEDS_REVIEW** — anything in between (60–90). Reasons list every individual failing check as `:`. -## 7. Persistence (`app/services/persistence.py`) +## 7. Persistence (`cross_document_validation/services/persistence.py`) A successful pipeline run is persisted in a single DB transaction: - One `Case` row (`applicant_ref`, `status` derived from the decision: PASS/FAIL/NEEDS_REVIEW). @@ -129,7 +129,7 @@ Document primary keys are resolved via an in-memory `doc_id -> Document.id` map | `SALARY_SLIP` | Declared income; multiple allowed per case (one per month) | | `BANK_STATEMENT` | Source of truth for actual salary credits | -## 9. Configuration reference (`app/services/validation_config.py`) +## 9. Configuration reference (`cross_document_validation/services/validation_config.py`) All thresholds/weights are centralized here as plain constants (intended to move into `app/config.py` / environment-driven settings as the app matures, without touching service logic):