From 4eaad6f4796d09fcc9d8d48b7c25539b01ba1c59 Mon Sep 17 00:00:00 2001 From: JuliaEdom Date: Sat, 4 Jul 2026 23:20:56 +0300 Subject: [PATCH 1/7] fix(g2-b1): live event-generator emits kitchen/RUB, not generic USD MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BLOCKER B1 from G2_AUDIT_REPORT.md. The live producer + edge emitter (activated on the F2 3-Space deploy) drove a generic USD electronics catalog into the serving витрина, overwriting the kitchen-legend SKUs, breaking bimodality and the out-of-stock-bestseller story. - event_producer.PRODUCT_CATALOG -> the 10 pinned kitchen SKUs (§9), RUB retail prices, each carrying its seeded stock. - Currency enum gains RUB (now the default for Order/Payment/Product events); USD/EUR/GBP kept valid for historical fixtures / cross-border. - generate_payment amount moved to the ₽ retail band (790-7990), deliberately below the 10k-25k bimodality dead-zone. - generate_product emits the SKU's seeded stock instead of randomising, so PROD-001 (kettle) stays out of stock under the live feed. - Serving-sink currency defaults 'USD'->'RUB' (local_pipeline DDL+upsert, clickhouse_sink, iceberg_sink x3). Consequence: the auto-generated stable order contracts drift, so config/contracts/order.v1+v2.yaml regenerated — currency enum +RUB and total_amount.unit USD->RUB (stale legacy annotation; orders were always RUB in the kitchen legend, so no version bump, breaking_changes stays []). Verified: runtime smoke (1200 events RUB-only; витрina mirrors §9 row-for- row; PROD-001 stays in_stock=False/qty=0 after 573 live orders). Unit suite 1665 passed, ruff+mypy clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- config/contracts/order.v1.yaml | 3 +- config/contracts/order.v2.yaml | 3 +- scripts/generate_contracts.py | 4 +- src/ingestion/producers/event_producer.py | 45 +++++++++++++++-------- src/ingestion/schemas/events.py | 11 ++++-- src/processing/clickhouse_sink.py | 2 +- src/processing/iceberg_sink.py | 6 +-- src/processing/local_pipeline.py | 4 +- tests/unit/test_event_schemas.py | 6 +-- 9 files changed, 52 insertions(+), 32 deletions(-) diff --git a/config/contracts/order.v1.yaml b/config/contracts/order.v1.yaml index f9b6c734..9b4beed1 100644 --- a/config/contracts/order.v1.yaml +++ b/config/contracts/order.v1.yaml @@ -19,11 +19,12 @@ fields: - name: total_amount type: float required: true - unit: USD + unit: RUB - name: currency type: enum required: true values: + - RUB - USD - EUR - GBP diff --git a/config/contracts/order.v2.yaml b/config/contracts/order.v2.yaml index c701be79..348567b0 100644 --- a/config/contracts/order.v2.yaml +++ b/config/contracts/order.v2.yaml @@ -19,11 +19,12 @@ fields: - name: total_amount type: float required: true - unit: USD + unit: RUB - name: currency type: enum required: true values: + - RUB - USD - EUR - GBP diff --git a/scripts/generate_contracts.py b/scripts/generate_contracts.py index cc6f5a67..cb0a4332 100644 --- a/scripts/generate_contracts.py +++ b/scripts/generate_contracts.py @@ -92,7 +92,7 @@ class ContractSpec: "values": [status.value for status in OrderStatus], }, "total_amount": { - "unit": "USD", + "unit": "RUB", }, "currency": { "type": "enum", @@ -125,7 +125,7 @@ class ContractSpec: "values": [status.value for status in OrderStatus], }, "total_amount": { - "unit": "USD", + "unit": "RUB", }, "currency": { "type": "enum", diff --git a/src/ingestion/producers/event_producer.py b/src/ingestion/producers/event_producer.py index 2d679f63..5dc31f45 100644 --- a/src/ingestion/producers/event_producer.py +++ b/src/ingestion/producers/event_producer.py @@ -46,17 +46,24 @@ class ProducerConfig(BaseSettings): model_config = {"env_prefix": "PRODUCER_"} +# Live catalog = the 10 pinned kitchen-appliance SKUs of the serving demo +# (generator-spec.md §9, mirrored row-for-row in the duckdb/clickhouse seeds): +# EN names, §3 category slugs, RUB retail prices, and each SKU's seeded +# stock. PROD-001 (kettle) is the deliberately out-of-stock bestseller — the +# oversell/freshness story needs it, so generate_product emits this stock +# verbatim rather than randomising it. +# (id, name, category slug, price ₽, stock_quantity) PRODUCT_CATALOG = [ - ("PROD-001", "Wireless Headphones", "electronics", Decimal("79.99")), - ("PROD-002", "Running Shoes", "footwear", Decimal("129.99")), - ("PROD-003", "Coffee Maker", "kitchen", Decimal("49.99")), - ("PROD-004", "Mechanical Keyboard", "electronics", Decimal("149.99")), - ("PROD-005", "Yoga Mat", "fitness", Decimal("34.99")), - ("PROD-006", "Backpack", "accessories", Decimal("89.99")), - ("PROD-007", "Water Bottle", "fitness", Decimal("24.99")), - ("PROD-008", "Desk Lamp", "home", Decimal("44.99")), - ("PROD-009", "Bluetooth Speaker", "electronics", Decimal("59.99")), - ("PROD-010", "Sunglasses", "accessories", Decimal("119.99")), + ("PROD-001", "Electric Kettle 1.7L 2200W", "kettles", Decimal("2190.00"), 0), + ("PROD-002", "Air Fryer Grill 5.5L", "grills", Decimal("5490.00"), 58), + ("PROD-003", "Immersion Blender Set 800W", "blenders", Decimal("2490.00"), 203), + ("PROD-004", "Stand Mixer 5L Planetary", "mixers", Decimal("6990.00"), 37), + ("PROD-005", "Drip Coffee Maker 1.2L", "coffee", Decimal("3490.00"), 94), + ("PROD-006", "Waffle Maker Double", "multibakers", Decimal("2290.00"), 142), + ("PROD-007", "Mini Chopper 500ml", "choppers", Decimal("1490.00"), 315), + ("PROD-008", "Cold-Press Juicer", "juicers", Decimal("4490.00"), 72), + ("PROD-009", "Digital Kitchen Scale 5kg", "scales", Decimal("990.00"), 421), + ("PROD-010", "Vacuum Sealer Compact", "vacuum-dry", Decimal("3290.00"), 167), ] PAGES = ["/", "/products", "/cart", "/checkout", "/account", "/search"] @@ -111,7 +118,7 @@ def generate_order() -> tuple[str, OrderEvent]: status=OrderStatus.PENDING, items=items, total_amount=total, - currency=Currency.USD, + currency=Currency.RUB, ) return "orders.raw", event @@ -126,8 +133,10 @@ def generate_payment(order_id: str | None = None) -> tuple[str, PaymentEvent]: payment_id=f"PAY-{_uuid()[:8]}", order_id=oid, user_id=f"USR-{random.randint(10000, 99999)}", - amount=Decimal(str(round(random.uniform(20, 500), 2))), - currency=Currency.USD, + # ₽-scale single-appliance retail range (§3 RRC band 790–7990), + # deliberately below the 10k–25k bimodality dead-zone (§12 #4). + amount=Decimal(str(round(random.uniform(790, 7990), 2))), + currency=Currency.RUB, method=random.choice(list(PaymentMethod)), status="initiated", ) @@ -155,6 +164,7 @@ def generate_click() -> tuple[str, ClickstreamEvent]: def generate_product() -> tuple[str, ProductEvent]: product = random.choice(PRODUCT_CATALOG) + stock_quantity = product[4] event = ProductEvent( event_id=_uuid(), event_type=EventType.PRODUCT_UPDATED, @@ -164,9 +174,12 @@ def generate_product() -> tuple[str, ProductEvent]: name=product[1], category=product[2], price=product[3], - currency=Currency.USD, - in_stock=random.random() > 0.1, - stock_quantity=random.randint(0, 500), + currency=Currency.RUB, + # Emit the SKU's seeded stock so the out-of-stock bestseller (PROD-001) + # stays out of stock even under the live feed — the oversell story + # depends on it, and random stock would clobber the seeded витрина. + in_stock=stock_quantity > 0, + stock_quantity=stock_quantity, ) return "products.cdc", event diff --git a/src/ingestion/schemas/events.py b/src/ingestion/schemas/events.py index 71a34437..4eaf08c3 100644 --- a/src/ingestion/schemas/events.py +++ b/src/ingestion/schemas/events.py @@ -38,6 +38,11 @@ class CdcSource(StrEnum): class Currency(StrEnum): + # RUB is the house currency of the kitchen-appliance importer legend + # (docs/domain.md §1, generator-spec.md §10); the live producer and every + # serving sink default to it. USD/EUR/GBP stay valid enum members so + # historical fixtures and cross-border edge cases still parse. + RUB = "RUB" USD = "USD" EUR = "EUR" GBP = "GBP" @@ -96,7 +101,7 @@ class OrderEvent(BaseEvent): status: OrderStatus items: list[OrderItem] = Field(..., min_length=1) total_amount: Decimal = Field(..., gt=0) - currency: Currency = Currency.USD + currency: Currency = Currency.RUB @field_validator("total_amount") @classmethod @@ -121,7 +126,7 @@ class PaymentEvent(BaseEvent): order_id: str = Field(..., pattern=r"^ORD-\d{8}-\d{4,}$") user_id: str amount: Decimal = Field(..., gt=0) - currency: Currency = Currency.USD + currency: Currency = Currency.RUB method: PaymentMethod status: str = Field(..., pattern=r"^(initiated|completed|failed|refunded)$") failure_reason: str | None = None @@ -142,6 +147,6 @@ class ProductEvent(BaseEvent): name: str = Field(..., min_length=1, max_length=500) category: str price: Decimal = Field(..., ge=0) - currency: Currency = Currency.USD + currency: Currency = Currency.RUB in_stock: bool stock_quantity: int = Field(..., ge=0) diff --git a/src/processing/clickhouse_sink.py b/src/processing/clickhouse_sink.py index 14e8f1d9..4c8b66ef 100644 --- a/src/processing/clickhouse_sink.py +++ b/src/processing/clickhouse_sink.py @@ -110,7 +110,7 @@ def upsert_order(self, event: dict) -> None: "user_id": event["user_id"], "status": event["status"], "total_amount": float(event["total_amount"]), - "currency": event.get("currency", "USD"), + "currency": event.get("currency", "RUB"), "created_at": datetime.fromisoformat(event["timestamp"]), } ], diff --git a/src/processing/iceberg_sink.py b/src/processing/iceberg_sink.py index 9128544c..e0eef8f9 100644 --- a/src/processing/iceberg_sink.py +++ b/src/processing/iceberg_sink.py @@ -264,7 +264,7 @@ def _normalize_order(self, record: dict[str, Any]) -> dict[str, Any]: "user_id": str(record["user_id"]), "status": str(record["status"]), "total_amount": float(record["total_amount"]), - "currency": str(record.get("currency", "USD")), + "currency": str(record.get("currency", "RUB")), "item_count": int( derived.get( "item_count", @@ -291,7 +291,7 @@ def _normalize_payment(self, record: dict[str, Any]) -> dict[str, Any]: "order_id": str(record["order_id"]), "user_id": str(record["user_id"]), "amount": float(record["amount"]), - "currency": str(record.get("currency", "USD")), + "currency": str(record.get("currency", "RUB")), "method": str(record["method"]), "status": str(record["status"]), "risk_score": (float(derived["risk_score"]) if "risk_score" in derived else None), @@ -331,7 +331,7 @@ def _normalize_inventory(self, record: dict[str, Any]) -> dict[str, Any]: "name": str(record["name"]), "category": str(record["category"]), "price": float(record["price"]), - "currency": str(record.get("currency", "USD")), + "currency": str(record.get("currency", "RUB")), "in_stock": bool(record["in_stock"]), "stock_quantity": int(record["stock_quantity"]), "created_at": self._coerce_timestamp(record.get("timestamp")), diff --git a/src/processing/local_pipeline.py b/src/processing/local_pipeline.py index 1f348451..fc5f00fe 100644 --- a/src/processing/local_pipeline.py +++ b/src/processing/local_pipeline.py @@ -49,7 +49,7 @@ def _ensure_tables(conn: duckdb.DuckDBPyConnection) -> None: user_id VARCHAR, status VARCHAR, total_amount DECIMAL(10,2), - currency VARCHAR DEFAULT 'USD', + currency VARCHAR DEFAULT 'RUB', created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ) """) @@ -327,7 +327,7 @@ def _upsert_order(conn: duckdb.DuckDBPyConnection, event: dict) -> None: event["user_id"], event["status"], float(event["total_amount"]), - event.get("currency", "USD"), + event.get("currency", "RUB"), datetime.fromisoformat(event["timestamp"]), ], ) diff --git a/tests/unit/test_event_schemas.py b/tests/unit/test_event_schemas.py index f1d73831..372ce969 100644 --- a/tests/unit/test_event_schemas.py +++ b/tests/unit/test_event_schemas.py @@ -14,7 +14,7 @@ def test_order_valid_minimal(sample_order_event) -> None: event = OrderEvent.model_validate(payload) - assert event.currency is Currency.USD + assert event.currency is Currency.RUB assert event.total_amount == Decimal("209.97") assert event.items[0].product_id == "PROD-001" @@ -50,13 +50,13 @@ def test_order_rejects_negative_amount(sample_order_event) -> None: ) -def test_order_currency_defaults_to_usd(sample_order_event) -> None: +def test_order_currency_defaults_to_rub(sample_order_event) -> None: payload = dict(sample_order_event) payload.pop("currency") event = OrderEvent.model_validate(payload) - assert event.currency is Currency.USD + assert event.currency is Currency.RUB def test_order_rejects_unknown_currency(sample_order_event) -> None: From 59471ef4d9cc0306cf114401ba93393557a32134 Mon Sep 17 00:00:00 2001 From: JuliaEdom Date: Sat, 4 Jul 2026 23:42:17 +0300 Subject: [PATCH 2/7] fix(g2-m5): G1 smoke total_amount to net-of-VAT, aligning with prod seed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MAJOR M5 + MINOR m1 from G2_AUDIT_REPORT.md. The bv_order_canonical smoke seed carried GROSS header totals (total_amount = subtotal + tax), while the production satellite_seed.sql and generator-spec.md §1 ("all money net of VAT") use NET (total_amount mirrors the pricing subtotal, tax separate). On ClickHouse this collapsed branch_pnl.gross_revenue == net_revenue. - Every header total_amount now equals its order's pricing subtotal (net): O1 2400->2000, O2 2600->2166.67 (winner) / 2100 / 2050, O3 3500->3000, O4 60000->50000 (both versions), O5 48000->40000, O6 36000->30000, O7 42000->40000. O8 (no pricing row) stays 30000 as a standalone net B2B. - Dissolves m1: O3's header (3500) no longer disagrees with subtotal+tax (3600) because total_amount IS the subtotal under the net convention. - Re-pinned verify_bv_order.sh (Σ 224500.00 -> 197166.67; O2 winner shipped|2600.00 -> shipped|2166.67) and README table + convention note. Verified: DuckDB replication of the view's is_deleted=0 + argMax(load_ts) collapse yields Σ=197166.67, O2=shipped|2166.67, O4=confirmed (tombstone excluded). Postgres parse-gate tests/unit/test_dv2_postgres_ddl.py 28/28. Live psql run stays the Mac-tail. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../agentflow/dv2/postgres/smoke/README.md | 20 ++++++++------ .../dv2/postgres/smoke/order_smoke_seed.sql | 26 ++++++++++--------- .../dv2/postgres/smoke/verify_bv_order.sh | 4 +-- 3 files changed, 28 insertions(+), 22 deletions(-) diff --git a/warehouse/agentflow/dv2/postgres/smoke/README.md b/warehouse/agentflow/dv2/postgres/smoke/README.md index 3326b739..392afeda 100644 --- a/warehouse/agentflow/dv2/postgres/smoke/README.md +++ b/warehouse/agentflow/dv2/postgres/smoke/README.md @@ -38,16 +38,19 @@ The standalone no-Docker recipe used for the governance verifies ## Expected canonical output (derived by hand from `order_smoke_seed.sql`) Eight orders, one canonical row each. RUB amounts; VAT is RU 20 % / UAE 5 %. +`total_amount` is **net of VAT** — it mirrors the pricing subtotal (the +production `satellite_seed.sql` convention: "subtotal mirrors header.total_amount +(pre-tax)"); tax is a separate column, never folded into the total. | order_bk | branch | channel | status | total | subtotal | tax | ship | wb_status | header_src | pricing_src | mkt_src | |---|---|---|---|---|---|---|---|---|---|---|---| -| mp__msk__0000001 | msk | marketplace | delivered | 2400.00 | 2000.00 | 400.00 | 199.00 | delivered | bitrix__msk | 1c__msk | wb__msk | -| mp__msk__0000002 | msk | marketplace | **shipped** | 2600.00 | **2166.67** | **433.33** | 199.00 | **delivering** | bitrix__msk | 1c__msk | wb__msk | -| site__msk__0008901 | msk | d2c | delivered | 3500.00 | 3000.00 | 600.00 | 299.00 | — | bitrix__msk | 1c__msk | — | -| bitrix__msk__0009181 | msk | b2b | **confirmed** | 60000.00 | 50000.00 | 10000.00 | 500.00 | — | bitrix__msk | 1c__msk | — | -| bitrix__spb__0009541 | spb | b2b | delivered | 48000.00 | 40000.00 | 8000.00 | 800.00 | — | bitrix__spb | 1c__spb | — | -| bitrix__ekb__0009721 | ekb | b2b | delivered | 36000.00 | 30000.00 | 6000.00 | 500.00 | — | bitrix__ekb | 1c__ekb | — | -| bitrix__dxb__0009851 | dxb | b2b | delivered | 42000.00 | 40000.00 | **2000.00** | 500.00 | — | bitrix__dxb | 1c__dxb | — | +| mp__msk__0000001 | msk | marketplace | delivered | 2000.00 | 2000.00 | 400.00 | 199.00 | delivered | bitrix__msk | 1c__msk | wb__msk | +| mp__msk__0000002 | msk | marketplace | **shipped** | **2166.67** | **2166.67** | **433.33** | 199.00 | **delivering** | bitrix__msk | 1c__msk | wb__msk | +| site__msk__0008901 | msk | d2c | delivered | 3000.00 | 3000.00 | 600.00 | 299.00 | — | bitrix__msk | 1c__msk | — | +| bitrix__msk__0009181 | msk | b2b | **confirmed** | 50000.00 | 50000.00 | 10000.00 | 500.00 | — | bitrix__msk | 1c__msk | — | +| bitrix__spb__0009541 | spb | b2b | delivered | 40000.00 | 40000.00 | 8000.00 | 800.00 | — | bitrix__spb | 1c__spb | — | +| bitrix__ekb__0009721 | ekb | b2b | delivered | 30000.00 | 30000.00 | 6000.00 | 500.00 | — | bitrix__ekb | 1c__ekb | — | +| bitrix__dxb__0009851 | dxb | b2b | delivered | 40000.00 | 40000.00 | **2000.00** | 500.00 | — | bitrix__dxb | 1c__dxb | — | | bitrix__ala__0009925 | ala | b2b | cancelled | 30000.00 | **—** | **—** | **—** | — | bitrix__ala | **—** | — | Bold cells are the invariants the smoke targets: @@ -69,7 +72,8 @@ Bold cells are the invariants the smoke targets: - **Per-jurisdiction VAT** — dxb effective rate `tax/subtotal = 5 %` (UAE), the RU branches `20 %`. - **Branch derivation** — `split_part(record_source,'__',2)` yields - msk×4 / spb / ekb / dxb / ala; `total_amount` sums to `224500.00`. + msk×4 / spb / ekb / dxb / ala; `total_amount` (net of VAT) sums to + `197166.67`. ## Honest scope diff --git a/warehouse/agentflow/dv2/postgres/smoke/order_smoke_seed.sql b/warehouse/agentflow/dv2/postgres/smoke/order_smoke_seed.sql index 034c7395..e4cb3d7b 100644 --- a/warehouse/agentflow/dv2/postgres/smoke/order_smoke_seed.sql +++ b/warehouse/agentflow/dv2/postgres/smoke/order_smoke_seed.sql @@ -104,31 +104,33 @@ ON CONFLICT (link_hk) DO NOTHING; -- hash_diff distinguishes SCD2 versions (msk O2 v1/v2, O4 v1/v2). INSERT INTO rv.sat_order_header__bitrix__msk (order_hk, load_ts, hash_diff, order_date, channel, order_status, total_amount, is_deleted) VALUES - -- O1: single delivered marketplace header - (decode(md5('mp__msk__0000001'),'hex'), '2026-07-01 09:00:00', decode(md5('mp__msk__0000001|hdr|v1'),'hex'), '2026-06-28 14:12:00', 'marketplace', 'delivered', 2400.00, 0), - -- O2: two versions — pending@09:00 then shipped@12:00 (latest wins) - (decode(md5('mp__msk__0000002'),'hex'), '2026-07-01 09:00:00', decode(md5('mp__msk__0000002|hdr|v1'),'hex'), '2026-06-29 10:05:00', 'marketplace', 'pending', 2600.00, 0), - (decode(md5('mp__msk__0000002'),'hex'), '2026-07-01 12:00:00', decode(md5('mp__msk__0000002|hdr|v2'),'hex'), '2026-06-29 10:05:00', 'marketplace', 'shipped', 2600.00, 0), + -- O1: single delivered marketplace header. total_amount = subtotal (net of + -- VAT), mirroring satellite_seed.sql — tax lives only in the pricing sat. + (decode(md5('mp__msk__0000001'),'hex'), '2026-07-01 09:00:00', decode(md5('mp__msk__0000001|hdr|v1'),'hex'), '2026-06-28 14:12:00', 'marketplace', 'delivered', 2000.00, 0), + -- O2: two versions — pending@09:00 then shipped@12:00 (latest wins). Net + -- totals track each version's pricing subtotal (2100 -> 2166.67). + (decode(md5('mp__msk__0000002'),'hex'), '2026-07-01 09:00:00', decode(md5('mp__msk__0000002|hdr|v1'),'hex'), '2026-06-29 10:05:00', 'marketplace', 'pending', 2100.00, 0), + (decode(md5('mp__msk__0000002'),'hex'), '2026-07-01 12:00:00', decode(md5('mp__msk__0000002|hdr|v2'),'hex'), '2026-06-29 10:05:00', 'marketplace', 'shipped', 2166.67, 0), -- O3: D2C delivered - (decode(md5('site__msk__0008901'),'hex'), '2026-07-01 10:00:00', decode(md5('site__msk__0008901|hdr|v1'),'hex'), '2026-06-30 18:40:00', 'd2c', 'delivered', 3500.00, 0), + (decode(md5('site__msk__0008901'),'hex'), '2026-07-01 10:00:00', decode(md5('site__msk__0008901|hdr|v1'),'hex'), '2026-06-30 18:40:00', 'd2c', 'delivered', 3000.00, 0), -- O4: active confirmed@10:00, then a soft-delete tombstone@14:00 (is_deleted=1, must NOT win) - (decode(md5('bitrix__msk__0009181'),'hex'), '2026-07-01 10:00:00', decode(md5('bitrix__msk__0009181|hdr|v1'),'hex'), '2026-06-27 09:15:00', 'b2b', 'confirmed', 60000.00, 0), - (decode(md5('bitrix__msk__0009181'),'hex'), '2026-07-01 14:00:00', decode(md5('bitrix__msk__0009181|hdr|v2'),'hex'), '2026-06-27 09:15:00', 'b2b', 'cancelled', 60000.00, 1) + (decode(md5('bitrix__msk__0009181'),'hex'), '2026-07-01 10:00:00', decode(md5('bitrix__msk__0009181|hdr|v1'),'hex'), '2026-06-27 09:15:00', 'b2b', 'confirmed', 50000.00, 0), + (decode(md5('bitrix__msk__0009181'),'hex'), '2026-07-01 14:00:00', decode(md5('bitrix__msk__0009181|hdr|v2'),'hex'), '2026-06-27 09:15:00', 'b2b', 'cancelled', 50000.00, 1) ON CONFLICT (order_hk, load_ts) DO NOTHING; INSERT INTO rv.sat_order_header__bitrix__spb (order_hk, load_ts, hash_diff, order_date, channel, order_status, total_amount, is_deleted) VALUES - (decode(md5('bitrix__spb__0009541'),'hex'), '2026-07-01 10:00:00', decode(md5('bitrix__spb__0009541|hdr|v1'),'hex'), '2026-06-26 11:00:00', 'b2b', 'delivered', 48000.00, 0) + (decode(md5('bitrix__spb__0009541'),'hex'), '2026-07-01 10:00:00', decode(md5('bitrix__spb__0009541|hdr|v1'),'hex'), '2026-06-26 11:00:00', 'b2b', 'delivered', 40000.00, 0) ON CONFLICT (order_hk, load_ts) DO NOTHING; INSERT INTO rv.sat_order_header__bitrix__ekb (order_hk, load_ts, hash_diff, order_date, channel, order_status, total_amount, is_deleted) VALUES - (decode(md5('bitrix__ekb__0009721'),'hex'), '2026-07-01 10:00:00', decode(md5('bitrix__ekb__0009721|hdr|v1'),'hex'), '2026-06-25 16:30:00', 'b2b', 'delivered', 36000.00, 0) + (decode(md5('bitrix__ekb__0009721'),'hex'), '2026-07-01 10:00:00', decode(md5('bitrix__ekb__0009721|hdr|v1'),'hex'), '2026-06-25 16:30:00', 'b2b', 'delivered', 30000.00, 0) ON CONFLICT (order_hk, load_ts) DO NOTHING; INSERT INTO rv.sat_order_header__bitrix__dxb (order_hk, load_ts, hash_diff, order_date, channel, order_status, total_amount, is_deleted) VALUES - (decode(md5('bitrix__dxb__0009851'),'hex'), '2026-07-01 10:00:00', decode(md5('bitrix__dxb__0009851|hdr|v1'),'hex'), '2026-06-24 08:20:00', 'b2b', 'delivered', 42000.00, 0) + (decode(md5('bitrix__dxb__0009851'),'hex'), '2026-07-01 10:00:00', decode(md5('bitrix__dxb__0009851|hdr|v1'),'hex'), '2026-06-24 08:20:00', 'b2b', 'delivered', 40000.00, 0) ON CONFLICT (order_hk, load_ts) DO NOTHING; INSERT INTO rv.sat_order_header__bitrix__ala @@ -140,7 +142,7 @@ ON CONFLICT (order_hk, load_ts) DO NOTHING; -- collapse across sources — the newer Bitrix shipped@12:00 row must still win. INSERT INTO rv.sat_order_header__1c__msk (order_hk, load_ts, hash_diff, order_date, channel, order_status, total_amount, is_deleted) VALUES - (decode(md5('mp__msk__0000002'),'hex'), '2026-07-01 08:00:00', decode(md5('mp__msk__0000002|hdr|1c'),'hex'), '2026-06-29 10:05:00', 'marketplace', 'pending', 2350.00, 0) + (decode(md5('mp__msk__0000002'),'hex'), '2026-07-01 08:00:00', decode(md5('mp__msk__0000002|hdr|1c'),'hex'), '2026-06-29 10:05:00', 'marketplace', 'pending', 2050.00, 0) ON CONFLICT (order_hk, load_ts) DO NOTHING; -- ============ ORDER PRICING (1C — RUB, per-jurisdiction VAT) ============ diff --git a/warehouse/agentflow/dv2/postgres/smoke/verify_bv_order.sh b/warehouse/agentflow/dv2/postgres/smoke/verify_bv_order.sh index 61a20d98..b58f4696 100644 --- a/warehouse/agentflow/dv2/postgres/smoke/verify_bv_order.sh +++ b/warehouse/agentflow/dv2/postgres/smoke/verify_bv_order.sh @@ -55,12 +55,12 @@ assert_eq "no branch escapes the five jurisdictions" "0" \ "SELECT count(*) FROM $BV WHERE $SCOPE AND branch NOT IN ('msk','spb','ekb','dxb','ala')" assert_eq "customer + store links all resolved" "8" \ "SELECT count(*) FILTER (WHERE customer_hk IS NOT NULL AND store_hk IS NOT NULL) FROM $BV WHERE $SCOPE" -assert_eq "total_amount sum (RUB)" "224500.00" \ +assert_eq "total_amount sum (RUB, net of VAT)" "197166.67" \ "SELECT sum(total_amount) FROM $BV WHERE $SCOPE" echo echo "=== SCD2: latest load_ts wins ===" -assert_eq "O2 header collapses across UNION to newest Bitrix version" "shipped|2600.00" \ +assert_eq "O2 header collapses across UNION to newest Bitrix version" "shipped|2166.67" \ "SELECT order_status||'|'||total_amount FROM $BV WHERE order_bk='mp__msk__0000002'" assert_eq "O2 pricing collapses to newest 1C version" "2166.67|433.33" \ "SELECT subtotal_amount||'|'||tax_amount FROM $BV WHERE order_bk='mp__msk__0000002'" From 724efb69e2718a40697b1ca3177c8fd5878624a8 Mon Sep 17 00:00:00 2001 From: JuliaEdom Date: Sat, 4 Jul 2026 23:54:45 +0300 Subject: [PATCH 3/7] =?UTF-8?q?fix(g2-m4):=20rebuild=20OLTP=20hot-tier=20s?= =?UTF-8?q?eeds=20to=20the=20kitchen=20legend=20(=C2=A71/=C2=A72)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MAJOR M4 from G2_AUDIT_REPORT.md. postgres_oltp/seed.sql and fanout/02_seed.sql carried channels (web/mobile/retail/call-center), statuses (new/paid/shipped/ refunded) and amounts (500-24,999 ₽ — mass inside the forbidden 10k-25k bimodality dead-zone) that predate the legend, and dxb looked retail rather than b2b re-export. These promote straight into bv_order_canonical. postgres_oltp/seed.sql (feeds promote_to_raw_vault -> vault): - Channels -> §2 vocabulary {marketplace, d2c, b2b}; msk is the marketplace- dominant mix (186 mp / 7 d2c / 7 b2b), dxb is 'b2b' only. - Statuses -> §2 ladder pending/confirmed/shipped/delivered/cancelled at the production 8/10/12/62/8 weights. - Amounts -> §1 bimodal bands (mp 1.5-3k, d2c 2-5k, b2b msk 30-80k, dxb 60-130k ₽ net), mirroring satellite_seed*.sql — none in the 10k-25k dead-zone. fanout/02_seed.sql (per-branch topology proof): §2 statuses only; msk RUB retail/d2c + a few b2b, dxb AED at export-pallet scale (2.5-5.1k AED); no amount in the RUB or AED dead-zone. Verified: arithmetic check confirms 0 dead-zone hits across all generated and literal amounts, bands correct, every status in the §2 ladder; both files parse under sqlglot's postgres dialect. No test pins the old seed values. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../dv2/postgres_oltp/fanout/02_seed.sql | 107 ++++++++++-------- .../agentflow/dv2/postgres_oltp/seed.sql | 41 ++++++- 2 files changed, 92 insertions(+), 56 deletions(-) diff --git a/warehouse/agentflow/dv2/postgres_oltp/fanout/02_seed.sql b/warehouse/agentflow/dv2/postgres_oltp/fanout/02_seed.sql index 97e5b646..8fec0cc3 100644 --- a/warehouse/agentflow/dv2/postgres_oltp/fanout/02_seed.sql +++ b/warehouse/agentflow/dv2/postgres_oltp/fanout/02_seed.sql @@ -21,37 +21,41 @@ INSERT INTO customers (customer_id, first_name, last_name, email, phone) VALUES ('msk-c-010', 'Tatiana','Fedorova','fedorova@example.ru','+74951110010') ON CONFLICT (customer_id) DO NOTHING; +-- Status vocabulary = generator-spec.md §2 ladder (pending / confirmed / +-- shipped / delivered / cancelled). msk amounts sit in the marketplace/d2c +-- retail bands (1.5k-5k ₽) with a few b2b tickets (30k-80k ₽); none fall in +-- the 10k-25k bimodality dead-zone (§12 #4). INSERT INTO orders (order_id, customer_id, status, total, currency) VALUES - ('msk-o-001','msk-c-001','paid', 4500.00,'RUB'), - ('msk-o-002','msk-c-001','paid', 2100.00,'RUB'), - ('msk-o-003','msk-c-002','paid', 8900.00,'RUB'), - ('msk-o-004','msk-c-003','pending', 1200.00,'RUB'), - ('msk-o-005','msk-c-003','paid', 6700.00,'RUB'), - ('msk-o-006','msk-c-004','paid', 3300.00,'RUB'), - ('msk-o-007','msk-c-004','refunded', 1850.00,'RUB'), - ('msk-o-008','msk-c-005','paid', 12500.00,'RUB'), - ('msk-o-009','msk-c-005','paid', 2400.00,'RUB'), - ('msk-o-010','msk-c-006','pending', 5600.00,'RUB'), - ('msk-o-011','msk-c-006','paid', 7800.00,'RUB'), - ('msk-o-012','msk-c-007','paid', 1900.00,'RUB'), - ('msk-o-013','msk-c-007','paid', 15300.00,'RUB'), - ('msk-o-014','msk-c-008','paid', 4100.00,'RUB'), - ('msk-o-015','msk-c-008','paid', 6200.00,'RUB'), - ('msk-o-016','msk-c-009','paid', 2700.00,'RUB'), - ('msk-o-017','msk-c-009','pending', 3400.00,'RUB'), - ('msk-o-018','msk-c-009','paid', 8100.00,'RUB'), - ('msk-o-019','msk-c-010','paid', 1100.00,'RUB'), - ('msk-o-020','msk-c-010','paid', 9300.00,'RUB'), - ('msk-o-021','msk-c-010','paid', 5700.00,'RUB'), - ('msk-o-022','msk-c-001','paid', 3800.00,'RUB'), - ('msk-o-023','msk-c-002','paid', 7400.00,'RUB'), - ('msk-o-024','msk-c-003','paid', 2900.00,'RUB'), - ('msk-o-025','msk-c-005','paid', 11200.00,'RUB'), - ('msk-o-026','msk-c-007','paid', 4600.00,'RUB'), - ('msk-o-027','msk-c-008','paid', 8500.00,'RUB'), - ('msk-o-028','msk-c-009','paid', 3100.00,'RUB'), - ('msk-o-029','msk-c-010','paid', 6900.00,'RUB'), - ('msk-o-030','msk-c-004','paid', 2200.00,'RUB') + ('msk-o-001','msk-c-001','delivered', 2450.00,'RUB'), + ('msk-o-002','msk-c-001','delivered', 2100.00,'RUB'), + ('msk-o-003','msk-c-002','shipped', 2890.00,'RUB'), + ('msk-o-004','msk-c-003','pending', 1650.00,'RUB'), + ('msk-o-005','msk-c-003','delivered', 3300.00,'RUB'), + ('msk-o-006','msk-c-004','delivered', 2750.00,'RUB'), + ('msk-o-007','msk-c-004','cancelled', 1850.00,'RUB'), + ('msk-o-008','msk-c-005','delivered',42000.00,'RUB'), + ('msk-o-009','msk-c-005','delivered', 2400.00,'RUB'), + ('msk-o-010','msk-c-006','pending', 4600.00,'RUB'), + ('msk-o-011','msk-c-006','confirmed', 3800.00,'RUB'), + ('msk-o-012','msk-c-007','delivered', 1900.00,'RUB'), + ('msk-o-013','msk-c-007','delivered',58000.00,'RUB'), + ('msk-o-014','msk-c-008','delivered', 2650.00,'RUB'), + ('msk-o-015','msk-c-008','shipped', 4200.00,'RUB'), + ('msk-o-016','msk-c-009','delivered', 2700.00,'RUB'), + ('msk-o-017','msk-c-009','pending', 3400.00,'RUB'), + ('msk-o-018','msk-c-009','delivered', 2950.00,'RUB'), + ('msk-o-019','msk-c-010','delivered', 1600.00,'RUB'), + ('msk-o-020','msk-c-010','delivered', 4700.00,'RUB'), + ('msk-o-021','msk-c-010','delivered', 3900.00,'RUB'), + ('msk-o-022','msk-c-001','delivered', 2200.00,'RUB'), + ('msk-o-023','msk-c-002','shipped', 4400.00,'RUB'), + ('msk-o-024','msk-c-003','delivered', 2900.00,'RUB'), + ('msk-o-025','msk-c-005','delivered',71000.00,'RUB'), + ('msk-o-026','msk-c-007','delivered', 3600.00,'RUB'), + ('msk-o-027','msk-c-008','confirmed', 4850.00,'RUB'), + ('msk-o-028','msk-c-009','delivered', 3100.00,'RUB'), + ('msk-o-029','msk-c-010','delivered', 4900.00,'RUB'), + ('msk-o-030','msk-c-004','delivered', 2200.00,'RUB') ON CONFLICT (order_id) DO NOTHING; \c ops_dxb_db @@ -67,25 +71,28 @@ INSERT INTO customers (customer_id, first_name, last_name, email, phone) VALUES ('dxb-c-008','Mariam', 'Al-Suwaidi','mariam@example.ae','+97150100008') ON CONFLICT (customer_id) DO NOTHING; +-- dxb is the b2b re-export branch (§1): every order is wholesale, priced in +-- AED at the export-pallet scale (≈ 2.5k-5.1k AED ≈ 60k-125k ₽ at the §10 FX +-- of 24.5 ₽/AED). §2 status ladder; no retail-scale tickets. INSERT INTO orders (order_id, customer_id, status, total, currency) VALUES - ('dxb-o-001','dxb-c-001','paid', 850.00,'AED'), - ('dxb-o-002','dxb-c-001','paid', 1240.00,'AED'), - ('dxb-o-003','dxb-c-002','paid', 420.00,'AED'), - ('dxb-o-004','dxb-c-003','paid', 2300.00,'AED'), - ('dxb-o-005','dxb-c-003','pending', 680.00,'AED'), - ('dxb-o-006','dxb-c-004','paid', 1100.00,'AED'), - ('dxb-o-007','dxb-c-005','paid', 1850.00,'AED'), - ('dxb-o-008','dxb-c-005','paid', 590.00,'AED'), - ('dxb-o-009','dxb-c-006','paid', 920.00,'AED'), - ('dxb-o-010','dxb-c-006','refunded', 380.00,'AED'), - ('dxb-o-011','dxb-c-007','paid', 2700.00,'AED'), - ('dxb-o-012','dxb-c-007','paid', 750.00,'AED'), - ('dxb-o-013','dxb-c-008','paid', 1450.00,'AED'), - ('dxb-o-014','dxb-c-008','paid', 980.00,'AED'), - ('dxb-o-015','dxb-c-001','paid', 310.00,'AED'), - ('dxb-o-016','dxb-c-002','paid', 1670.00,'AED'), - ('dxb-o-017','dxb-c-003','paid', 820.00,'AED'), - ('dxb-o-018','dxb-c-004','paid', 2150.00,'AED'), - ('dxb-o-019','dxb-c-005','paid', 460.00,'AED'), - ('dxb-o-020','dxb-c-006','paid', 1320.00,'AED') + ('dxb-o-001','dxb-c-001','delivered', 3200.00,'AED'), + ('dxb-o-002','dxb-c-001','delivered', 4100.00,'AED'), + ('dxb-o-003','dxb-c-002','shipped', 2800.00,'AED'), + ('dxb-o-004','dxb-c-003','delivered', 4600.00,'AED'), + ('dxb-o-005','dxb-c-003','pending', 2500.00,'AED'), + ('dxb-o-006','dxb-c-004','delivered', 3400.00,'AED'), + ('dxb-o-007','dxb-c-005','delivered', 4900.00,'AED'), + ('dxb-o-008','dxb-c-005','delivered', 2650.00,'AED'), + ('dxb-o-009','dxb-c-006','delivered', 3700.00,'AED'), + ('dxb-o-010','dxb-c-006','cancelled', 2450.00,'AED'), + ('dxb-o-011','dxb-c-007','delivered', 5100.00,'AED'), + ('dxb-o-012','dxb-c-007','delivered', 2900.00,'AED'), + ('dxb-o-013','dxb-c-008','shipped', 4300.00,'AED'), + ('dxb-o-014','dxb-c-008','delivered', 3550.00,'AED'), + ('dxb-o-015','dxb-c-001','delivered', 2750.00,'AED'), + ('dxb-o-016','dxb-c-002','delivered', 4750.00,'AED'), + ('dxb-o-017','dxb-c-003','confirmed', 3150.00,'AED'), + ('dxb-o-018','dxb-c-004','delivered', 4400.00,'AED'), + ('dxb-o-019','dxb-c-005','delivered', 2600.00,'AED'), + ('dxb-o-020','dxb-c-006','delivered', 3900.00,'AED') ON CONFLICT (order_id) DO NOTHING; diff --git a/warehouse/agentflow/dv2/postgres_oltp/seed.sql b/warehouse/agentflow/dv2/postgres_oltp/seed.sql index 7db41a7b..9f5e745c 100644 --- a/warehouse/agentflow/dv2/postgres_oltp/seed.sql +++ b/warehouse/agentflow/dv2/postgres_oltp/seed.sql @@ -52,6 +52,11 @@ CREATE TABLE IF NOT EXISTS ops_dxb.orders ( ); -- ============ Seed: 50 msk customers + 200 msk orders ============ +-- Channels / statuses / amounts mirror satellite_seed.sql (generator-spec.md +-- §1/§2): msk carries the marketplace-dominant mix (marketplace 1.5k-3k, d2c +-- 2k-5k, b2b 30k-80k ₽ net-of-VAT), status ladder pending/confirmed/shipped/ +-- delivered/cancelled at 8/10/12/62/8. Amounts stay clear of the 10k-25k +-- bimodality dead-zone (§12 #4). INSERT INTO ops_msk.customers (customer_id, first_name, last_name, email, phone) SELECT 'CUST-MSK-' || lpad(n::text, 4, '0'), @@ -67,13 +72,31 @@ SELECT 'OLTP-MSK-' || lpad(n::text, 6, '0'), 'CUST-MSK-' || lpad(((n % 50) + 1)::text, 4, '0'), now() - (n || ' hour')::interval, - (ARRAY['web','mobile','retail','call-center'])[(n % 4) + 1], - (ARRAY['new','paid','shipped'])[(n % 3) + 1], - (500 + (n * 31) % 24500)::numeric(18, 2) + CASE + WHEN n <= 186 THEN 'marketplace' + WHEN n <= 193 THEN 'd2c' + ELSE 'b2b' + END, + CASE + WHEN n % 100 < 8 THEN 'pending' + WHEN n % 100 < 18 THEN 'confirmed' + WHEN n % 100 < 30 THEN 'shipped' + WHEN n % 100 < 92 THEN 'delivered' + ELSE 'cancelled' + END, + CASE + WHEN n <= 186 THEN (1500 + (n * 17) % 1501)::numeric(18, 2) -- marketplace 1.5k-3.0k + WHEN n <= 193 THEN (2000 + (n * 23) % 3001)::numeric(18, 2) -- d2c 2.0k-5.0k + ELSE (30000 + (n * 137) % 50001)::numeric(18, 2) -- b2b msk 30k-80k + END FROM generate_series(1, 200) AS n ON CONFLICT (order_id) DO NOTHING; -- ============ Seed: 20 dxb customers + 80 dxb orders ============ +-- dxb is the b2b re-export branch (generator-spec.md §1: no marketplace/D2C +-- volume). All orders are 'b2b'; amounts follow the export-pallet band +-- (60k-130k ₽ net, mirrors satellite_seed_all_branches.sql), well above the +-- 10k-25k dead-zone. INSERT INTO ops_dxb.customers (customer_id, first_name, last_name, email, phone) SELECT 'CUST-DXB-' || lpad(n::text, 4, '0'), @@ -89,8 +112,14 @@ SELECT 'OLTP-DXB-' || lpad(n::text, 6, '0'), 'CUST-DXB-' || lpad(((n % 20) + 1)::text, 4, '0'), now() - (n || ' hour')::interval, - (ARRAY['web','mobile','retail'])[(n % 3) + 1], - (ARRAY['new','paid','shipped'])[(n % 3) + 1], - (500 + (n * 31) % 24500)::numeric(18, 2) + 'b2b', + CASE + WHEN n % 100 < 8 THEN 'pending' + WHEN n % 100 < 18 THEN 'confirmed' + WHEN n % 100 < 30 THEN 'shipped' + WHEN n % 100 < 92 THEN 'delivered' + ELSE 'cancelled' + END, + (60000 + (n * 191) % 70001)::numeric(18, 2) -- b2b dxb export pallets 60k-130k FROM generate_series(1, 80) AS n ON CONFLICT (order_id) DO NOTHING; From 9355159742f255e5eea6475681b5634727d10130 Mon Sep 17 00:00:00 2001 From: JuliaEdom Date: Sun, 5 Jul 2026 00:04:43 +0300 Subject: [PATCH 4/7] fix(g2-docs): correct admin-analytics path, live ops surfaces, marking-code split MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Autonomous-safe doc↔code accuracy batch from G2_AUDIT_REPORT.md. - M6: domain.md §6 cited `/v1/analytics/*` for the e-com and data-engineer personas, but the router is admin-gated — real path `/v1/admin/analytics/*` (admin.py prefix `/admin` + require_admin_key; openapi confirms). Corrected both cells and flagged the admin-key requirement. - m11: §4 called the three ops surfaces "planned"; Order 360 timeline, stuck-orders worklist and exception inbox are live (D2/D3/D4), already listed as live endpoints in §6. Dropped "planned", named the live routes. - n1: demo_evidence.md called all 12,160 marking codes "per-unit"; it's 160 SKU-level GTIN templates + 12,000 per-unit codes (§11 / synthetic_seed.sql). Skipped m2 ("32/32"→29): not reproducible — the primary evidence doc (vault-pii-governance-verify-2026-07-02.md) and every surviving citation (README, clickhouse-cutover-plan) consistently say 32/32; the audit's "29" and its RELEASE_STATUS.md/architecture.md line refs don't check out. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/domain.md | 10 ++++++---- docs/dv2-multi-branch/demo_evidence.md | 3 ++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/domain.md b/docs/domain.md index 81ffcad0..d743b9dc 100644 --- a/docs/domain.md +++ b/docs/domain.md @@ -149,8 +149,10 @@ requirements: the ops layer. 3. **Kill-the-five-programs triage.** Where is order X? Which orders are stuck between confirmation and shipment longer than the stage SLA? Which failed - events need a manual decision? These map to the three planned ops surfaces: - **Order 360 timeline**, **stuck-orders worklist**, and **exception inbox**. + events need a manual decision? These map to the three ops surfaces — + **Order 360 timeline**, **stuck-orders worklist**, and **exception inbox** — + now live (`GET /v1/entity/order/{id}/timeline`, `/v1/ops/stuck-orders`, + `/v1/ops/exceptions`). ## 5. Reinterpretation dictionary @@ -224,10 +226,10 @@ the operational-layer roadmap; everything else is live API surface. | ------- | ------------------ | ------- | | **Owner / CEO** | "Revenue today vs yesterday? Orders during the НГ peak? Is AOV holding after the price move?" | `GET /v1/metrics/revenue` · `order_count` · `avg_order_value`; `POST /v1/query` (NL: "top products this week") | | **B2B account manager** | "What has dealer X ordered this quarter? Are they on track for the retro-bonus threshold? Which of my contacts has a birthday before March 8?" | `GET /v1/entity/user/{id}`; `POST /v1/query`; vault: `bv_customer_mdm`, `sat_customer_loyalty__bitrix__*` | -| **E-com / marketplace manager** | "Site conversion this week? Are WB orders flowing or did the feed break? Top SKUs by orders during the sale event?" | `GET /v1/metrics/conversion_rate` · `active_sessions` · `error_rate`; `GET /v1/analytics/top-entities`; `POST /v1/query` | +| **E-com / marketplace manager** | "Site conversion this week? Are WB orders flowing or did the feed break? Top SKUs by orders during the sale event?" | `GET /v1/metrics/conversion_rate` · `active_sessions` · `error_rate`; `GET /v1/admin/analytics/top-entities` (admin-key); `POST /v1/query` | | **Operations manager** | "Everything about ORD-20260404-1001 — now, in one place. Which orders are stuck pre-shipment past SLA? What failed overnight and needs a human?" | `GET /v1/entity/order/{id}`; `/v1/alerts` (+history/test); `/v1/deadletter` (+replay/dismiss); `GET /v1/entity/order/{id}/timeline`; `GET /v1/ops/stuck-orders`; `GET /v1/ops/exceptions` (+stats/acknowledge/resolve) | | **Category / procurement manager** | "Which SKUs sell fastest per branch? What is on hand vs reserved? When does the next container land?" | `GET /v1/search`; `GET /v1/entity/product/{id}`; `POST /v1/query`; vault: `sat_product_stock__wms__*`; container ETA — **planned** (today: `excel__*` manifests) | -| **Data engineer / analyst** | "Which events move this metric? What is the contract and its staleness budget? What changed between contract v3 and v4? Are we inside SLO?" | `GET /v1/catalog`; `/v1/contracts/*` (+versions/diff/validate); `/v1/lineage/*`; `/v1/slo`; `/v1/analytics/*` | +| **Data engineer / analyst** | "Which events move this metric? What is the contract and its staleness budget? What changed between contract v3 and v4? Are we inside SLO?" | `GET /v1/catalog`; `/v1/contracts/*` (+versions/diff/validate); `/v1/lineage/*`; `/v1/slo`; `/v1/admin/analytics/*` (admin-key) | | **AI agent / integration** | Any of the above, programmatically — the agent is one consumer, not the product | Python/TS SDKs, MCP/LangChain/LlamaIndex integrations over the same API: `POST /v1/query` + `/v1/query/explain`, `/v1/entity/*`, `POST /v1/batch`, `/v1/webhooks`, `/v1/stream` | | **PII officer (per jurisdiction)** | "Who can read dealer-contact PII in `dxb`? Prove RU data never crosses the border" | Not REST: DV2 governance — jurisdiction-scoped officer roles, column-limited analyst grants, per-branch row policies (`warehouse/agentflow/dv2/governance/`, ClickHouse and Postgres variants) | diff --git a/docs/dv2-multi-branch/demo_evidence.md b/docs/dv2-multi-branch/demo_evidence.md index efa3185a..28037103 100644 --- a/docs/dv2-multi-branch/demo_evidence.md +++ b/docs/dv2-multi-branch/demo_evidence.md @@ -170,7 +170,8 @@ SELECT count() FROM rv.lnk_order_product; -- 14853 basket profile: marketplace and D2C orders are predominantly single-item, while the B2B dealer orders carry multi-line baskets, pulling the mean just above 1. `hub_product` holds **160 SKU**; `hub_marking_code` holds **12,160** -per-unit Chestny ZNAK marking codes (issued / in-circulation / withdrawn). +Chestny ZNAK marking codes — **160** SKU-level GTIN templates + **12,000** +per-unit codes (issued / in-circulation / withdrawn). ## 8. Business Vault — populated views with MDM conflict resolution From ea0484be5203c6a6e003b7851aec48a95f7e6ab0 Mon Sep 17 00:00:00 2001 From: JuliaEdom Date: Sun, 5 Jul 2026 03:51:00 +0300 Subject: [PATCH 5/7] fix(g2-m3): dbt returns metrics measure 'cancelled', not phantom 'returned' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MINOR m3 from G2_AUDIT_REPORT.md. branch_pnl, returns_velocity and customer_360 counted `order_status = 'returned'`, but no order ever carries that status — the §2 vocabulary is pending/confirmed/shipped/delivered/ cancelled (OrderStatus enum, contracts, seeds). Every returns metric (returned_orders, returned_value, return_rate, returned_tax_unrecovered) was therefore a constant 0. The legend has no dedicated return status: its terminal-negative bucket 'cancelled' folds cancellations and marketplace returns together (§2 "cancel/return allowance"; satellite_seed.sql — "marketplace cancels dominate the last bucket"). Remapped the filters to 'cancelled' so the metrics measure that ~8% bucket. Column names kept (load-test queries + demo materials reference them); a comment documents the semantics in each model. Verified: no 'returned' filter remains; 'cancelled' is emitted by the seed status ladder (number%100>=92); test_dv2_governance_ddl (customer_360 PII-free contract) 9/9. Live dbt run stays the Mac-tail. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../agentflow/dv2/dbt/models/marts/branch_pnl.sql | 9 +++++++-- .../agentflow/dv2/dbt/models/marts/customer_360.sql | 7 +++++-- .../dv2/dbt/models/marts/returns_velocity.sql | 12 ++++++++---- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/warehouse/agentflow/dv2/dbt/models/marts/branch_pnl.sql b/warehouse/agentflow/dv2/dbt/models/marts/branch_pnl.sql index 4f930599..8f647778 100644 --- a/warehouse/agentflow/dv2/dbt/models/marts/branch_pnl.sql +++ b/warehouse/agentflow/dv2/dbt/models/marts/branch_pnl.sql @@ -15,9 +15,14 @@ SELECT sum(toFloat64(subtotal_amount)) AS net_revenue, sum(toFloat64(discount_amount)) AS discounts, sum(toFloat64(shipping_cost)) AS shipping, - countIf(order_status = 'returned') AS returned_orders, + -- The legend has no dedicated 'returned' status (§2 vocabulary is + -- pending/confirmed/shipped/delivered/cancelled). Its terminal-negative + -- bucket 'cancelled' carries both cancellations and marketplace returns + -- (§2 "cancel/return allowance"; satellite_seed.sql — "marketplace cancels + -- dominate the last bucket"), so returns are measured off 'cancelled'. + countIf(order_status = 'cancelled') AS returned_orders, sumIf(toFloat64(total_amount), - order_status = 'returned') AS returned_value, + order_status = 'cancelled') AS returned_value, round(sum(toFloat64(tax_amount)) / nullIf(sum(toFloat64(subtotal_amount)), 0), 4) AS effective_tax_rate diff --git a/warehouse/agentflow/dv2/dbt/models/marts/customer_360.sql b/warehouse/agentflow/dv2/dbt/models/marts/customer_360.sql index ec4cbd66..81ae281a 100644 --- a/warehouse/agentflow/dv2/dbt/models/marts/customer_360.sql +++ b/warehouse/agentflow/dv2/dbt/models/marts/customer_360.sql @@ -49,9 +49,12 @@ order_agg AS ( sum(toFloat64(total_amount)) AS lifetime_value, min(order_date) AS first_order_dt, max(order_date) AS last_order_dt, - countIf(order_status = 'returned') AS returned_orders, + -- Returns off the 'cancelled' bucket: no dedicated 'returned' status + -- exists in the §2 vocabulary; 'cancelled' folds cancellations and + -- marketplace returns together (satellite_seed.sql). + countIf(order_status = 'cancelled') AS returned_orders, sumIf(toFloat64(total_amount), - order_status = 'returned') AS returned_value + order_status = 'cancelled') AS returned_value FROM {{ source('rv', 'bv_order_canonical_mat') }} WHERE customer_hk != toFixedString('', 16) GROUP BY customer_hk, branch diff --git a/warehouse/agentflow/dv2/dbt/models/marts/returns_velocity.sql b/warehouse/agentflow/dv2/dbt/models/marts/returns_velocity.sql index 175071e4..0b12a4a1 100644 --- a/warehouse/agentflow/dv2/dbt/models/marts/returns_velocity.sql +++ b/warehouse/agentflow/dv2/dbt/models/marts/returns_velocity.sql @@ -11,13 +11,17 @@ SELECT channel AS channel, toStartOfWeek(order_date) AS week, count() AS orders, - countIf(order_status = 'returned') AS returned_orders, - round(countIf(order_status = 'returned') * 1.0 / + -- Returns are measured off the 'cancelled' bucket: the legend has no + -- dedicated 'returned' status (§2), and its terminal-negative bucket folds + -- cancellations and marketplace returns together (§2 "cancel/return + -- allowance"; satellite_seed.sql — "marketplace cancels dominate"). + countIf(order_status = 'cancelled') AS returned_orders, + round(countIf(order_status = 'cancelled') * 1.0 / count(), 4) AS return_rate, sumIf(toFloat64(total_amount), - order_status = 'returned') AS returned_value, + order_status = 'cancelled') AS returned_value, sumIf(toFloat64(tax_amount), - order_status = 'returned') AS returned_tax_unrecovered + order_status = 'cancelled') AS returned_tax_unrecovered FROM {{ source('rv', 'bv_order_canonical_mat') }} WHERE order_date IS NOT NULL AND channel IS NOT NULL From fba42d351ef499c47827d1010b4aed364f83b283 Mon Sep 17 00:00:00 2001 From: JuliaEdom Date: Sun, 5 Jul 2026 04:10:13 +0300 Subject: [PATCH 6/7] refactor(g2-x5): de-brand loader-independent X5 references (M3/m8 partial) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First slice of the X5 -> synthetic track (G2_AUDIT_REPORT.md). These sites describe the *vault data*, which is synthetic (synthetic_seed.sql), not the X5 load-test path — so the X5 branding was both off-legend and factually wrong, and removing it needs no loader decision: - reference/__init__.py (M3): "grocery reference ... transactional X5 feed" -> "kitchen-appliance reference ... transactional order feed". - bv_order_canonical.sql + postgres/03_business_vault.sql (m8): the "1C / X5 Retail Hero order headers, so real X5 volume flows through" comment claimed X5 volume in a view fed by synthetic order headers -> "1C order headers ... full order volume". - customer_360.sql: "At X5 scale" perf rationale -> "At benchmark scale". Deferred (loader-coupled, needs the replacement-shape decision): the x5_retail_hero loader itself, its by-name references in vault_mapping.py / load_postgres.py / pg_vault_writer.py, schema_dv2.md Sources table, dbt/README scale note, and the load-test infra. CHANGELOG history left as-is. Verified: postgres parse-gate 28/28; diffs minimal, no EOL flips. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../agentflow/dv2/business_vault/bv_order_canonical.sql | 4 ++-- warehouse/agentflow/dv2/dbt/models/marts/customer_360.sql | 2 +- warehouse/agentflow/dv2/postgres/03_business_vault.sql | 4 ++-- warehouse/agentflow/dv2/reference/__init__.py | 8 ++++---- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/warehouse/agentflow/dv2/business_vault/bv_order_canonical.sql b/warehouse/agentflow/dv2/business_vault/bv_order_canonical.sql index ff17cd3a..c64000e2 100644 --- a/warehouse/agentflow/dv2/business_vault/bv_order_canonical.sql +++ b/warehouse/agentflow/dv2/business_vault/bv_order_canonical.sql @@ -32,8 +32,8 @@ WITH UNION ALL SELECT order_hk, order_date, channel, order_status, total_amount, load_ts FROM rv.sat_order_header__bitrix__ala WHERE is_deleted = 0 - -- 1C / X5 Retail Hero order headers (per-branch), so real X5 volume - -- flows through to bv_order_canonical and the branch_pnl mart. + -- 1C order headers (per-branch), so the full order volume flows + -- through to bv_order_canonical and the branch_pnl mart. UNION ALL SELECT order_hk, order_date, channel, order_status, total_amount, load_ts FROM rv.sat_order_header__1c__msk WHERE is_deleted = 0 diff --git a/warehouse/agentflow/dv2/dbt/models/marts/customer_360.sql b/warehouse/agentflow/dv2/dbt/models/marts/customer_360.sql index 81ae281a..08ffaffe 100644 --- a/warehouse/agentflow/dv2/dbt/models/marts/customer_360.sql +++ b/warehouse/agentflow/dv2/dbt/models/marts/customer_360.sql @@ -1,6 +1,6 @@ {# customer_bk first in the sort key: the mart's serving pattern is a point - lookup by business key (load-test 03_customer360_point). At X5 scale the + lookup by business key (load-test 03_customer360_point). At benchmark scale the old (branch, customer_hk) key meant every bk lookup full-scanned the mart: p99 250-470 ms vs the 200 ms point budget. diff --git a/warehouse/agentflow/dv2/postgres/03_business_vault.sql b/warehouse/agentflow/dv2/postgres/03_business_vault.sql index 2233d077..d092b3aa 100644 --- a/warehouse/agentflow/dv2/postgres/03_business_vault.sql +++ b/warehouse/agentflow/dv2/postgres/03_business_vault.sql @@ -35,8 +35,8 @@ WITH UNION ALL SELECT order_hk, order_date, channel, order_status, total_amount, load_ts FROM rv.sat_order_header__bitrix__ala WHERE is_deleted = 0 - -- 1C / X5 Retail Hero order headers (per-branch), so real X5 volume - -- flows through to bv_order_canonical and the branch_pnl mart. + -- 1C order headers (per-branch), so the full order volume flows + -- through to bv_order_canonical and the branch_pnl mart. UNION ALL SELECT order_hk, order_date, channel, order_status, total_amount, load_ts FROM rv.sat_order_header__1c__msk WHERE is_deleted = 0 diff --git a/warehouse/agentflow/dv2/reference/__init__.py b/warehouse/agentflow/dv2/reference/__init__.py index 026da4dd..b1a2e942 100644 --- a/warehouse/agentflow/dv2/reference/__init__.py +++ b/warehouse/agentflow/dv2/reference/__init__.py @@ -1,9 +1,9 @@ """AgentFlow DV2 supplier / product reference. -A real, reproducible grocery reference (suppliers, products, GS1 marking -codes, sourcing) that fills the catalog / tnved / marking slots the -transactional X5 feed leaves empty, and is published to cloud object storage -(a Hugging Face Dataset) as the project's genuine cloud component. +A real, reproducible kitchen-appliance reference (suppliers, products, GS1 +marking codes, sourcing) that fills the catalog / tnved / marking slots the +transactional order feed leaves empty, and is published to cloud object +storage (a Hugging Face Dataset) as the project's genuine cloud component. Public API: build_reference -> deterministic reference tables From 1fce26350dddcac4ee30d708aa619c55be317cc1 Mon Sep 17 00:00:00 2001 From: JuliaEdom Date: Sun, 5 Jul 2026 04:14:21 +0300 Subject: [PATCH 7/7] refactor(g2-x5): de-brand dataflow.html diagram to synthetic source (M2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MAJOR M2 from G2_AUDIT_REPORT.md. The public DV2 diagram titled the lane "Data Vault 2.0 на реальных данных X5" and drew the source node as "X5 Retail Hero CSV / 45.8M строк / Kaggle" — presenting a third-party retailer's dataset as the project's real data, which contradicts the finalized canon (desc_for_julia_fin.md: external source is a deliberate non-goal; the demo runs on synthetic data). Retitled to "на синтетических данных" and replaced the source node with "Синтетический генератор / кухонная легенда · ₽ / clients·products·purchases, детерминир. seed, масштабируемый объём". No fabricated row count — the scale-generator is a separate (Mac-tail) build. Remaining X5 sites are loader-coupled (schema_dv2 Sources table, vault_mapping/load_postgres/pg_vault_writer, the x5_retail_hero loader + its X5-grocery-shaped vault schemas, load-test infra) and belong to the loader replacement sub-project. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/dataflow.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/dataflow.html b/docs/dataflow.html index 910328e9..eb290051 100644 --- a/docs/dataflow.html +++ b/docs/dataflow.html @@ -360,10 +360,10 @@

AgentFlow — движение данных

]; /* DV2 demo lane */ -const DV2={x:40,w:2280,y:1226,title:"Контур DV2.0 — Data Vault 2.0 на реальных данных X5", +const DV2={x:40,w:2280,y:1226,title:"Контур DV2.0 — Data Vault 2.0 на синтетических данных", chip:"демо · kind-кластер · хост 8 GB", flow:[ - {t:"X5 Retail Hero CSV",c:"45.8M строк",d:"clients · products · purchases (4.5 GB), Kaggle-датасет"}, + {t:"Синтетический генератор",c:"кухонная легенда · ₽",d:"clients · products · purchases · детерминир. seed, масштабируемый объём"}, {t:"loader.py",c:"pandas + clickhouse-driver",d:"чанки 50K · PartsThrottle (--max-active-parts) — backpressure по system.parts"}, {t:"ClickHouse",c:"25.5 · k8s",d:"Raw Vault: хабы · линки · сателлиты (ReplacingMergeTree, ветки __1c__/__bitrix__); merge-throttle в config.d"}, {t:"Business Vault",c:"SQL-вьюхи",d:"bv_order_canonical (argMax-дедуп) · bv_customer_mdm × 5 веток"},