fix: close BUG-022..027, BUG-029, BUG-031, BUG-033 from QA sweep

.github/pull_request_template.md

@@ -0,0 +1,51 @@
+# Pull Request
+
+## Summary
+
+<!-- What does this PR change and why? Link related issues. -->
+
+## Type of Change
+
+- [ ] Bug fix
+- [ ] New feature
+- [ ] Refactor / code quality
+- [ ] CI / tooling
+- [ ] Documentation
+
+## Checklist
+
+### Tests Added
+
+- [ ] New/changed behaviour is covered by unit tests
+- [ ] Full backend test suite passes locally (`pytest backend/tests/`)
+- [ ] Legacy migration tests still pass where applicable
+      (`pytest backend/migration/tests/`)
+
+### Verified Workflow Paths
+
+- [ ] I ran the workflow path verifier and it passed:
+      `python3 tools/verify_workflow_paths.py`
+      (verifies every file/script referenced by CI workflows exists)
+- [ ] Any moved/renamed files are updated in workflows, scripts
+      (`scripts/`, `preflight.*`), and docs
+
+### Safety & Security
+
+- [ ] No new `# nosec` / `# noqa` suppressions — or each new suppression is
+      documented in `docs/security/Rationale.md`
+- [ ] No secrets, credentials, or internal paths added to code or logs
+- [ ] Shell scripts validate their inputs (env names, table names, paths)
+
+### API & Data
+
+- [ ] New/changed endpoints declare a Pydantic `response_model`
+- [ ] Database state transitions follow the `RunStatus` lifecycle
+      (`ALLOWED_TRANSITIONS` in `backend/database/models.py`)
+
+## Screenshots / Output
+
+<!-- Test runs, CLI output, or UI screenshots where relevant. -->
+
+## Notes for Reviewers
+
+<!-- Anything reviewers should pay special attention to. -->

.github/workflows/quality-gate.yml

@@ -8,6 +8,38 @@ on:
   pull_request:
 
 jobs:
+  # BUG-023: build and lint the React frontend on every PR. Without this job,
+  # TypeScript errors, unresolved imports, and broken components can merge
+  # to main with all other checks green.
+  frontend-build:
+    name: frontend (build + lint)
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: frontend
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+          cache: 'npm'
+          cache-dependency-path: frontend/package-lock.json
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Lint
+        run: npm run lint
+
+      - name: Build
+        # vite build performs the full production build; nitro then type-checks
+        # the SSR bundle. Any TypeScript error, missing import, or broken route
+        # tree fails this step.
+        run: npm run build
+
   # Fast, deterministic checks — no database, no network beyond pip.
   free-tier:
     name: free-tier (unit, lint, health, evals-p)

.gitignore

@@ -55,3 +55,6 @@ terraform-provider-*.log
 
 # Serena tool workspace
 .serena/
+
+# CI diagnostic dumps
+*-failed.log

api/config.py

@@ -23,6 +23,15 @@ class Settings:
 
     MAX_UPLOAD_BYTES: int = int(os.environ.get("MAX_UPLOAD_BYTES", str(50 * 1024 * 1024)))
 
+    # BUG-027: cap total rows accepted per upload so a pathological file (millions
+    # of rows all failing type-cast) can't monopolise the API. Default 100k rows.
+    MAX_ROWS: int = int(os.environ.get("API_MAX_ROWS", "100000"))
+
+    # BUG-027: cap the per-row error list in the response body. Prevents 100MB
+    # JSON responses when a whole file fails validation. The summary counts
+    # still reflect the true failed-row count.
+    MAX_ROW_ERRORS_REPORTED: int = int(os.environ.get("API_MAX_ROW_ERRORS", "200"))
+
     # Gate for the DELETE /api/csv/files/{id} endpoint.
     # Set API_ALLOW_DESTRUCTIVE=false in shared/prod to prevent accidental table drops.
     allow_destructive: bool = os.environ.get("API_ALLOW_DESTRUCTIVE", "true").lower() in (

api/db.py

@@ -1,25 +1,52 @@
 """Connection pool + one-time schema bootstrap for the uploads registry."""
 
+import logging
+import time
+
 import psycopg2
 import psycopg2.pool
 from psycopg2 import sql
 
 from api.config import settings
 
+logger = logging.getLogger(__name__)
+
 _pool: psycopg2.pool.SimpleConnectionPool | None = None
 
 
-def init_pool() -> None:
+def init_pool(max_attempts: int = 30, base_delay: float = 1.0) -> None:
+    """Create the pool, retrying on connection errors.
+
+    BUG-026: on containerised infra Postgres may not accept connections when
+    the API starts. Retry with exponential backoff (capped at 10s) instead of
+    dying at boot. Total wait is bounded by ``max_attempts`` * cap.
+    """
     global _pool
-    _pool = psycopg2.pool.SimpleConnectionPool(
-        minconn=1,
-        maxconn=8,
-        host=settings.PG_HOST,
-        port=settings.PG_PORT,
-        user=settings.PG_USER,
-        password=settings.PG_PASSWORD,
-        dbname=settings.PG_DATABASE,
-    )
+    last_err: Exception | None = None
+    for attempt in range(1, max_attempts + 1):
+        try:
+            _pool = psycopg2.pool.SimpleConnectionPool(
+                minconn=1,
+                maxconn=8,
+                host=settings.PG_HOST,
+                port=settings.PG_PORT,
+                user=settings.PG_USER,
+                password=settings.PG_PASSWORD,
+                dbname=settings.PG_DATABASE,
+            )
+            if attempt > 1:
+                logger.info("Connected to Postgres on attempt %d", attempt)
+            return
+        except psycopg2.OperationalError as exc:
+            last_err = exc
+            delay = min(base_delay * (2 ** (attempt - 1)), 10.0)
+            logger.warning(
+                "Postgres not reachable (attempt %d/%d): %s. Retrying in %.1fs.",
+                attempt, max_attempts, str(exc).split("\n")[0][:120], delay,
+            )
+            time.sleep(delay)
+    assert last_err is not None
+    raise last_err
 
 
 def close_pool() -> None:

api/main.py

@@ -79,17 +79,27 @@ def pool_exhausted_handler(request: Request, exc: psycopg2.pool.PoolError) -> JS
 
 @app.get("/api/health", tags=["health"])
 def health() -> dict:
+    # BUG-031: verify the uploads schema is present, not just that Postgres is
+    # up. Otherwise a half-bootstrapped API reports "ok" and then every /upload
+    # call throws UndefinedTable.
+    from psycopg2 import sql as _sql
     try:
         with db.Conn() as conn:
             with conn.cursor() as cur:
                 cur.execute("SELECT version()")
                 pg_version = cur.fetchone()[0]
+                cur.execute(
+                    _sql.SQL("SELECT 1 FROM {}.csv_files LIMIT 0").format(
+                        _sql.Identifier(settings.UPLOADS_SCHEMA)
+                    )
+                )
         return {
             "status": "ok",
             "database": settings.PG_DATABASE,
             "host": f"{settings.PG_HOST}:{settings.PG_PORT}",
             "postgres": pg_version.split(" on ")[0],
+            "uploads_schema": settings.UPLOADS_SCHEMA,
         }
     except Exception as exc:  # noqa: BLE001 — surface DB reachability to the UI
         logger.warning("Health check failed: %s", exc)
-        return {"status": "degraded", "error": "database unreachable"}
+        return {"status": "degraded", "error": str(exc).split("\n")[0][:200]}

api/routers/csv_routes.py

@@ -33,10 +33,23 @@ class UploadRequest(BaseModel):
 def preview(req: PreviewRequest) -> dict:
     if len(req.content) > settings.MAX_UPLOAD_BYTES:
         raise HTTPException(413, "File too large")
-    result = build_preview(req.content)
+    # BUG-025: mirror the try/except contract that /upload has. An unexpected
+    # parse error (regex catastrophic backtracking, memory error, etc.) must
+    # not surface as a raw 500 to the frontend.
+    try:
+        result = build_preview(req.content)
+    except Exception as exc:  # noqa: BLE001 — surface any parser failure as structured JSON
+        return {
+            "status": "invalid_structure",
+            "reason": "parse_failed",
+            "message": f"The CSV couldn't be parsed: {str(exc)[:200]}",
+        }
     if result.get("status") == "ok":
         # Suggest a T&E table if the columns fit one (drives the mode picker in the UI)
-        result["teTableMatch"] = match_te_table(result["columns"])
+        try:
+            result["teTableMatch"] = match_te_table(result["columns"])
+        except Exception:  # noqa: BLE001 — T&E match is best-effort, never block the preview
+            result["teTableMatch"] = None
     return result
 
 

api/services/dynamic_loader.py

@@ -82,6 +82,25 @@ def upload_dynamic(
             "logs": logs,
         }
 
+    # BUG-027 guard: reject files above the configured row cap before doing any
+    # per-row work. `rows` includes the header, so subtract one.
+    data_row_count = len(rows) - 1
+    if data_row_count > settings.MAX_ROWS:
+        _log(
+            logs,
+            "error",
+            f"CSV has {data_row_count} data rows; max allowed is {settings.MAX_ROWS}.",
+            "error",
+        )
+        return {
+            "status": "error",
+            "message": (
+                f"CSV has {data_row_count} data rows, but the API is configured to "
+                f"accept at most {settings.MAX_ROWS}. Split the file or raise API_MAX_ROWS."
+            ),
+            "logs": logs,
+        }
+
     schema = settings.UPLOADS_SCHEMA
 
     try:
@@ -235,15 +254,28 @@ def _do_upload(
                     raw_joined.append(cell)
                     ok, val, reason = cast_value(cell, col_types[c])
                     if not ok:
-                        row_errors.append(
-                            {"rowNumber": row_number, "column": columns[c], "value": cell, "reason": reason}
-                        )
+                        # BUG-027: cap row_errors to keep response bodies bounded on
+                        # pathological files. The count in the summary line still
+                        # reflects every failed row.
+                        if len(row_errors) < settings.MAX_ROW_ERRORS_REPORTED:
+                            row_errors.append(
+                                {
+                                    "rowNumber": row_number,
+                                    "column": columns[c],
+                                    "value": cell,
+                                    "reason": reason,
+                                }
+                            )
                         failed = True
                         break
                     values.append(val)
                 if failed:
                     continue
-                row_hash = hashlib.sha256("
".join(raw_joined).encode("utf-8")).hexdigest()
+                # BUG-022: ASCII unit separator (U+001F) between cells prevents
+                # ["ab","cd"] and ["a","bcd"] from hashing to the same value.
+                row_hash = hashlib.sha256(
+                    "\x1f".join(raw_joined).encode("utf-8")
+                ).hexdigest()
                 if row_hash in seen:
                     duplicates += 1
                     continue

frontend/.env.example

@@ -4,3 +4,9 @@ VITE_API_URL=http://localhost:8000
 # Must match the backend's API_KEY env var. Leave blank for local dev where
 # the backend has no API_KEY set (see API_INTEGRATION.md#authentication).
 VITE_API_KEY=
+
+# BUG-029: enable lovable.dev's window.__lovableEvents error-reporting hook.
+# Off by default — self-hosted deployments should leave this unset. Only set to
+# "true" if you deploy on lovable.dev's platform and want their error telemetry.
+VITE_ENABLE_LOVABLE_ANALYTICS=
+

frontend/scripts/README.md

@@ -2,14 +2,14 @@
 
 ## Fixture CSVs (`scripts/fixtures/`)
 
-| File | What it exercises |
-| --- | --- |
-| `basic.csv` | Typed inference — int8, numeric, boolean, date |
-| `ambiguous-headers.csv` | Column names that used to collide with PL/pgSQL loop vars (`i`, `t`, `s`, `idx`) |
-| `quoted.csv` | Quoted fields, embedded newlines, escaped `""` |
-| `bad-types.csv` | Rows that should land in the row-error report, not the table |
-| `dup-rows.csv` | In-file duplicate row de-duplication via `_row_hash` |
-| `empty.csv` | Header-only — should surface the "must have a header row and at least one data row" error |
+| File                    | What it exercises                                                                         |
+| ----------------------- | ----------------------------------------------------------------------------------------- |
+| `basic.csv`             | Typed inference — int8, numeric, boolean, date                                            |
+| `ambiguous-headers.csv` | Column names that used to collide with PL/pgSQL loop vars (`i`, `t`, `s`, `idx`)          |
+| `quoted.csv`            | Quoted fields, embedded newlines, escaped `""`                                            |
+| `bad-types.csv`         | Rows that should land in the row-error report, not the table                              |
+| `dup-rows.csv`          | In-file duplicate row de-duplication via `_row_hash`                                      |
+| `empty.csv`             | Header-only — should surface the "must have a header row and at least one data row" error |
 
 Drop them onto the UI in one batch to exercise the preview + import + diagnostics + error export end to end.
 

frontend/src/hooks/use-local-storage.ts

@@ -2,7 +2,9 @@ import { useCallback, useEffect, useRef, useState } from "react";
 
 const KEY = "csv-migrator:jobs:v1";
 
-export function useLocalStorageState<T>(defaultValue: T): [T, (updater: T | ((prev: T) => T)) => void, () => void] {
+export function useLocalStorageState<T>(
+  defaultValue: T,
+): [T, (updater: T | ((prev: T) => T)) => void, () => void] {
   const [state, setState] = useState<T>(defaultValue);
   const hydrated = useRef(false);