feat(clickhouse): infer mixed-type JSON arrays as Array(Dynamic) on insert (#4095)

## ✅ Checklist

- [ ] I have followed every step in the [contributing
guide](https://github.com/triggerdotdev/trigger.dev/blob/main/CONTRIBUTING.md)
- [x] The PR title follows the convention.
- [ ] I ran and tested the code works

---

## Testing

`pnpm run typecheck --filter @internal/clickhouse` passes. This only
adds a ClickHouse input-format setting to existing insert calls; the
setting affects type inference for newly-inserted/merged data and is
non-destructive to existing rows.

---

## Changelog

Sets
`input_format_json_infer_array_of_dynamic_from_array_of_different_types
= 1` on every native-JSON insert path:

- `task_runs_v2` (`output`, `error`) — `insertTaskRuns`,
`insertTaskRunsCompactArrays`, and the async-insert variants
- `task_events_v1` / `task_events_v2` (`attributes`)
- `metrics_v1`
- `sessions_v1`

### Why

Our JSON columns contain arrays with mixed element types (e.g.
`[{"key":"value"}, "string", "string"]`). With this setting off — which
is the effective default under `24.12` compatibility — ClickHouse infers
those as deeply nested unnamed `Tuple(JSON, Nullable(String), …)` types.
ClickHouse 26.2 introduced `input_format_binary_max_type_complexity`
(default 1000), and those tuple type trees exceeded the limit, causing
background merges to fail with **Code 117**.

With the setting on (the default since 25.8), mixed-type arrays are
inferred as a single `Array(Dynamic)` — a simpler, flatter type
representation that never approaches the complexity limit, even once the
upstream default limit is restored.

Setting this explicitly at insert time keeps behavior deterministic and
version-controlled, so it does not depend on the server profile or a
future compatibility bump. This is a forward-only change: it only
affects newly inserted/merged data and does not rewrite existing parts.
Our read path re-serializes these columns to strings (`toJSONString` via
the materialized `*_text` columns), so the internal Tuple →
Array(Dynamic) representation change is transparent to the application.

### Companion server-side setting

To also apply this on the ClickHouse side (covers merges and any writes
not going through these code paths), set it on the default user:

```sql
ALTER USER default SETTINGS input_format_json_infer_array_of_dynamic_from_array_of_different_types = 1;
```

---

## Screenshots

_N/A_

💯

🤖 Generated with [Claude Code](https://claude.com/claude-code)

https://claude.ai/code/session_01AaChyhestFMBYBWh6bgcCF

---
_Generated by [Claude
Code](https://claude.ai/code/session_01AaChyhestFMBYBWh6bgcCF)_

---------

Co-authored-by: Claude <noreply@anthropic.com>

Author: matt-aitken

.server-changes/clickhouse-array-of-dynamic-inference.md

@@ -0,0 +1,6 @@
+---
+area: webapp
+type: fix
+---
+
+Infer mixed-type JSON arrays as Array(Dynamic) instead of nested tuples when writing run, event, metric, and session data to avoid ClickHouse type-complexity merge failures

internal-packages/clickhouse/src/metrics.ts

@@ -22,6 +22,7 @@ export function insertMetrics(ch: ClickhouseWriter) {
     settings: {
       enable_json_type: 1,
       type_json_skip_duplicated_paths: 1,
+      input_format_json_infer_array_of_dynamic_from_array_of_different_types: 1,
       input_format_json_throw_on_bad_escape_sequence: 0,
       input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects: 1,
     },

internal-packages/clickhouse/src/sessions.ts

@@ -118,6 +118,7 @@ export function insertSessionsCompactArrays(ch: ClickhouseWriter, settings?: Cli
     settings: {
       enable_json_type: 1,
       type_json_skip_duplicated_paths: 1,
+      input_format_json_infer_array_of_dynamic_from_array_of_different_types: 1,
       ...settings,
     },
   });
@@ -131,6 +132,7 @@ export function insertSessions(ch: ClickhouseWriter, settings?: ClickHouseSettin
     settings: {
       enable_json_type: 1,
       type_json_skip_duplicated_paths: 1,
+      input_format_json_infer_array_of_dynamic_from_array_of_different_types: 1,
       ...settings,
     },
   });

internal-packages/clickhouse/src/taskEvents.ts

@@ -31,6 +31,7 @@ export function insertTaskEvents(ch: ClickhouseWriter, settings?: ClickHouseSett
     settings: {
       enable_json_type: 1,
       type_json_skip_duplicated_paths: 1,
+      input_format_json_infer_array_of_dynamic_from_array_of_different_types: 1,
       input_format_json_throw_on_bad_escape_sequence: 0,
       input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects: 1,
       ...settings,
@@ -206,6 +207,7 @@ export function insertTaskEventsV2(ch: ClickhouseWriter, settings?: ClickHouseSe
     settings: {
       enable_json_type: 1,
       type_json_skip_duplicated_paths: 1,
+      input_format_json_infer_array_of_dynamic_from_array_of_different_types: 1,
       input_format_json_throw_on_bad_escape_sequence: 0,
       input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects: 1,
       ...settings,

internal-packages/clickhouse/src/taskRuns.test.ts

@@ -156,6 +156,111 @@ describe("Task Runs V2", () => {
     );
   });
 
+  clickhouseTest(
+    "should insert and read back JSON arrays with mixed element types",
+    async ({ clickhouseContainer }) => {
+      // Regression test for input_format_json_infer_array_of_dynamic_from_array_of_different_types.
+      // Arrays with mixed element types (e.g. [1, "hello", {...}, [...]]) must be inferred as
+      // Array(Dynamic) rather than deeply nested Tuple types, which otherwise blow up the binary
+      // type-complexity limit during background merges (ClickHouse Code 117).
+      const client = new ClickhouseClient({
+        name: "test",
+        url: clickhouseContainer.getConnectionUrl(),
+      });
+
+      const insert = insertTaskRunsCompactArrays(client, {
+        async_insert: 0, // turn off async insert for this test
+      });
+
+      const mixedArray = [1, "hello", { nested: "object" }, [1, 2, 3]];
+
+      const now = Date.now();
+      const taskRunData: TaskRunInsertArray = [
+        "env_mixed", // environment_id
+        "org_mixed", // organization_id
+        "project_mixed", // project_id
+        "run_mixed", // run_id
+        now, // updated_at
+        now, // created_at
+        "COMPLETED_SUCCESSFULLY", // status
+        "DEVELOPMENT", // environment_type
+        "friendly_mixed", // friendly_id
+        1, // attempt
+        "V2", // engine
+        "my-task", // task_identifier
+        "my-queue", // queue
+        "", // schedule_id
+        "", // batch_id
+        null, // completed_at
+        null, // started_at
+        null, // executed_at
+        null, // delay_until
+        null, // queued_at
+        null, // expired_at
+        0, // usage_duration_ms
+        0, // cost_in_cents
+        0, // base_cost_in_cents
+        { data: { items: mixedArray } }, // output
+        { data: null }, // error
+        "", // error_fingerprint
+        [], // tags
+        "", // task_version
+        "", // sdk_version
+        "", // cli_version
+        "", // machine_preset
+        "", // root_run_id
+        "", // parent_run_id
+        0, // depth
+        "span_mixed", // span_id
+        "trace_mixed", // trace_id
+        "", // idempotency_key
+        "", // idempotency_key_user
+        "", // idempotency_key_scope
+        "", // expiration_ttl
+        true, // is_test
+        "1", // _version
+        0, // _is_deleted
+        "", // concurrency_key
+        [], // bulk_action_group_ids
+        "", // worker_queue
+        "", // region
+        "", // plan_type
+        null, // max_duration_in_seconds
+        "", // trigger_source
+        "", // root_trigger_source
+        "", // task_kind
+        null, // is_warm_start
+      ];
+
+      const [insertError, insertResult] = await insert([taskRunData]);
+
+      expect(insertError).toBeNull();
+      expect(insertResult).toEqual(expect.objectContaining({ executed: true }));
+      expect(insertResult?.summary?.written_rows).toEqual("1");
+
+      // output_text is a materialized String column that extracts the `data` field, so it
+      // round-trips the mixed-type array back out as JSON regardless of the internal storage type.
+      const query = client.query({
+        name: "query-task-runs-mixed",
+        query:
+          "SELECT run_id, output_text FROM trigger_dev.task_runs_v2 WHERE run_id = {run_id: String}",
+        schema: z.object({
+          run_id: z.string(),
+          output_text: z.string(),
+        }),
+        params: z.object({
+          run_id: z.string(),
+        }),
+      });
+
+      const [queryError, result] = await query({ run_id: "run_mixed" });
+
+      expect(queryError).toBeNull();
+      expect(result).toHaveLength(1);
+      expect(JSON.parse(result![0].output_text)).toEqual({ items: mixedArray });
+    }
+  );
+
   clickhouseTest("should deduplicate on the _version column", async ({ clickhouseContainer }) => {
     const client = new ClickhouseClient({
       name: "test",

internal-packages/clickhouse/src/taskRuns.ts

@@ -211,6 +211,7 @@ export function insertTaskRunsCompactArrays(ch: ClickhouseWriter, settings?: Cli
     settings: {
       enable_json_type: 1,
       type_json_skip_duplicated_paths: 1,
+      input_format_json_infer_array_of_dynamic_from_array_of_different_types: 1,
       ...settings,
     },
   });
@@ -225,6 +226,7 @@ export function insertTaskRuns(ch: ClickhouseWriter, settings?: ClickHouseSettin
     settings: {
       enable_json_type: 1,
       type_json_skip_duplicated_paths: 1,
+      input_format_json_infer_array_of_dynamic_from_array_of_different_types: 1,
       ...settings,
     },
   });
@@ -349,6 +351,7 @@ export function insertRawTaskRunPayloadsCompactArrays(
       async_insert_busy_timeout_ms: 1000,
       enable_json_type: 1,
       type_json_skip_duplicated_paths: 1,
+      input_format_json_infer_array_of_dynamic_from_array_of_different_types: 1,
       ...settings,
     },
   });
@@ -367,6 +370,7 @@ export function insertRawTaskRunPayloads(ch: ClickhouseWriter, settings?: ClickH
       async_insert_busy_timeout_ms: 1000,
       enable_json_type: 1,
       type_json_skip_duplicated_paths: 1,
+      input_format_json_infer_array_of_dynamic_from_array_of_different_types: 1,
       ...settings,
     },
   });