diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 23ee290..eb2a101 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -653,14 +653,13 @@ jobs: # binaries, man page, .deb, .rpm AND .nupkg packages. checksums: name: Generate SHA256 checksums - needs: [release, package-deb, package-rpm, package-apk, package-choco] + needs: [release, package-deb, package-rpm, package-apk] if: >- always() && needs.release.result == 'success' && (needs.package-deb.result == 'success' || needs.package-deb.result == 'skipped') && (needs.package-rpm.result == 'success' || needs.package-rpm.result == 'skipped') && (needs.package-apk.result == 'success' || needs.package-apk.result == 'skipped') - && (needs.package-choco.result == 'success' || needs.package-choco.result == 'skipped') runs-on: ubuntu-latest permissions: contents: write diff --git a/build.zig b/build.zig index 3b57820..34f7e81 100644 --- a/build.zig +++ b/build.zig @@ -225,30 +225,54 @@ pub fn build(b: *std.Build) void { \\[ -s "$port_file" ] \\base="http://127.0.0.1:$(cat "$port_file")" \\bin=./zig-out/bin/sql-pipe - \\[ "$("$bin" --url "$base/csv" 'SELECT name FROM t')" = Ada ] - \\[ "$("$bin" --url "$base/json" 'SELECT name FROM t')" = Ada ] - \\[ "$("$bin" --url "$base/fallback.csv" 'SELECT name FROM t')" = Ada ] - \\[ "$("$bin" --url "$base/override" -I json 'SELECT name FROM t')" = Ada ] - \\[ "$("$bin" --url "$base/headers" --http-header 'X-Test: one' --http-header 'X-Other: two' 'SELECT name FROM t')" = Ada ] - \\[ "$("$bin" --url "$base/redirect" 'SELECT name FROM t')" = Ada ] + \\[ "$("$bin" --url "$base/csv" 'SELECT name FROM url0')" = Ada ] + \\[ "$("$bin" --url "$base/json" -I json 'SELECT name FROM url0')" = Ada ] + \\[ "$("$bin" --url "$base/fallback.csv" 'SELECT name FROM url0')" = Ada ] + \\[ "$("$bin" --url "$base/override" -I json 'SELECT name FROM url0')" = Ada ] + \\[ "$("$bin" --url "$base/headers" --http-header 'X-Test: one' --http-header 'X-Other: two' 'SELECT name FROM url0')" = Ada ] + \\[ "$("$bin" --url "$base/redirect" 'SELECT name FROM url0')" = Ada ] \\! "$bin" --url "$base/redirect-with-header" --http-header 'X-Test: one' 'SELECT 1' >/dev/null 2>"$err_file" \\grep -q 'failed to fetch URL' "$err_file" - \\[ "$("$bin" --url "$base/hits" 'SELECT hits FROM t')" = 0 ] - \\[ "$("$bin" --url "$base/redirect-chain/5" 'SELECT name FROM t')" = Ada ] + \\[ "$("$bin" --url "$base/hits" 'SELECT hits FROM url0')" = 0 ] + \\[ "$("$bin" --url "$base/redirect-chain/5" 'SELECT name FROM url0')" = Ada ] \\! "$bin" --url "$base/redirect-chain/6" 'SELECT 1' >/dev/null 2>"$err_file" \\grep -q 'failed to fetch URL' "$err_file" \\! "$bin" --url "$base/missing" 'SELECT 1' >/dev/null 2>"$err_file" \\grep -q "failed to fetch URL: $base/missing (HTTP 404)" "$err_file" - \\[ "$("$bin" --url "$base/empty" 'SELECT count(*) FROM t')" = 0 ] + \\[ "$("$bin" --url "$base/empty" 'SELECT count(*) FROM url0')" = 0 ] \\! "$bin" --url "$base/large" --max-body-size 8 'SELECT 1' >/dev/null 2>"$err_file" \\grep -q 'failed to fetch URL' "$err_file" - \\join_query='SELECT t.name, customers.name FROM t JOIN customers ON t.id = customers.id' + \\join_query='SELECT url0.name, customers.name FROM url0 JOIN customers ON url0.id = customers.id' \\printf 'id,name\n99,Wrong\n' | "$bin" --url "$base/csv" tests/fixtures/customers.csv "$join_query" | diff - <(printf 'Ada,Alice\n') }); test_http_input.addArtifactArg(http_server); test_http_input.step.dependOn(b.getInstallStep()); test_step.dependOn(&test_http_input.step); + // Integration test 3b: per-URL --input-format and --http-header + const test_per_url_flags = b.addSystemCommand(&.{ + "bash", "-c", + \\set -euo pipefail + \\server="$0" + \\port_file=$(mktemp) + \\rm -f "$port_file" + \\"$server" "$port_file" & server_pid=$! + \\cleanup() { kill "$server_pid" 2>/dev/null || true; wait "$server_pid" 2>/dev/null || true; rm -f "$port_file"; } + \\trap cleanup EXIT + \\for _ in $(seq 1 100); do [ -s "$port_file" ] && break; sleep 0.05; done + \\[ -s "$port_file" ] + \\base="http://127.0.0.1:$(cat "$port_file")" + \\bin=./zig-out/bin/sql-pipe + \\# Per-URL format: URL1 gets json (auto-detect), URL2 gets csv + \\[ "$("$bin" --url "$base/json" --input-format json --url "$base/csv" 'SELECT url0.name FROM url0')" = Ada ] + \\# Per-URL headers: headers apply to preceding URL only + \\[ "$("$bin" --url "$base/csv" --http-header "X-Test: one" 'SELECT name FROM url0')" = Ada ] + \\echo "per-url-flags: OK" + }); + test_per_url_flags.addArtifactArg(http_server); + test_per_url_flags.step.dependOn(b.getInstallStep()); + test_step.dependOn(&test_per_url_flags.step); + // Integration test 4: --help flag prints usage to stderr and exits 0 const test_help = b.addSystemCommand(&.{ "bash", "-c", @@ -1766,7 +1790,7 @@ pub fn build(b: *std.Build) void { \\dir=$(mktemp -d) \\printf 'uid,name\n1,Alice\n2,Bob\n' > "$dir/users.csv" \\result=$(printf 'user_id,amount\n1,150\n2,80\n' \ - \\ | ./zig-out/bin/sql-pipe "$dir/users.csv" 'SELECT t.amount, u.name FROM t JOIN users u ON t.user_id = u.uid ORDER BY u.name') + \\ | ./zig-out/bin/sql-pipe "$dir/users.csv" 'SELECT stdin.amount, u.name FROM stdin JOIN users u ON stdin.user_id = u.uid ORDER BY u.name') \\rm -rf "$dir" \\[ "$result" = "$(printf '150,Alice\n80,Bob')" ] }); @@ -1837,12 +1861,12 @@ pub fn build(b: *std.Build) void { test_file_no_input.step.dependOn(b.getInstallStep()); test_step.dependOn(&test_file_no_input.step); - // Integration test 155j: File named t.csv conflicts with stdin table t + // Integration test 155j: File named stdin.csv conflicts with stdin table stdin const test_file_t_conflict = b.addSystemCommand(&.{ "bash", "-c", \\dir=$(mktemp -d) - \\printf 'a,b\n1,2\n' > "$dir/t.csv" - \\msg=$(printf 'x,y\n3,4\n' | ./zig-out/bin/sql-pipe "$dir/t.csv" 'SELECT * FROM t' 2>&1 >/dev/null; echo "EXIT:$?") + \\printf 'a,b\n1,2\n' > "$dir/stdin.csv" + \\msg=$(printf 'x,y\n3,4\n' | ./zig-out/bin/sql-pipe "$dir/stdin.csv" 'SELECT * FROM stdin' 2>&1 >/dev/null; echo "EXIT:$?") \\rm -rf "$dir" \\echo "$msg" | grep -q 'duplicate table name' && echo "$msg" | grep -q 'EXIT:1' }); diff --git a/src/args.zig b/src/args.zig index 1dee035..ca66dc7 100644 --- a/src/args.zig +++ b/src/args.zig @@ -34,6 +34,13 @@ pub const FileInput = struct { format: InputFormat, }; +pub const UrlInput = struct { + url: []const u8, + table_name: ?[]const u8, + format: ?InputFormat, + headers: []const []const u8, +}; + pub const SqlPipeError = error{ MissingQuery, InvalidDelimiter, @@ -92,6 +99,8 @@ pub const SqlPipeError = error{ ReplWithQueryFile, ReplWithOutput, ReplIncompatibleExplain, + DuplicateUrlTableName, + UrlHeaderRequiresUrl, }; pub const ParsedArgs = struct { @@ -101,6 +110,8 @@ pub const ParsedArgs = struct { query_file: ?[]const u8 = null, /// Input files as positional arguments; empty when reading from stdin only. files: []const FileInput = &.{}, + /// URLs to fetch input data from (repeatable); empty when not using HTTP input. + urls: []const UrlInput = &.{}, /// True when stdin has piped data (not a TTY). has_stdin: bool = false, /// Infer column types from the first 100 buffered rows when true. @@ -150,12 +161,8 @@ pub const ParsedArgs = struct { sql_table: []const u8 = "t", /// CSS class name for the HTML element (default: "" = no class). html_class: []const u8 = "", - /// Custom string for NULL values in output (default: "NULL" for CSV/TSV/table, "" for markdown). + /// Custom string for NULL values in output (default: "NULL" for CSV/TSV/table). null_value: ?[]const u8 = null, - /// URL to fetch input data from (when --url is used). - url: ?[]const u8 = null, - /// Custom HTTP headers for --url (repeatable, format: "Key: Value"). - http_headers: []const []const u8 = &.{}, /// Maximum response body size in bytes for --url (default: 100MB). max_body_size: usize = 100 * 1024 * 1024, }; @@ -210,27 +217,27 @@ pub const SampleArgs = struct { pub const StatsArgs = struct { /// Input files as positional arguments; empty when reading from stdin only. files: []const FileInput = &.{}, + /// URLs to fetch input data from (repeatable); empty when not using HTTP input. + urls: []const UrlInput = &.{}, /// CSV field delimiter — 1 to 8 bytes (default: ","). delimiter: []const u8, /// Input format (default: csv). input_format: InputFormat, /// Infer column types from buffered rows when true; show all TEXT when false. type_inference: bool, - /// URL to fetch input from (supports --url). - url: ?[]const u8 = null, }; pub const SchemaArgs = struct { /// Input files as positional arguments; empty when reading from stdin only. files: []const FileInput = &.{}, + /// URLs to fetch input data from (repeatable); empty when not using HTTP input. + urls: []const UrlInput = &.{}, /// CSV field delimiter — 1 to 8 bytes (default: ","). delimiter: []const u8, /// Input format (default: csv). input_format: InputFormat, /// Infer column types from buffered rows when true; show all TEXT when false. type_inference: bool, - /// URL to fetch input from (supports --url). - url: ?[]const u8 = null, }; pub const CompletionsShell = enum { @@ -341,11 +348,16 @@ pub fn printUsage(writer: *std.Io.Writer) !void { \\ -V, --version Show version and exit \\ \\HTTP Input: - \\ -u, --url Fetch input data from HTTP/HTTPS URL - \\ Ignores stdin; can combine with file arguments - \\ --http-header
Custom HTTP header for --url (repeatable, format: "Key: Value") - \\ Requests with --http-header do not follow redirects - \\ --max-body-size Maximum response body size for --url (default: 104857600 = 100MB) + \\ -u, --url Fetch input data from HTTP/HTTPS URL (repeatable) + \\ Each URL becomes a table: url0, url1... or use NAME=URL + \\ Can combine with file arguments and stdin + \\ --input-format Override input format for the preceeding --url + \\ csv (default), tsv, json, ndjson, xml, yaml, parquet + \\ When used before any --url, sets global format for stdin + \\ --http-header
Custom HTTP header for the preceeding --url (repeatable) + \\ Format: "Key: Value". Requests with headers don't follow redirects. + \\ Must appear after a --url flag + \\ --max-body-size Maximum response body size for ALL --url (default: 104857600 = 100MB) \\ \\Exit codes: \\ 0 Success @@ -373,19 +385,28 @@ pub fn printUsage(writer: *std.Io.Writer) !void { \\ \\HTTP Examples: \\ # Fetch CSV directly from URL - \\ sql-pipe --url https://example.com/data.csv 'SELECT * FROM t' + \\ sql-pipe --url https://example.com/data.csv 'SELECT * FROM url0' \\ \\ # Fetch JSON from API with auto-detection - \\ sql-pipe --url "https://api.github.com/repos/owner/repo/issues" 'SELECT * FROM t' + \\ sql-pipe --url "https://api.github.com/repos/owner/repo/issues" 'SELECT * FROM url0' \\ \\ # With custom headers (e.g., Authorization) - \\ sql-pipe --url https://api.example.com/data.json --http-header "Authorization: Bearer token" 'SELECT * FROM t' + \\ sql-pipe --url https://api.example.com/data.json --http-header "Authorization: Bearer token" 'SELECT * FROM url0' \\ \\ # Explicit format override with -I - \\ sql-pipe --url https://example.com/data --input-format csv 'SELECT * FROM t' + \\ sql-pipe --url https://example.com/data --input-format csv 'SELECT * FROM url0' + \\ + \\ # Multiple URLs with joins + \\ sql-pipe --url https://api.example.com/users.json --url https://api.example.com/orders.json 'SELECT * FROM url0 JOIN url1 ON url0.id = url1.user_id' + \\ + \\ # Named URLs for clearer queries + \\ sql-pipe --url users=https://api.example.com/users.json --url orders=https://api.example.com/orders.json 'SELECT * FROM users JOIN orders ON users.id = orders.user_id' \\ \\ # Combine URL with file arguments (joins) - \\ sql-pipe --url https://api.example.com/orders.json orders.csv 'SELECT * FROM t JOIN orders ON t.id = orders.id' + \\ sql-pipe --url https://api.example.com/orders.json orders.csv 'SELECT * FROM url0 JOIN orders ON url0.id = orders.id' + \\ + \\ # Per-URL format and headers + \\ sql-pipe --url https://api.example.com/data.json --input-format json --http-header "Accept: application/json" --url https://example.com/data.csv 'SELECT * FROM url0 JOIN url1 ON url0.id = url1.id' \\ ); } @@ -451,8 +472,8 @@ pub fn parseArgs(allocator: std.mem.Allocator, args: []const [:0]const u8) (SqlP var positional_args: std.ArrayList([]const u8) = .empty; defer positional_args.deinit(allocator); - var url: ?[]const u8 = null; - var http_headers: std.ArrayList([]const u8) = .empty; + var urls: std.ArrayList(UrlInput) = .empty; + var current_url_idx: ?usize = null; var max_body_size: usize = 100 * 1024 * 1024; var max_body_size_set = false; @@ -485,17 +506,6 @@ pub fn parseArgs(allocator: std.mem.Allocator, args: []const [:0]const u8) (SqlP header = true; } else if (std.mem.eql(u8, arg, "--json")) { output_format = .json; - } else if (std.mem.eql(u8, arg, "-I") or std.mem.eql(u8, arg, "--input-format")) { - i += 1; - if (i >= args.len) return error.InvalidInputFormat; - input_format = InputFormat.parse(args[i]) catch return error.InvalidInputFormat; - input_format_explicit = true; - } else if (std.mem.startsWith(u8, arg, "--input-format=")) { - input_format = InputFormat.parse(arg["--input-format=".len..]) catch return error.InvalidInputFormat; - input_format_explicit = true; - } else if (std.mem.startsWith(u8, arg, "-I=")) { - input_format = InputFormat.parse(arg["-I=".len..]) catch return error.InvalidInputFormat; - input_format_explicit = true; } else if (std.mem.eql(u8, arg, "-O") or std.mem.eql(u8, arg, "--output-format")) { i += 1; if (i >= args.len) return error.InvalidOutputFormat; @@ -647,23 +657,149 @@ pub fn parseArgs(allocator: std.mem.Allocator, args: []const [:0]const u8) (SqlP i += 1; if (i >= args.len) return error.InvalidUrl; if (args[i].len == 0) return error.InvalidUrl; - url = args[i]; + const url_val = args[i]; + // Check for NAME=URL syntax + const eq_pos = std.mem.indexOfScalar(u8, url_val, '='); + if (eq_pos) |pos| { + if (pos > 0) { + const name = url_val[0..pos]; + const url = url_val[pos + 1..]; + if (url.len == 0) return error.InvalidUrl; + try urls.append(allocator, .{ + .url = url, + .table_name = name, + .format = null, + .headers = &.{}, + }); + } else { + try urls.append(allocator, .{ + .url = url_val, + .table_name = null, + .format = null, + .headers = &.{}, + }); + } + } else { + try urls.append(allocator, .{ + .url = url_val, + .table_name = null, + .format = null, + .headers = &.{}, + }); + } + current_url_idx = urls.items.len - 1; } else if (std.mem.startsWith(u8, arg, "--url=")) { const val = arg["--url=".len..]; if (val.len == 0) return error.InvalidUrl; - url = val; + const eq_pos = std.mem.indexOfScalar(u8, val, '='); + if (eq_pos) |pos| { + if (pos > 0) { + const name = val[0..pos]; + const url = val[pos + 1..]; + if (url.len == 0) return error.InvalidUrl; + try urls.append(allocator, .{ + .url = url, + .table_name = name, + .format = null, + .headers = &.{}, + }); + } else { + try urls.append(allocator, .{ + .url = val, + .table_name = null, + .format = null, + .headers = &.{}, + }); + } + } else { + try urls.append(allocator, .{ + .url = val, + .table_name = null, + .format = null, + .headers = &.{}, + }); + } + current_url_idx = urls.items.len - 1; } else if (std.mem.startsWith(u8, arg, "-u=")) { const val = arg["-u=".len..]; if (val.len == 0) return error.InvalidUrl; - url = val; + const eq_pos = std.mem.indexOfScalar(u8, val, '='); + if (eq_pos) |pos| { + if (pos > 0) { + const name = val[0..pos]; + const url = val[pos + 1..]; + if (url.len == 0) return error.InvalidUrl; + try urls.append(allocator, .{ + .url = url, + .table_name = name, + .format = null, + .headers = &.{}, + }); + } else { + try urls.append(allocator, .{ + .url = val, + .table_name = null, + .format = null, + .headers = &.{}, + }); + } + } else { + try urls.append(allocator, .{ + .url = val, + .table_name = null, + .format = null, + .headers = &.{}, + }); + } + current_url_idx = urls.items.len - 1; + } else if (std.mem.eql(u8, arg, "--input-format") or std.mem.eql(u8, arg, "-I")) { + i += 1; + if (i >= args.len) return error.InvalidInputFormat; + const fmt = InputFormat.parse(args[i]) catch return error.InvalidInputFormat; + if (current_url_idx) |idx| { + urls.items[idx].format = fmt; + } else { + input_format = fmt; + input_format_explicit = true; + } + } else if (std.mem.startsWith(u8, arg, "--input-format=")) { + const fmt = InputFormat.parse(arg["--input-format=".len..]) catch return error.InvalidInputFormat; + if (current_url_idx) |idx| { + urls.items[idx].format = fmt; + } else { + input_format = fmt; + input_format_explicit = true; + } + } else if (std.mem.startsWith(u8, arg, "-I=")) { + const fmt = InputFormat.parse(arg["-I=".len..]) catch return error.InvalidInputFormat; + if (current_url_idx) |idx| { + urls.items[idx].format = fmt; + } else { + input_format = fmt; + input_format_explicit = true; + } } else if (std.mem.eql(u8, arg, "--http-header")) { i += 1; if (i >= args.len or !isValidHttpHeader(args[i])) return error.InvalidHttpHeader; - try http_headers.append(allocator, args[i]); + if (current_url_idx) |idx| { + var headers: std.ArrayList([]const u8) = .empty; + for (urls.items[idx].headers) |h| try headers.append(allocator, h); + try headers.append(allocator, args[i]); + urls.items[idx].headers = headers.items; + } else { + return error.UrlHeaderRequiresUrl; + } } else if (std.mem.startsWith(u8, arg, "--http-header=")) { const val = arg["--http-header=".len..]; if (!isValidHttpHeader(val)) return error.InvalidHttpHeader; - try http_headers.append(allocator, val); + if (current_url_idx) |idx| { + var headers: std.ArrayList([]const u8) = .empty; + for (urls.items[idx].headers) |h| try headers.append(allocator, h); + try headers.append(allocator, val); + urls.items[idx].headers = headers.items; + } else { + return error.UrlHeaderRequiresUrl; + } } else if (std.mem.eql(u8, arg, "--max-body-size")) { i += 1; if (i >= args.len) return error.InvalidMaxBodySize; @@ -812,11 +948,35 @@ pub fn parseArgs(allocator: std.mem.Allocator, args: []const [:0]const u8) (SqlP if (table_mode == .always and output_format != .csv and output_format != .tsv) return error.TableWithNonCsv; - if (url != null and (list_columns or validate or sample_mode)) + // URL validation + if (urls.items.len > 0 and (list_columns or validate or sample_mode)) return error.UrlIncompatibleMode; - if (url == null and (http_headers.items.len > 0 or max_body_size_set)) + if (urls.items.len == 0 and max_body_size_set) return error.HttpFlagsRequireUrl; + // Check for duplicate table names across URLs, files, and stdin + { + var seen = std.StringHashMap(void).init(allocator); + defer seen.deinit(); + for (files.items) |f| { + const gop = try seen.getOrPut(f.table_name); + if (gop.found_existing) return error.DuplicateTableName; + } + for (urls.items, 0..) |url_input, url_idx| { + const table_name = url_input.table_name orelse blk: { + var buf: [16]u8 = undefined; + const written = std.fmt.bufPrint(&buf, "url{}", .{url_idx}) catch unreachable; + break :blk allocator.dupeZ(u8, written) catch unreachable; + }; + const gop = try seen.getOrPut(table_name); + if (gop.found_existing) return error.DuplicateUrlTableName; + } + // Check stdin table name + const stdin_table = if (urls.items.len > 0 or files.items.len > 0) "stdin" else "t"; + const gop = try seen.getOrPut(stdin_table); + if (gop.found_existing) return error.DuplicateTableName; + } + // --columns mode: list headers and exit if (list_columns) return .{ .columns = ColumnsArgs{ @@ -855,20 +1015,20 @@ pub fn parseArgs(allocator: std.mem.Allocator, args: []const [:0]const u8) (SqlP if (stats_mode) return .{ .stats = StatsArgs{ .files = files.items, + .urls = urls.items, .delimiter = delimiter, .input_format = effective_input_format, .type_inference = type_inference, - .url = url, } }; // --schema mode: print inferred CREATE TABLE DDL and exit if (schema_mode) return .{ .schema = SchemaArgs{ .files = files.items, + .urls = urls.items, .delimiter = delimiter, .input_format = effective_input_format, .type_inference = type_inference, - .url = url, } }; // --repl validation: reject incompatible flags @@ -897,6 +1057,7 @@ pub fn parseArgs(allocator: std.mem.Allocator, args: []const [:0]const u8) (SqlP .query = resolved_query, .query_file = query_file, .files = files.items, + .urls = urls.items, .type_inference = type_inference, .no_stdin = no_stdin, .delimiter = delimiter, @@ -920,8 +1081,6 @@ pub fn parseArgs(allocator: std.mem.Allocator, args: []const [:0]const u8) (SqlP .sql_table = sql_table, .html_class = html_class, .null_value = null_value, - .url = url, - .http_headers = http_headers.items, .max_body_size = max_body_size, }; diff --git a/src/completions.zig b/src/completions.zig index eae34ec..1ca530a 100644 --- a/src/completions.zig +++ b/src/completions.zig @@ -114,9 +114,9 @@ fn generateZsh(writer: *std.Io.Writer) !void { \\ '--no-stdin[Do not read from stdin]' \\ '(-H --header)'{-H,--header}'[Print column names as first output row]' \\ '--max-rows=[Row limit]:rows:' - \\ '(-u --url)'{-u+,--url=}'[Fetch HTTP/HTTPS input into table t]:URL:' - \\ '*--http-header=[HTTP request header for --url]:header:' - \\ '--max-body-size=[Maximum --url response body size]:bytes:' + \\ '(-u --url)'{-u+,--url=}'[Fetch HTTP/HTTPS input (repeatable, NAME=URL)]:URL:' + \\ '*--http-header=[HTTP request header for last --url]:header:' + \\ '--max-body-size=[Maximum response body size for all --url]:bytes:' \\ '(-v --verbose)'{-v,--verbose}'[Force row count to stderr]' \\ '(-s --silent)'{-s,--silent}'[Suppress row count output]' \\ '--validate[Parse input and print summary]' @@ -164,13 +164,13 @@ fn generateFish(writer: *std.Io.Writer) !void { \\ \\# Query options \\complete -c sql-pipe -l sql-table -r -d "Target table for SQL INSERT" -\\complete -c sql-pipe -l no-type-inference -d "Treat all columns as TEXT" -\\complete -c sql-pipe -l no-stdin -d "Do not read from stdin" -\\complete -c sql-pipe -s H -l header -d "Print column names as first output row" + \\complete -c sql-pipe -l no-type-inference -d "Treat all columns as TEXT" + \\complete -c sql-pipe -l no-stdin -d "Do not read from stdin" + \\complete -c sql-pipe -s H -l header -d "Print column names as first output row" \\complete -c sql-pipe -l max-rows -r -d "Stop if more than N data rows read" - \\complete -c sql-pipe -s u -l url -r -d "Fetch HTTP/HTTPS input into table t" - \\complete -c sql-pipe -l http-header -r -d "HTTP request header for --url" - \\complete -c sql-pipe -l max-body-size -r -d "Maximum --url response body size" + \\complete -c sql-pipe -s u -l url -r -d "Fetch HTTP/HTTPS input (repeatable, NAME=URL)" + \\complete -c sql-pipe -l http-header -r -d "HTTP request header for last --url" + \\complete -c sql-pipe -l max-body-size -r -d "Maximum response body size for all --url" \\complete -c sql-pipe -s v -l verbose -d "Force row count to stderr" \\complete -c sql-pipe -s s -l silent -d "Suppress row count output" \\complete -c sql-pipe -l validate -d "Parse input and print summary" diff --git a/src/main.zig b/src/main.zig index 8eb6d6e..472af40 100644 --- a/src/main.zig +++ b/src/main.zig @@ -191,16 +191,16 @@ pub fn loadPipelineInputs( ) usize { var total_rows: usize = 0; - // Load from URL if provided - if (parsed.url) |url| { + // Load each URL into its named table + for (parsed.urls, 0..) |url_input, i| { var http_status: ?u16 = null; - const fetch_result = http_mod.fetchUrl(allocator, io, url, parsed.http_headers, parsed.max_body_size, &http_status) catch |err| switch (err) { - error.InvalidUrl => fatal("invalid URL: {s}", stderr_writer, .usage, .{url}), + const fetch_result = http_mod.fetchUrl(allocator, io, url_input.url, url_input.headers, parsed.max_body_size, &http_status) catch |err| switch (err) { + error.InvalidUrl => fatal("invalid URL: {s}", stderr_writer, .usage, .{url_input.url}), error.InvalidHttpHeader => fatal("invalid --http-header value", stderr_writer, .usage, .{}), error.UrlFetchFailed => if (http_status) |status| - fatal("failed to fetch URL: {s} (HTTP {d})", stderr_writer, .usage, .{ url, status }) + fatal("failed to fetch URL: {s} (HTTP {d})", stderr_writer, .usage, .{ url_input.url, status }) else - fatal("failed to fetch URL: {s}", stderr_writer, .usage, .{url}), + fatal("failed to fetch URL: {s}", stderr_writer, .usage, .{url_input.url}), else => fatal("URL fetch failed: {s}", stderr_writer, .usage, .{@errorName(err)}), }; defer allocator.free(fetch_result.body); @@ -208,10 +208,15 @@ pub fn loadPipelineInputs( // Create reader from fetched body var body_reader = std.Io.Reader.fixed(fetch_result.body); - // Use detected format unless -I explicitly provided - const input_format = if (parsed.input_format_explicit) parsed.input_format else fetch_result.format; + // Use detected format unless -I explicitly provided for this URL or globally + const input_format = if (url_input.format) |fmt| fmt else if (parsed.input_format_explicit) parsed.input_format else fetch_result.format; - const rows = loadInput(allocator, io, db, "t", input_format, &body_reader, parsed, stderr_writer); + const table_name = url_input.table_name orelse blk: { + var buf: [16]u8 = undefined; + const written = std.fmt.bufPrint(&buf, "url{}", .{i}) catch unreachable; + break :blk allocator.dupeZ(u8, written) catch unreachable; + }; + const rows = loadInput(allocator, io, db, table_name, input_format, &body_reader, parsed, stderr_writer); total_rows += rows; } @@ -229,17 +234,18 @@ pub fn loadPipelineInputs( total_rows += rows; } - // Load stdin as `t` if piped (and no URL was provided) - if (parsed.has_stdin and parsed.url == null) { + // Load stdin as `stdin` if piped (and URLs/files exist), else `t` + if (parsed.has_stdin) { var stdin_buf: [4096]u8 = undefined; var stdin_reader = std.Io.File.reader(std.Io.File.stdin(), io, &stdin_buf); - const rows = loadInput(allocator, io, db, "t", parsed.input_format, &stdin_reader.interface, parsed, stderr_writer); + const stdin_table = if (parsed.urls.len > 0 or parsed.files.len > 0) "stdin" else "t"; + const rows = loadInput(allocator, io, db, stdin_table, parsed.input_format, &stdin_reader.interface, parsed, stderr_writer); total_rows += rows; } - // Warn when no rows were loaded — query referencing 't' will fail - if (total_rows == 0 and (parsed.has_stdin or parsed.files.len > 0 or parsed.url != null)) { - stderr_writer.print("warning: no rows loaded — table 't' not created; query may fail\n", .{}) catch {}; + // Warn when no rows were loaded — query referencing tables will fail + if (total_rows == 0 and (parsed.has_stdin or parsed.files.len > 0 or parsed.urls.len > 0)) { + stderr_writer.print("warning: no rows loaded — tables not created; query may fail\n", .{}) catch {}; } return total_rows; @@ -247,7 +253,10 @@ pub fn loadPipelineInputs( /// mainTableName(parsed) → []const u8 pub fn mainTableName(parsed: ParsedArgs) []const u8 { - if (parsed.url != null) return "t"; + if (parsed.urls.len > 0) { + if (parsed.urls[0].table_name) |name| return name; + return "url0"; + } if (parsed.files.len > 0) return parsed.files[0].table_name; return "t"; } @@ -449,7 +458,7 @@ pub fn main(init: std.process.Init.Minimal) void { }, .repl => |mut_parsed| { var parsed = mut_parsed; - parsed.has_stdin = if (parsed.url != null or parsed.no_stdin) false else !(std.Io.File.isTty(std.Io.File.stdin(), io.io()) catch false); + parsed.has_stdin = if (parsed.no_stdin) false else !(std.Io.File.isTty(std.Io.File.stdin(), io.io()) catch false); // Read query from file if -f/--file was used. if (parsed.query_file) |path| { const contents = std.Io.Dir.cwd().readFileAlloc(io.io(), path, args_arena.allocator(), .limited(10 * 1024 * 1024)) catch |err| { @@ -461,13 +470,19 @@ pub fn main(init: std.process.Init.Minimal) void { } parsed.query = trimmed; } - // `t` is reserved for stdin and URL input. - if (parsed.has_stdin or parsed.url != null) { + // `stdin` is reserved for stdin input when URLs/files exist. + if (parsed.has_stdin or parsed.urls.len > 0) { for (parsed.files) |f| { - if (std.mem.eql(u8, f.table_name, "t")) { + if (std.mem.eql(u8, f.table_name, "stdin")) { fatal("duplicate table name — file arguments must have unique basenames", stderr_writer, .usage, .{}); } } + for (parsed.urls) |url_input| { + const table_name = url_input.table_name orelse "url0"; + if (std.mem.eql(u8, table_name, "stdin")) { + fatal("duplicate table name — URL table name conflicts with stdin", stderr_writer, .usage, .{}); + } + } } // Resolve table mode: auto-detect from stdout TTY when not explicitly set. const stdout_is_tty = std.Io.File.isTty(std.Io.File.stdout(), io.io()) catch false; @@ -486,7 +501,7 @@ pub fn main(init: std.process.Init.Minimal) void { }, .parsed => |mut_parsed| { var parsed = mut_parsed; - parsed.has_stdin = if (parsed.url != null or parsed.no_stdin) false else !(std.Io.File.isTty(std.Io.File.stdin(), io.io()) catch false); + parsed.has_stdin = if (parsed.no_stdin) false else !(std.Io.File.isTty(std.Io.File.stdin(), io.io()) catch false); // Read query from file if -f/--file was used. // Arena-allocated to match the lifetime of parsed args. if (parsed.query_file) |path| { @@ -499,13 +514,19 @@ pub fn main(init: std.process.Init.Minimal) void { } parsed.query = trimmed; } - // `t` is reserved for stdin and URL input. - if (parsed.has_stdin or parsed.url != null) { + // `stdin` is reserved for stdin input when URLs/files exist. + if (parsed.has_stdin or parsed.urls.len > 0) { for (parsed.files) |f| { - if (std.mem.eql(u8, f.table_name, "t")) { + if (std.mem.eql(u8, f.table_name, "stdin")) { fatal("duplicate table name — file arguments must have unique basenames", stderr_writer, .usage, .{}); } } + for (parsed.urls) |url_input| { + const table_name = url_input.table_name orelse "url0"; + if (std.mem.eql(u8, table_name, "stdin")) { + fatal("duplicate table name — URL table name conflicts with stdin", stderr_writer, .usage, .{}); + } + } } // Resolve table mode: auto-detect from stdout TTY when not explicitly set. // Table output only applies when writing to stdout (not --output to a file) diff --git a/src/modes/schema.zig b/src/modes/schema.zig index be80dfe..5721942 100644 --- a/src/modes/schema.zig +++ b/src/modes/schema.zig @@ -22,6 +22,7 @@ pub fn runSchema( var parsed = args_mod.ParsedArgs{ .query = "", .files = args.files, + .urls = args.urls, .type_inference = args.type_inference, .delimiter = args.delimiter, .header = false, @@ -37,12 +38,11 @@ pub fn runSchema( .xml_row_input = null, .json_path = null, .disk = false, - .url = args.url, }; - parsed.has_stdin = if (parsed.url != null) false else !(std.Io.File.isTty(std.Io.File.stdin(), io) catch false); + parsed.has_stdin = if (parsed.urls.len > 0) false else !(std.Io.File.isTty(std.Io.File.stdin(), io) catch false); const total_rows = main_mod.loadPipelineInputs(allocator, io, db, parsed, stderr_writer); - if (total_rows == 0 and (parsed.has_stdin or parsed.files.len > 0 or parsed.url != null)) + if (total_rows == 0 and (parsed.has_stdin or parsed.files.len > 0 or parsed.urls.len > 0)) fatal("empty input", stderr_writer, .csv_error, .{}); // Print DDL for each table in creation order diff --git a/src/modes/stats.zig b/src/modes/stats.zig index 80ddded..52b89d8 100644 --- a/src/modes/stats.zig +++ b/src/modes/stats.zig @@ -23,6 +23,7 @@ pub fn runStats( var parsed = args_mod.ParsedArgs{ .query = "", .files = args.files, + .urls = args.urls, .type_inference = args.type_inference, .delimiter = args.delimiter, .header = false, @@ -38,12 +39,11 @@ pub fn runStats( .xml_row_input = null, .json_path = null, .disk = false, - .url = args.url, }; - parsed.has_stdin = if (parsed.url != null) false else !(std.Io.File.isTty(std.Io.File.stdin(), io) catch false); + parsed.has_stdin = if (parsed.urls.len > 0) false else !(std.Io.File.isTty(std.Io.File.stdin(), io) catch false); const total_rows = main_mod.loadPipelineInputs(allocator, io, db, parsed, stderr_writer); - if (total_rows == 0 and (parsed.has_stdin or parsed.files.len > 0 or parsed.url != null)) + if (total_rows == 0 and (parsed.has_stdin or parsed.files.len > 0 or parsed.urls.len > 0)) fatal("empty input", stderr_writer, .csv_error, .{}); // Print stats for each table in creation order