From 09fe7fb2539cf9ec37005c434b8940090faecd9a Mon Sep 17 00:00:00 2001 From: Nathan Flurry Date: Wed, 8 Jul 2026 01:26:40 -0700 Subject: [PATCH] fix(wget): drop unsupported wget package --- README.md | 1 - docs-internal/registry-parity-worklist.md | 40 ++- docs/wasmvm/supported-commands.md | 1 - packages/core/src/runtime-compat.ts | 2 - .../core/tests/helpers/registry-commands.ts | 4 +- packages/runtime-core/src/test-runtime.ts | 2 - packages/shell/package.json | 1 - pnpm-lock.yaml | 24 -- software/CONTRIBUTING.md | 2 +- software/README.md | 6 +- software/wget/agentos-package.json | 12 - software/wget/native/c/wget.c | 174 ------------- software/wget/package.json | 33 --- software/wget/src/index.ts | 5 - software/wget/test/wget.test.ts | 239 ------------------ software/wget/tsconfig.json | 8 - toolchain/Makefile | 2 +- toolchain/c/Makefile | 21 +- 18 files changed, 35 insertions(+), 542 deletions(-) delete mode 100644 software/wget/agentos-package.json delete mode 100644 software/wget/native/c/wget.c delete mode 100644 software/wget/package.json delete mode 100644 software/wget/src/index.ts delete mode 100644 software/wget/test/wget.test.ts delete mode 100644 software/wget/tsconfig.json diff --git a/README.md b/README.md index e9357918ca..2d1e80bfb8 100644 --- a/README.md +++ b/README.md @@ -165,7 +165,6 @@ Browse pre-built agents, tools, filesystems, and software packages at the [agent | `@agentos-software/tar` | tar | GNU tar archiver | rust | - | - | | `@agentos-software/tree` | tree | tree directory listing | rust | - | - | | `@agentos-software/unzip` | unzip | unzip archive extraction | c | - | - | -| `@agentos-software/wget` | wget | GNU wget HTTP client | c | - | - | | `@agentos-software/yq` | yq | yq YAML/JSON processor | rust | - | - | | `@agentos-software/zip` | zip | zip archive creation | c | - | - | diff --git a/docs-internal/registry-parity-worklist.md b/docs-internal/registry-parity-worklist.md index b44dbc4528..fc56da6932 100644 --- a/docs-internal/registry-parity-worklist.md +++ b/docs-internal/registry-parity-worklist.md @@ -88,7 +88,7 @@ actual backing: | Command | Status | What it actually is | Replace with | |---|---|---|---| | **curl** | TODO | our custom driver over a libcurl fork | real `curl` CLI (upstream `src/tool_*.c`) | -| **wget** | TODO | our 174-line `wget.c` | real GNU wget, or drop (curl covers it) | +| **wget** | NOT POSSIBLE (WASI) | our 174-line `wget.c` | dropped; real `curl` covers network downloads | | **http-get** | TODO | our 95-line `http_get.c` | drop, or a real tool | | **git** | TODO | our hand-rolled git from `sha1`+`flate2` | **real git** (upstream C), patched for WASI — **NOT gitoxide** | | **fd** | TODO | our `secureexec-fd` on raw `regex` (not sharkdp/fd) | real **fd** (sharkdp) | @@ -233,14 +233,28 @@ so a reader sees the whole board at a glance. ## P2 — Build / compile failures -### 8. wget — does not compile (`duplicate symbol: getpeername`) -- **Broken:** link error — wget's stub `getpeername` collides with the patched - sysroot's socket impl. -- **Objective:** wget compiles cleanly against the patched sysroot (drop the stub; - use the real sysroot socket symbols) **and** downloads files e2e like Linux wget. -- **Proof:** `make -C toolchain cmd/wget` succeeds; `software/wget/test/` passes - un-skipped (real download over the runtime network). -- **rev:** `fix(wget): drop conflicting getpeername stub; build against sysroot sockets` +### 8. wget — NOT POSSIBLE (WASI) +- **Broken:** the shipped command was a custom 174-line `wget.c` wrapper, not real + GNU Wget. The original compile failure was a duplicate socket stub, but fixing + that would still leave a custom reimplementation, violating Cross-cutting #0. +- **Tried:** GNU Wget 1.24.5 was configured for `wasm32-wasip1` with HTTP-only + features (`--without-ssl --without-zlib --without-libpsl --disable-iri` + and related auth/thread options disabled) against the patched sysroot. It + needed WASI patches for `inet_addr`, `O_BINARY`, process-group terminal checks, + `spawn.h`/`--use-askpass`, interactive `getpass`, gnulib `NSIG`, `F_DUPFD`, and + `flock`, then continued into more gnulib replacements requiring unsupported + `getrlimit`/`RLIMIT_NOFILE` and incompatible `getgroups` declarations. +- **Outcome:** dropped the `wget` package and runtime command instead of shipping + another custom shim. The real upstream `curl` CLI built in #6 is the supported + network downloader. +- **Proof:** `pnpm --dir packages/shell check-types` passes in + `2026-07-08T01-28-53-0700-item8-shell-check-types.txt`; + `pnpm --dir packages/runtime-core check-types` passes in + `2026-07-08T01-28-53-0700-item8-runtime-core-check-types.txt`; + `pnpm --dir packages/core check-types` passes in + `2026-07-08T01-28-54-0700-item8-core-check-types.txt`; lockfile validation + passes in `2026-07-08T01-29-03-0700-item8-pnpm-lockfile-check.txt`. +- **rev:** `mxkxpnpn` — `fix(wget): drop unsupported wget package` ### 9. codex-cli — not buildable in-checkout (needs external fork) - **Broken:** requires the external `codex-rs` fork (`CODEX_REPO` absent); tests @@ -268,14 +282,13 @@ so a reader sees the whole board at a glance. For each: replace `describe.skip` with `describeIf(binaryPresent)` **and** write real e2e tests that prove Linux-parity behavior — not smoke tests. -### 11. Disabled suites — git, duckdb, wget, codex +### 11. Disabled suites — git, duckdb, codex - **Broken:** tests exist but are `describe.skip`, so nothing runs even when the binary is present. - **Objective (per command, Linux parity):** - **git** — clone/commit/log/diff/branch against a real repo & remote. - **duckdb** — real analytical SQL, CSV read/write, file-backed DBs (the bar example: real duckdb, not a WASI-stripped `SELECT 1`). - - **wget** — real download (after #8). - **codex** — real run (after #9). - **Proof:** each un-skipped suite passes with real behavior. - **rev:** one per command, e.g. `test(duckdb): real analytical-SQL e2e; un-skip` @@ -306,6 +319,5 @@ real e2e tests that prove Linux-parity behavior — not smoke tests. ## Sequencing note P0 first — several P1/P3 items depend on it: curl (#6) needs sockets/HTTP; -sqlite3 file-DB tests (#11) need pwrite (#4); wget (#8) needs sockets. Fix the -runtime layer, then the commands that ride on it, then backfill coverage. One jj -rev per item throughout. +sqlite3 file-DB tests (#11) need pwrite (#4). Fix the runtime layer, then the +commands that ride on it, then backfill coverage. One jj rev per item throughout. diff --git a/docs/wasmvm/supported-commands.md b/docs/wasmvm/supported-commands.md index 3b40bc3ba2..eb036f31ed 100644 --- a/docs/wasmvm/supported-commands.md +++ b/docs/wasmvm/supported-commands.md @@ -213,7 +213,6 @@ | Command | just-bash | Status | Implementation | Target | |---------|-----------|--------|----------------|--------| | curl | yes | done | `curl_cli.c` (libcurl HTTP-only, C program via `host_net`) | — | -| wget | yes | done | `wget.c` (libcurl-based, C program via `host_net`) | — | ## Version Control diff --git a/packages/core/src/runtime-compat.ts b/packages/core/src/runtime-compat.ts index 7dd2dd45f1..ed3ec8341d 100644 --- a/packages/core/src/runtime-compat.ts +++ b/packages/core/src/runtime-compat.ts @@ -1460,7 +1460,6 @@ export const WASMVM_COMMANDS = Object.freeze([ "unzip", "sqlite3", "curl", - "wget", "git", "git-remote-http", "git-remote-https", @@ -1629,7 +1628,6 @@ export const DEFAULT_FIRST_PARTY_TIERS: Readonly< tac: "read-only", tsort: "read-only", curl: "full", - wget: "full", sqlite3: "read-write", }); diff --git a/packages/core/tests/helpers/registry-commands.ts b/packages/core/tests/helpers/registry-commands.ts index 1d2dfdeb67..65cdcb9ed6 100644 --- a/packages/core/tests/helpers/registry-commands.ts +++ b/packages/core/tests/helpers/registry-commands.ts @@ -13,7 +13,7 @@ * command set still fails loudly. * * The only sanctioned exception is the C-sysroot package set (duckdb, - * http-get, sqlite3, wget, zip, unzip): those need the patched wasi C sysroot + * http-get, sqlite3, zip, unzip): those need the patched wasi C sysroot * that most checkouts don't have, so `cSysrootPackageSkipReason` reports a * skip reason instead of throwing. Everything else is load-or-throw. */ @@ -161,7 +161,7 @@ export function requireBuilt( /** * Skip reason for the C-sysroot package set ONLY (duckdb, http-get, sqlite3, - * wget, zip, unzip). These need the patched wasi C sysroot + * zip, unzip). These need the patched wasi C sysroot * (`make -C toolchain/c`), which most checkouts don't build — a missing * artifact is an environment limitation, not a forgotten build, so suites may * skip with this reason instead of throwing. diff --git a/packages/runtime-core/src/test-runtime.ts b/packages/runtime-core/src/test-runtime.ts index 16ae3e40cb..8ca761958c 100644 --- a/packages/runtime-core/src/test-runtime.ts +++ b/packages/runtime-core/src/test-runtime.ts @@ -1587,7 +1587,6 @@ export const WASMVM_COMMANDS = Object.freeze([ "unzip", "sqlite3", "curl", - "wget", "git", "git-remote-http", "git-remote-https", @@ -1756,7 +1755,6 @@ export const DEFAULT_FIRST_PARTY_TIERS: Readonly< tac: "read-only", tsort: "read-only", curl: "full", - wget: "full", sqlite3: "read-write", }); diff --git a/packages/shell/package.json b/packages/shell/package.json index 254004e6b4..4c9e6765b5 100644 --- a/packages/shell/package.json +++ b/packages/shell/package.json @@ -35,7 +35,6 @@ "@agentos-software/tree": "workspace:*", "@agentos-software/unzip": "workspace:*", "@agentos-software/vim": "workspace:*", - "@agentos-software/wget": "workspace:*", "@agentos-software/yq": "workspace:*", "@agentos-software/zip": "workspace:*", "@rivet-dev/agentos": "workspace:*", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 38e3dc4dc6..acea4b1ad2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2877,9 +2877,6 @@ importers: '@agentos-software/vim': specifier: workspace:* version: link:../../software/vim - '@agentos-software/wget': - specifier: workspace:* - version: link:../../software/wget '@agentos-software/yq': specifier: workspace:* version: link:../../software/yq @@ -3738,27 +3735,6 @@ importers: specifier: ^2.1.9 version: 2.1.9(@types/node@22.19.15) - software/wget: - devDependencies: - '@agentos-software/manifest': - specifier: workspace:* - version: link:../../packages/manifest - '@agentos/test-harness': - specifier: workspace:* - version: link:../../test-harness - '@rivet-dev/agentos-toolchain': - specifier: workspace:* - version: link:../../packages/agentos-toolchain - '@types/node': - specifier: ^22.10.2 - version: 22.19.15 - typescript: - specifier: ^5.9.2 - version: 5.9.3 - vitest: - specifier: ^2.1.9 - version: 2.1.9(@types/node@22.19.15) - software/yq: devDependencies: '@agentos-software/manifest': diff --git a/software/CONTRIBUTING.md b/software/CONTRIBUTING.md index 2ab561cf43..ac86bfa6c5 100644 --- a/software/CONTRIBUTING.md +++ b/software/CONTRIBUTING.md @@ -34,7 +34,7 @@ From the repo root: ```bash just toolchain-build # compile the fast native wasm command gate -just toolchain-cmd # build one command (required for git, duckdb, vim, wget, codex) +just toolchain-cmd # build one command (required for git, duckdb, vim, codex) just software-build # stage bin/ + assemble dist/package/ pnpm --filter './software/*' test ``` diff --git a/software/README.md b/software/README.md index 7f15771bb8..de31afdf9c 100644 --- a/software/README.md +++ b/software/README.md @@ -68,7 +68,7 @@ per-binary entry point; it dispatches to whichever toolchain owns the command: | kind | commands | what it runs | |---|---|---| | Rust | any `software//native/crates/cmd-` command crate (sh, ls, rg, …) | `cargo build -p cmd-` (build-std) + `wasm-opt` | -| C | `zip unzip envsubst sqlite3 curl wget duckdb` | `make -C c sysroot build/` + per-command install | +| C | `zip unzip envsubst sqlite3 curl duckdb` | `make -C c sysroot build/` + per-command install | | codex | `codex`, `codex-exec` | the codex fork build (needs the fork checkout) | | C | `vim` (pinned upstream clone + bridge in `c/vim/`) | `make -C c sysroot build/vim` + install | @@ -77,7 +77,7 @@ The default native build (`toolchain`) compiles the fast command gate to `wasm-opt -O3`, and drops the binaries in `toolchain/target/wasm32-wasip1/release/commands/`. The bulk gate intentionally excludes slow/heavy or non-default commands: `git`, `duckdb`, -`vim`, `wget`, and the external `codex`/`codex-exec` fork build. Build those explicitly with +`vim` and the external `codex`/`codex-exec` fork build. Build those explicitly with `just toolchain-cmd ` when working on them. Package builds then run `agentos-toolchain stage` (with `--if-missing skip`, so a checkout without the native build still assembles valid empty placeholders) followed by `tsc` and @@ -90,7 +90,7 @@ packages are pnpm workspace members, so tests and examples resolve them via Exceptions: - `software/codex/wasm/` is the install target of the codex fork's build (`make -C toolchain codex`); `software/codex-cli` stages from it. -- C-built commands (sqlite3, zip, unzip, curl, wget, duckdb) need the patched +- C-built commands (sqlite3, zip, unzip, curl, duckdb) need the patched sysroot; `just toolchain-cmd ` builds it on demand. Without it those packages stay empty placeholders. - `vim` builds from source: `just toolchain-cmd vim` clones the pinned diff --git a/software/wget/agentos-package.json b/software/wget/agentos-package.json deleted file mode 100644 index 245dc4dd4f..0000000000 --- a/software/wget/agentos-package.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "commands": [ - "wget" - ], - "registry": { - "title": "wget", - "description": "GNU wget file downloader.", - "priority": 55, - "image": "/images/registry/wget.svg", - "category": "networking" - } -} diff --git a/software/wget/native/c/wget.c b/software/wget/native/c/wget.c deleted file mode 100644 index e0e0ca02fd..0000000000 --- a/software/wget/native/c/wget.c +++ /dev/null @@ -1,174 +0,0 @@ -/* - * wget.c - minimal wget implementation built on libcurl - * - * Supports common wget options for HTTP/HTTPS downloads: - * URL Download file to current directory (basename from URL) - * -O FILE Write output to specific file ("-" for stdout) - * -q Quiet mode (suppress progress/messages) - * -L Follow redirects (enabled by default, like real wget) - * --no-check-certificate Skip TLS certificate verification - */ - -#include -#include -#include -#include - -/* Write callback: write to FILE* */ -static size_t write_callback(char *ptr, size_t size, size_t nmemb, - void *userdata) { - FILE *out = (FILE *)userdata; - return fwrite(ptr, size, nmemb, out); -} - -/* Extract filename from URL path, fallback to "index.html" */ -static const char *basename_from_url(const char *url) { - /* Skip scheme */ - const char *p = strstr(url, "://"); - if (p) p += 3; else p = url; - - /* Find last '/' in path (before query string) */ - const char *last_slash = NULL; - const char *q = p; - while (*q && *q != '?' && *q != '#') { - if (*q == '/') last_slash = q; - q++; - } - - if (last_slash && last_slash[1] && last_slash[1] != '?' && last_slash[1] != '#') { - /* Extract filename between last slash and query/end */ - const char *start = last_slash + 1; - size_t len = 0; - while (start[len] && start[len] != '?' && start[len] != '#') len++; - if (len > 0) { - static char buf[256]; - if (len >= sizeof(buf)) len = sizeof(buf) - 1; - memcpy(buf, start, len); - buf[len] = '\0'; - return buf; - } - } - - return "index.html"; -} - -int main(int argc, char *argv[]) { - const char *url = NULL; - const char *output_file = NULL; - int quiet = 0; - int no_check_cert = 0; - int output_to_stdout = 0; - - /* Parse arguments */ - for (int i = 1; i < argc; i++) { - if (strcmp(argv[i], "-O") == 0 && i + 1 < argc) { - output_file = argv[++i]; - if (strcmp(output_file, "-") == 0) { - output_to_stdout = 1; - output_file = NULL; - } - } else if (strcmp(argv[i], "-q") == 0) { - quiet = 1; - } else if (strcmp(argv[i], "-L") == 0) { - /* Follow redirects — already default, accept silently */ - } else if (strcmp(argv[i], "--no-check-certificate") == 0) { - no_check_cert = 1; - } else if (argv[i][0] != '-') { - url = argv[i]; - } else { - /* Unknown option — skip silently for forward compat */ - } - } - - if (!url) { - fprintf(stderr, "wget: missing URL\nUsage: wget [OPTION]... [URL]...\n"); - return 1; - } - - CURLcode res; - curl_global_init(CURL_GLOBAL_DEFAULT); - - CURL *curl = curl_easy_init(); - if (!curl) { - fprintf(stderr, "wget: failed to initialize\n"); - curl_global_cleanup(); - return 1; - } - - /* Determine output destination */ - FILE *out = NULL; - const char *dest_name = NULL; - - if (output_to_stdout) { - out = stdout; - } else { - dest_name = output_file ? output_file : basename_from_url(url); - out = fopen(dest_name, "wb"); - if (!out) { - fprintf(stderr, "wget: cannot open '%s' for writing\n", dest_name); - curl_easy_cleanup(curl); - curl_global_cleanup(); - return 1; - } - } - - curl_easy_setopt(curl, CURLOPT_URL, url); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, out); - - /* wget follows redirects by default */ - curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 20L); - - /* Suppress progress meter */ - curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L); - - /* TLS: skip certificate verification */ - if (no_check_cert) { - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L); - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L); - } - - /* Perform request */ - res = curl_easy_perform(curl); - - int exit_code = 0; - - if (res != CURLE_OK) { - if (!quiet) { - fprintf(stderr, "wget: failed: %s\n", curl_easy_strerror(res)); - } - exit_code = 1; - /* Remove partial download on failure (unless stdout) */ - if (!output_to_stdout && dest_name) { - fclose(out); - out = NULL; - remove(dest_name); - } - } else { - /* Check HTTP response code */ - long http_code = 0; - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); - if (http_code >= 400) { - if (!quiet) { - fprintf(stderr, "wget: server returned HTTP %ld\n", http_code); - } - exit_code = 8; /* wget uses exit code 8 for server errors */ - /* Remove error response file (unless stdout) */ - if (!output_to_stdout && dest_name) { - fclose(out); - out = NULL; - remove(dest_name); - } - } - } - - /* Cleanup */ - curl_easy_cleanup(curl); - if (out && out != stdout) { - fclose(out); - } - curl_global_cleanup(); - - return exit_code; -} diff --git a/software/wget/package.json b/software/wget/package.json deleted file mode 100644 index 47433367e9..0000000000 --- a/software/wget/package.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "name": "@agentos-software/wget", - "version": "0.3.3", - "type": "module", - "license": "Apache-2.0", - "description": "GNU wget HTTP client for secure-exec VMs", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "files": [ - "dist", - "!dist/package", - "!dist/package.tar" - ], - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } - }, - "scripts": { - "build": "agentos-toolchain stage --commands-dir ../../toolchain/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", - "check-types": "tsc --noEmit", - "test": "vitest run test/ --passWithNoTests" - }, - "devDependencies": { - "@agentos-software/manifest": "workspace:*", - "@rivet-dev/agentos-toolchain": "workspace:*", - "@types/node": "^22.10.2", - "typescript": "^5.9.2", - "@agentos/test-harness": "workspace:*", - "vitest": "^2.1.9" - } -} diff --git a/software/wget/src/index.ts b/software/wget/src/index.ts deleted file mode 100644 index dba100e2e1..0000000000 --- a/software/wget/src/index.ts +++ /dev/null @@ -1,5 +0,0 @@ -import type { SoftwarePackageRef } from "@agentos-software/manifest"; - -const packagePath = new URL("./package.aospkg", import.meta.url).pathname; - -export default { packagePath } satisfies SoftwarePackageRef; diff --git a/software/wget/test/wget.test.ts b/software/wget/test/wget.test.ts deleted file mode 100644 index 0b4e66263c..0000000000 --- a/software/wget/test/wget.test.ts +++ /dev/null @@ -1,239 +0,0 @@ -/** - * Integration tests for wget C command (libcurl-based). - * - * Verifies HTTP download operations via kernel.exec() with real WASM binaries: - * - Basic GET download to file - * - Download to specified file (-O) - * - Quiet mode (-q) - * - Error handling for 404 URLs - * - Follow redirects (default behavior) - * - * Tests start a local HTTP server in beforeAll and make wget requests against it. - */ - -import { describe, it, expect, afterEach, beforeAll, afterAll } from 'vitest'; -import { createWasmVmRuntime } from '@agentos/test-harness'; -import { C_BUILD_DIR, COMMANDS_DIR, createKernel, describeIf, hasCWasmBinaries } from '@agentos/test-harness'; -import type { Kernel } from '@agentos/test-harness'; -import { createServer, type Server, type IncomingMessage, type ServerResponse } from 'node:http'; - -// Minimal in-memory VFS for kernel tests -class SimpleVFS { - private files = new Map(); - private dirs = new Set(['/']); - - async readFile(path: string): Promise { - const data = this.files.get(path); - if (!data) throw new Error(`ENOENT: ${path}`); - return data; - } - async readTextFile(path: string): Promise { - return new TextDecoder().decode(await this.readFile(path)); - } - async readDir(path: string): Promise { - const prefix = path === '/' ? '/' : path + '/'; - const entries: string[] = []; - for (const p of [...this.files.keys(), ...this.dirs]) { - if (p !== path && p.startsWith(prefix)) { - const rest = p.slice(prefix.length); - if (!rest.includes('/')) entries.push(rest); - } - } - return entries; - } - async readDirWithTypes(path: string) { - return (await this.readDir(path)).map(name => ({ - name, - isDirectory: this.dirs.has(path === '/' ? `/${name}` : `${path}/${name}`), - })); - } - async writeFile(path: string, content: string | Uint8Array): Promise { - const data = typeof content === 'string' ? new TextEncoder().encode(content) : content; - this.files.set(path, new Uint8Array(data)); - const parts = path.split('/').filter(Boolean); - for (let i = 1; i < parts.length; i++) { - this.dirs.add('/' + parts.slice(0, i).join('/')); - } - } - async createDir(path: string) { this.dirs.add(path); } - async mkdir(path: string, _options?: { recursive?: boolean }) { - this.dirs.add(path); - const parts = path.split('/').filter(Boolean); - for (let i = 1; i < parts.length; i++) { - this.dirs.add('/' + parts.slice(0, i).join('/')); - } - } - async exists(path: string): Promise { - return this.files.has(path) || this.dirs.has(path); - } - async stat(path: string) { - const isDir = this.dirs.has(path); - const data = this.files.get(path); - if (!isDir && !data) throw new Error(`ENOENT: ${path}`); - return { - mode: isDir ? 0o40755 : 0o100644, - size: data?.length ?? 0, - isDirectory: isDir, - isSymbolicLink: false, - atimeMs: Date.now(), - mtimeMs: Date.now(), - ctimeMs: Date.now(), - birthtimeMs: Date.now(), - ino: 0, - nlink: 1, - uid: 1000, - gid: 1000, - }; - } - async chmod(_path: string, _mode: number) {} - async lstat(path: string) { return this.stat(path); } - async removeFile(path: string) { this.files.delete(path); } - async removeDir(path: string) { this.dirs.delete(path); } - async rename(oldPath: string, newPath: string) { - const data = this.files.get(oldPath); - if (data) { - this.files.set(newPath, data); - this.files.delete(oldPath); - } - } - async pread(path: string, buffer: Uint8Array, offset: number, length: number, position: number): Promise { - const data = this.files.get(path); - if (!data) throw new Error(`ENOENT: ${path}`); - const available = Math.min(length, data.length - position); - if (available <= 0) return 0; - buffer.set(data.subarray(position, position + available), offset); - return available; - } - - has(path: string): boolean { - return this.files.has(path); - } - getContent(path: string): string | undefined { - const data = this.files.get(path); - return data ? new TextDecoder().decode(data) : undefined; - } - getRawContent(path: string): Uint8Array | undefined { - return this.files.get(path); - } -} - -// TODO(P6): requires wget WASM artifact, intentionally excluded from the fast software-build gate. -describeIf(hasCWasmBinaries('wget'), 'wget command', () => { - let kernel: Kernel; - let server: Server; - let port: number; - - beforeAll(async () => { - server = createServer((req: IncomingMessage, res: ServerResponse) => { - const url = req.url ?? '/'; - - if (url === '/file.txt') { - res.writeHead(200, { 'Content-Type': 'text/plain' }); - res.end('downloaded content'); - return; - } - - if (url === '/data.json') { - res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ status: 'ok' })); - return; - } - - if (url === '/redirect') { - const addr = server.address() as import('node:net').AddressInfo; - res.writeHead(302, { 'Location': `http://127.0.0.1:${addr.port}/redirected` }); - res.end(); - return; - } - - if (url === '/redirected') { - res.writeHead(200, { 'Content-Type': 'text/plain' }); - res.end('arrived after redirect'); - return; - } - - if (url === '/binary') { - const buf = Buffer.alloc(1024); - for (let i = 0; i < buf.length; i++) buf[i] = i & 0xff; - res.writeHead(200, { - 'Content-Type': 'application/octet-stream', - 'Content-Length': String(buf.length), - }); - res.end(buf); - return; - } - - res.writeHead(404, { 'Content-Type': 'text/plain' }); - res.end('not found'); - }); - - await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); - port = (server.address() as import('node:net').AddressInfo).port; - }); - - afterAll(async () => { - await new Promise((resolve) => server.close(() => resolve())); - }); - - afterEach(async () => { - await kernel?.dispose(); - }); - - it('downloads file to VFS using URL basename', async () => { - const vfs = new SimpleVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); - - await kernel.exec(`wget http://127.0.0.1:${port}/file.txt`); - - const content = vfs.getContent('/file.txt'); - expect(content).toBe('downloaded content'); - }); - - it('-O saves to specified filename', async () => { - const vfs = new SimpleVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); - - await kernel.exec(`wget -O /output.txt http://127.0.0.1:${port}/data.json`); - - const content = vfs.getContent('/output.txt'); - expect(content).toBeDefined(); - expect(content).toContain('"status":"ok"'); - }); - - it('-q suppresses progress output', async () => { - const vfs = new SimpleVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); - - const result = await kernel.exec(`wget -q -O /output.txt http://127.0.0.1:${port}/file.txt`); - - // Quiet mode should produce no stderr - expect(result.stderr).toBe(''); - // File should still be downloaded - expect(vfs.getContent('/output.txt')).toBe('downloaded content'); - }); - - it('returns non-zero exit code for 404 URL', async () => { - const vfs = new SimpleVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); - - const result = await kernel.exec(`wget http://127.0.0.1:${port}/nonexistent`); - - // Should report error on stderr - expect(result.stderr).toMatch(/wget|404|error|server/i); - }); - - it('follows redirects by default', async () => { - const vfs = new SimpleVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); - - await kernel.exec(`wget -O /output.txt http://127.0.0.1:${port}/redirect`); - - const content = vfs.getContent('/output.txt'); - expect(content).toBe('arrived after redirect'); - }); -}); diff --git a/software/wget/tsconfig.json b/software/wget/tsconfig.json deleted file mode 100644 index 03ce790ab7..0000000000 --- a/software/wget/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*"] -} diff --git a/toolchain/Makefile b/toolchain/Makefile index e14e52b982..da4ce02f8e 100644 --- a/toolchain/Makefile +++ b/toolchain/Makefile @@ -210,7 +210,7 @@ wasm: vendor patch-vendor patch-std wasm-opt-check # codex the codex fork build (codex, codex-exec) # `just toolchain-cmd ` calls this. `make wasm` builds the fast # Rust set; `make -C c programs install` builds/installs the fast C set. -C_COMMANDS := zip unzip envsubst sqlite3 curl wget duckdb vim http_get +C_COMMANDS := zip unzip envsubst sqlite3 curl duckdb vim http_get .PHONY: cmd/% cmd/%: diff --git a/toolchain/c/Makefile b/toolchain/c/Makefile index 8c431c0bb8..27c949cd0a 100644 --- a/toolchain/c/Makefile +++ b/toolchain/c/Makefile @@ -71,11 +71,11 @@ COMMANDS_DIR ?= ../target/wasm32-wasip1/release/commands COMMANDS := zip unzip envsubst sqlite3 curl http_get # Programs requiring patched sysroot (Tier 2+ custom host imports) -PATCHED_PROGRAMS := http_get_test isatty_test getpid_test getppid_test getppid_verify userinfo pipe_test dup_test spawn_child spawn_exit_code pipeline kill_child waitpid_return waitpid_edge syscall_coverage getpwuid_test signal_tests sigaction_self sigaction_behavior delayed_tcp_echo delayed_kill pipe_edge tcp_accept_spawn tcp_echo tcp_server http_server udp_echo unix_socket signal_handler http_get dns_lookup sqlite3_cli curl wget fs_probe +PATCHED_PROGRAMS := http_get_test isatty_test getpid_test getppid_test getppid_verify userinfo pipe_test dup_test spawn_child spawn_exit_code pipeline kill_child waitpid_return waitpid_edge syscall_coverage getpwuid_test signal_tests sigaction_self sigaction_behavior delayed_tcp_echo delayed_kill pipe_edge tcp_accept_spawn tcp_echo tcp_server http_server udp_echo unix_socket signal_handler http_get dns_lookup sqlite3_cli curl fs_probe # Discover all package command and test-program C source files. ALL_SOURCES := $(foreach dir,$(C_SOURCE_DIRS),$(wildcard $(dir)/*.c)) -SKIPPED_BULK_PROGRAMS := wget +SKIPPED_BULK_PROGRAMS := # Exclude patched-sysroot programs when only vanilla sysroot is available ifeq ($(wildcard $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a),) @@ -560,23 +560,6 @@ $(BUILD_DIR)/curl: scripts/build-curl-upstream.sh $(CURL_UPSTREAM_OVERLAY_FILES) --ranlib "$(abspath $(WASI_SDK_DIR)/bin/llvm-ranlib)" \ --output "$(abspath $@)" -# wget: minimal wget built on libcurl -$(BUILD_DIR)/wget: $(call c_source,wget) $(CURL_SRCS) $(WASI_SDK_DIR)/bin/clang - @mkdir -p $(BUILD_DIR) - $(CC) $(WASM_CFLAGS) $(CURL_LIB_DEFS) $(CURL_INCLUDES) \ - -lwasi-emulated-signal \ - -o $@.wasm $(call c_source,wget) $(CURL_SRCS) - @if [ "$(HAS_WASM_OPT)" = "1" ]; then \ - wasm-opt -O3 --strip-debug --all-features $@.wasm -o $@; \ - else \ - cp $@.wasm $@; \ - fi - @rm -f $@.wasm - -$(NATIVE_DIR)/wget: $(call c_source,wget) - @mkdir -p $(NATIVE_DIR) - $(NATIVE_CC) $(NATIVE_CFLAGS) -o $@ $(call c_source,wget) -lcurl - # duckdb: upstream DuckDB CLI built from source with our patched WASI/POSIX sysroot $(BUILD_DIR)/duckdb: libs/duckdb/CMakeLists.txt $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang $(WASI_SDK_DIR)/bin/clang++ cmake/FindThreads.cmake scripts/build-duckdb.sh include/fcntl.h include/ifaddrs.h include/net/if.h include/sys/ioctl.h @mkdir -p $(BUILD_DIR)