From b6512f47a459bfeddd73f1bc676d7b2522ad1d5d Mon Sep 17 00:00:00 2001 From: Trustable User Date: Mon, 13 Jul 2026 11:24:44 +0200 Subject: [PATCH] Add automatic runtime builds to IDE deploy --- cloud/k3s/docopts.md | 4 +- cloud/k3s/opsfile.yml | 12 + cloud/k3s/registry.test.ts | 33 +++ cloud/k3s/registry.ts | 97 ++++++++ ide/deploy/Builder.md | 42 ++++ ide/deploy/builder.js | 209 ++++++++++++++++++ ide/deploy/builder.test.js | 178 +++++++++++++++ ide/deploy/deploy.js | 28 ++- ide/deploy/index.js | 17 +- ide/deploy/scan.js | 7 +- ide/deploy/watch.js | 28 ++- ide/opsfile.yml | 6 +- setup/kubernetes/crds/whisk-crd.yaml | 5 +- .../roles/nuvolaris-wsku-roles.yaml | 5 + setup/kubernetes/whisk.yaml | 1 + setup/nuvolaris/system-api/api-template.yaml | 11 + setup/nuvolaris/system-api/builders.json | 7 + setup/nuvolaris/system-api/builders.test.ts | 43 ++++ setup/nuvolaris/system-api/builders.ts | 74 +++++++ setup/nuvolaris/system-api/opsfile.yml | 20 +- 20 files changed, 809 insertions(+), 18 deletions(-) create mode 100644 cloud/k3s/registry.test.ts create mode 100644 cloud/k3s/registry.ts create mode 100644 ide/deploy/Builder.md create mode 100644 ide/deploy/builder.js create mode 100644 ide/deploy/builder.test.js create mode 100644 setup/nuvolaris/system-api/builders.json create mode 100644 setup/nuvolaris/system-api/builders.test.ts create mode 100644 setup/nuvolaris/system-api/builders.ts diff --git a/cloud/k3s/docopts.md b/cloud/k3s/docopts.md index 49a4bd10..d92355b1 100644 --- a/cloud/k3s/docopts.md +++ b/cloud/k3s/docopts.md @@ -28,6 +28,7 @@ Usage: k3s delete [] k3s info k3s kubeconfig [] + k3s registry [] k3s status ``` @@ -38,5 +39,6 @@ Usage: delete uninstall k3s with ssh in using with sudo info info on the server kubeconfig recover the kubeconfig from a k3s server with user + registry configure the private registry endpoint used by K3s containerd status status of the server -``` \ No newline at end of file +``` diff --git a/cloud/k3s/opsfile.yml b/cloud/k3s/opsfile.yml index efcaa899..e837a6f4 100644 --- a/cloud/k3s/opsfile.yml +++ b/cloud/k3s/opsfile.yml @@ -65,6 +65,7 @@ tasks: k3sup install --k3s-version="{{.K3S_VERSION}}" --host="{{._server_}}" --user="{{.INSTALL_USER}}" --local-path=$OPS_TMP/kubeconfig + - bun registry.ts "{{._server_}}" "{{.INSTALL_USER}}" cert-manager: @@ -78,10 +79,21 @@ tasks: desc: create a k3s with ssh in using with sudo cmds: - config OPERATOR_CONFIG_KUBE=k3s + - config REGISTRY_CONFIG_PULL_HOSTNAME="{{._server_}}:32000" - task: install - task: cert-manager - cp "$OPS_TMP/kubeconfig" "$OPS_TMP/k3s-{{._server_}}.kubeconfig" + registry: + silent: true + desc: configure the OpenServerless registry endpoint on a K3s node + vars: + INSTALL_USER: '{{._user_ | default "root"}}' + cmds: + - test -n "{{._server_}}" || die "required ip or hostname" + - bun registry.ts "{{._server_}}" "{{.INSTALL_USER}}" + - config REGISTRY_CONFIG_PULL_HOSTNAME="{{._server_}}:32000" + delete: silent: true desc: uninstall with ssh in using with sudo diff --git a/cloud/k3s/registry.test.ts b/cloud/k3s/registry.test.ts new file mode 100644 index 00000000..6ed61f6f --- /dev/null +++ b/cloud/k3s/registry.test.ts @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import {expect, test} from "bun:test"; +import {renderRegistryConfig} from "./registry"; + +test("renders an explicit HTTP mirror for the K3s node registry", () => { + expect(renderRegistryConfig("192.0.2.10")).toBe( + "# Managed by Apache OpenServerless\n" + + "mirrors:\n" + + " \"192.0.2.10:32000\":\n" + + " endpoint:\n" + + " - \"http://192.0.2.10:32000\"\n", + ); +}); + +test("rejects shell syntax in a K3s host", () => { + expect(() => renderRegistryConfig("host;reboot")).toThrow("invalid K3s server"); +}); diff --git a/cloud/k3s/registry.ts b/cloud/k3s/registry.ts new file mode 100644 index 00000000..581f4512 --- /dev/null +++ b/cloud/k3s/registry.ts @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const HOST = /^[A-Za-z0-9][A-Za-z0-9.-]*$/; +const USER = /^[A-Za-z0-9_][A-Za-z0-9_.-]*$/; + +export function renderRegistryConfig(server: string): string { + if (!HOST.test(server)) throw new Error(`invalid K3s server: ${server}`); + return `# Managed by Apache OpenServerless\nmirrors:\n "${server}:32000":\n endpoint:\n - "http://${server}:32000"\n`; +} + +async function capture(args: string[]): Promise<{code: number; stdout: string; stderr: string}> { + const process = Bun.spawn(args, {stdout: "pipe", stderr: "pipe"}); + const [stdout, stderr, code] = await Promise.all([ + new Response(process.stdout).text(), + new Response(process.stderr).text(), + process.exited, + ]); + return {code, stdout, stderr}; +} + +async function writeRemote(args: string[], content: string): Promise { + const process = Bun.spawn(args, {stdin: "pipe", stdout: "inherit", stderr: "inherit"}); + process.stdin.write(content); + process.stdin.end(); + const code = await process.exited; + if (code !== 0) throw new Error(`command failed (${code}): ${args.join(" ")}`); +} + +async function configure(server: string, user: string): Promise { + if (!USER.test(user)) throw new Error(`invalid SSH user: ${user}`); + const destination = `${user}@${server}`; + const ssh = ["ssh", "-oStrictHostKeyChecking=no", destination]; + const expected = renderRegistryConfig(server); + const current = await capture([...ssh, "sudo", "cat", "/etc/rancher/k3s/registries.yaml"]); + if (current.code === 0 && current.stdout === expected) { + console.log(`K3s registry already configured for ${server}:32000`); + return; + } + if (current.code === 0 && current.stdout.trim()) { + throw new Error( + "/etc/rancher/k3s/registries.yaml already contains unmanaged configuration; refusing to overwrite it", + ); + } + + const exists = await capture([ + ...ssh, "sudo", "test", "-e", "/etc/rancher/k3s/registries.yaml", + ]); + if (exists.code === 0) { + throw new Error("cannot read the existing /etc/rancher/k3s/registries.yaml"); + } + + const directory = await capture([...ssh, "sudo", "mkdir", "-p", "/etc/rancher/k3s"]); + if (directory.code !== 0) throw new Error(directory.stderr || "cannot create K3s config directory"); + await writeRemote( + [...ssh, "sudo", "tee", "/etc/rancher/k3s/registries.yaml"], + expected, + ); + const restart = await capture([...ssh, "sudo", "systemctl", "restart", "k3s"]); + if (restart.code !== 0) throw new Error(restart.stderr || "cannot restart K3s"); + + for (let attempt = 0; attempt < 60; attempt += 1) { + const active = await capture([...ssh, "sudo", "systemctl", "is-active", "k3s"]); + if (active.code === 0 && active.stdout.trim() === "active") { + console.log(`K3s registry configured for ${server}:32000`); + return; + } + await Bun.sleep(1000); + } + throw new Error("K3s did not become active after registry configuration"); +} + +if (import.meta.main) { + const args = Bun.argv.slice(2); + if (args[0] === "--render") { + console.log(renderRegistryConfig(args[1])); + } else { + const server = args[0]; + const user = args[1] || "root"; + if (!server) throw new Error("K3s server is required"); + await configure(server, user); + } +} diff --git a/ide/deploy/Builder.md b/ide/deploy/Builder.md new file mode 100644 index 00000000..a9843dea --- /dev/null +++ b/ide/deploy/Builder.md @@ -0,0 +1,42 @@ +# Automatic runtime image builder + +`ops ide deploy` can ensure custom runtime images before updating actions. The +build runs as a Kubernetes BuildKit Job through the OpenServerless system API; +the developer machine does not need Docker. + +Projects opt in with generic runtime profiles in their root `package.json`: + +```json +{ + "openserverless": { + "runtimeProfiles": { + "python-custom": { + "builder": "python:3.13", + "requirements": "runtime/python-custom.txt", + "actions": ["samples/python-custom"] + } + } + } +} +``` + +Each action can belong to at most one profile. Requirements paths are relative +to the project root and cannot escape it. The builder id is resolved by the +server from an allowlisted catalog; projects cannot select an arbitrary source +image, target repository, namespace, or registry. + +The system API derives a content digest from the builder contract, source +image, and requirements. It either returns an existing image or starts a Job. +The deploy task waits for a terminal state and only then adds `--docker` to the +action update. A failed or timed-out build makes `ops ide deploy` fail before +the action is changed. + +An explicit `--docker` action argument takes precedence over the generated +profile image. `--dry-run` validates and reports profiles without contacting +the builder API. + +Registry endpoints are supplied by cluster configuration: + +- the push endpoint is reachable from the BuildKit pod; +- the pull endpoint is reachable from the node container runtime; +- neither endpoint is hardcoded in the deploy task. diff --git a/ide/deploy/builder.js b/ide/deploy/builder.js new file mode 100644 index 00000000..fcb343bb --- /dev/null +++ b/ide/deploy/builder.js @@ -0,0 +1,209 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import fs from "fs/promises"; +import path from "path"; + +const PROFILE_NAME = /^[a-z0-9][a-z0-9._-]{0,62}$/; +const BUILDER_ID = /^[a-z0-9][a-z0-9._:-]{0,62}$/; +const ACTION_NAME = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/; +const IMAGE_REFERENCE = /^[A-Za-z0-9][A-Za-z0-9._:/@-]{0,511}$/; + +function projectDirectory(options = {}) { + return options.projectDir || process.env.OPS_PWD || process.cwd(); +} + +export async function loadRuntimeProfiles(options = {}) { + const packagePath = path.join(projectDirectory(options), "package.json"); + let document; + try { + document = JSON.parse(await fs.readFile(packagePath, "utf8")); + } catch (error) { + if (error.code === "ENOENT") return {}; + throw new Error(`cannot read ${packagePath}: ${error.message}`); + } + + const profiles = document.openserverless?.runtimeProfiles || {}; + if (typeof profiles !== "object" || Array.isArray(profiles)) { + throw new Error("openserverless.runtimeProfiles must be an object"); + } + + const result = {}; + const assignedActions = new Set(); + for (const [name, profile] of Object.entries(profiles)) { + if (!PROFILE_NAME.test(name)) throw new Error(`invalid runtime profile name: ${name}`); + if (!profile || typeof profile !== "object" || Array.isArray(profile)) { + throw new Error(`runtime profile ${name} must be an object`); + } + if (!BUILDER_ID.test(profile.builder || "")) { + throw new Error(`runtime profile ${name} has an invalid builder`); + } + if (typeof profile.requirements !== "string" || !profile.requirements.trim()) { + throw new Error(`runtime profile ${name} requires a requirements file`); + } + if (!Array.isArray(profile.actions) || profile.actions.length === 0) { + throw new Error(`runtime profile ${name} requires at least one action`); + } + for (const action of profile.actions) { + if (typeof action !== "string" || !ACTION_NAME.test(action)) { + throw new Error(`runtime profile ${name} has invalid action ${action}`); + } + if (assignedActions.has(action)) { + throw new Error(`action ${action} is assigned to more than one runtime profile`); + } + assignedActions.add(action); + } + result[name] = { + builder: profile.builder, + requirements: profile.requirements, + actions: [...new Set(profile.actions)], + }; + } + return result; +} + +export async function runtimeProfileWatchEntries(options = {}) { + const profiles = await loadRuntimeProfiles(options); + return Object.entries(profiles).map(([name, profile]) => ({ + name, + path: path.resolve(projectDirectory(options), profile.requirements), + actions: profile.actions, + })); +} + +async function runOpsProperty(property) { + const ops = process.env.OPS || "ops"; + const proc = Bun.spawn([ops, "-wsk", "property", "get", property], { + env: process.env, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, code] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + if (code !== 0) throw new Error(stderr.trim() || `cannot read ${property}`); + const line = stdout.split("\n").find((item) => item.trim()) || ""; + const value = line.replace(/^whisk\s+(?:API host|auth)\s+/i, "").trim(); + if (!value) throw new Error(`empty ${property} returned by ops`); + return value; +} + +async function defaultCredentials() { + return { + apiHost: (await runOpsProperty("--apihost")).replace(/\/$/, ""), + auth: await runOpsProperty("--auth"), + }; +} + +async function responsePayload(response) { + const text = await response.text(); + try { + return JSON.parse(text); + } catch { + throw new Error(`builder returned HTTP ${response.status}: ${text}`); + } +} + +function validateImage(image) { + if (typeof image !== "string" || !IMAGE_REFERENCE.test(image)) { + throw new Error("builder returned an invalid action image reference"); + } + return image; +} + +export async function ensureRuntimeProfiles(actionFilter = null, options = {}) { + const profiles = await loadRuntimeProfiles(options); + const selectedActions = actionFilter ? new Set(actionFilter) : null; + const selectedProfiles = Object.entries(profiles).filter(([, profile]) => + !selectedActions || profile.actions.some((action) => selectedActions.has(action)) + ); + const runtimeImages = new Map(); + if (selectedProfiles.length === 0) return runtimeImages; + + if (options.dryRun) { + for (const [name, profile] of selectedProfiles) { + console.log(`[dry-run] ensure runtime profile ${name} with ${profile.builder}`); + } + return runtimeImages; + } + + const fetchImpl = options.fetchImpl || fetch; + const sleep = options.sleep || ((milliseconds) => Bun.sleep(milliseconds)); + const credentials = options.credentials || await defaultCredentials(); + const pollInterval = options.pollInterval ?? 2000; + const timeout = options.timeout ?? 15 * 60 * 1000; + + for (const [name, profile] of selectedProfiles) { + const requirementsPath = path.resolve(projectDirectory(options), profile.requirements); + const projectRoot = path.resolve(projectDirectory(options)); + if (requirementsPath !== projectRoot && !requirementsPath.startsWith(`${projectRoot}${path.sep}`)) { + throw new Error(`runtime profile ${name} requirements must stay inside the project`); + } + let requirements; + try { + requirements = await fs.readFile(requirementsPath); + } catch (error) { + throw new Error(`cannot read runtime profile ${name} requirements: ${error.message}`); + } + + const ensureResponse = await fetchImpl(`${credentials.apiHost}/system/api/v1/build/ensure`, { + method: "POST", + headers: { + "authorization": credentials.auth, + "content-type": "application/json", + }, + body: JSON.stringify({ + builder: profile.builder, + file: requirements.toString("base64"), + }), + }); + const initial = await responsePayload(ensureResponse); + if (!ensureResponse.ok && ensureResponse.status !== 202) { + throw new Error(initial.message || `runtime profile ${name} build failed`); + } + + let state = initial; + const startedAt = Date.now(); + while (state.state === "queued" || state.state === "running") { + if (!state.id) throw new Error(`runtime profile ${name} response has no build id`); + if (Date.now() - startedAt >= timeout) { + throw new Error(`runtime profile ${name} build timed out`); + } + await sleep(pollInterval); + const statusResponse = await fetchImpl( + `${credentials.apiHost}/system/api/v1/build/${state.id}`, + {headers: {"authorization": credentials.auth}}, + ); + state = await responsePayload(statusResponse); + if (!statusResponse.ok && statusResponse.status !== 202 && state.state !== "failed") { + throw new Error(state.message || `cannot query runtime profile ${name}`); + } + } + + if (state.state !== "succeeded") { + throw new Error(state.message || `runtime profile ${name} build failed`); + } + const image = validateImage(state.image); + console.log(`Runtime profile ${name}: ${image}`); + for (const action of profile.actions) { + if (!selectedActions || selectedActions.has(action)) runtimeImages.set(action, image); + } + } + return runtimeImages; +} diff --git a/ide/deploy/builder.test.js b/ide/deploy/builder.test.js new file mode 100644 index 00000000..5343bb28 --- /dev/null +++ b/ide/deploy/builder.test.js @@ -0,0 +1,178 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import {afterEach, describe, expect, test} from "bun:test"; +import fs from "fs/promises"; +import os from "os"; +import path from "path"; + +import {ensureRuntimeProfiles, loadRuntimeProfiles} from "./builder.js"; +import {appendRuntimeImage} from "./deploy.js"; + +const temporaryDirectories = []; + +async function project(profile = {}) { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "ops-builder-test-")); + temporaryDirectories.push(directory); + await fs.writeFile( + path.join(directory, "package.json"), + JSON.stringify({openserverless: {runtimeProfiles: profile}}), + ); + await fs.mkdir(path.join(directory, "runtime")); + await fs.writeFile(path.join(directory, "runtime", "python.txt"), "sample-lib==1.0\n"); + return directory; +} + +function response(status, payload) { + return new Response(JSON.stringify(payload), { + status, + headers: {"content-type": "application/json"}, + }); +} + +afterEach(async () => { + await Promise.all(temporaryDirectories.splice(0).map((directory) => fs.rm(directory, {recursive: true}))); +}); + +describe("runtime profile builder", () => { + test("adds a generated image without replacing an explicit docker argument", () => { + expect(appendRuntimeImage("--web true", "registry.example/sample:image")).toBe( + "--web true --docker registry.example/sample:image", + ); + expect(appendRuntimeImage("--docker explicit/image:tag --web true", "generated/image:tag")).toBe( + "--docker explicit/image:tag --web true", + ); + }); + + test("polls until the image is ready and maps it to the action", async () => { + const directory = await project({ + "python-custom": { + builder: "python:3.13", + requirements: "runtime/python.txt", + actions: ["samples/custom"], + }, + }); + const replies = [ + response(202, {state: "queued", id: "a".repeat(64)}), + response(202, {state: "running", id: "a".repeat(64)}), + response(200, { + state: "succeeded", + id: "a".repeat(64), + image: "registry.example:32000/sample:python-digest", + }), + ]; + const requests = []; + + const images = await ensureRuntimeProfiles(null, { + projectDir: directory, + credentials: {apiHost: "https://api.example", auth: "uuid:key"}, + fetchImpl: async (url, options) => { + requests.push({url, options}); + return replies.shift(); + }, + sleep: async () => {}, + pollInterval: 0, + }); + + expect(images.get("samples/custom")).toBe( + "registry.example:32000/sample:python-digest", + ); + expect(requests).toHaveLength(3); + expect(JSON.parse(requests[0].options.body).builder).toBe("python:3.13"); + }); + + test("uses an immediate server cache hit without polling", async () => { + const directory = await project({ + cached: { + builder: "python:3.13", + requirements: "runtime/python.txt", + actions: ["samples/cached"], + }, + }); + let calls = 0; + + const images = await ensureRuntimeProfiles(null, { + projectDir: directory, + credentials: {apiHost: "https://api.example", auth: "uuid:key"}, + fetchImpl: async () => { + calls += 1; + return response(200, { + state: "succeeded", + image: "registry.example:32000/sample:python-cached", + }); + }, + }); + + expect(calls).toBe(1); + expect(images.get("samples/cached")).toEndWith("python-cached"); + }); + + test("dry-run validates profiles without contacting the server", async () => { + const directory = await project({ + dry: { + builder: "python:3.13", + requirements: "runtime/python.txt", + actions: ["samples/dry"], + }, + }); + + const images = await ensureRuntimeProfiles(null, { + projectDir: directory, + dryRun: true, + fetchImpl: async () => { + throw new Error("fetch must not be called"); + }, + }); + + expect(images.size).toBe(0); + }); + + test("rejects actions assigned to multiple profiles", async () => { + const directory = await project({ + first: { + builder: "python:3.13", + requirements: "runtime/python.txt", + actions: ["samples/shared"], + }, + second: { + builder: "python:3.13", + requirements: "runtime/python.txt", + actions: ["samples/shared"], + }, + }); + + await expect(loadRuntimeProfiles({projectDir: directory})).rejects.toThrow( + "assigned to more than one runtime profile", + ); + }); + + test("propagates a failed remote build", async () => { + const directory = await project({ + broken: { + builder: "python:3.13", + requirements: "runtime/python.txt", + actions: ["samples/broken"], + }, + }); + + await expect(ensureRuntimeProfiles(null, { + projectDir: directory, + credentials: {apiHost: "https://api.example", auth: "uuid:key"}, + fetchImpl: async () => response(409, {state: "failed", message: "dependency install failed"}), + })).rejects.toThrow("dependency install failed"); + }); +}); diff --git a/ide/deploy/deploy.js b/ide/deploy/deploy.js index 4b932174..3404fcc3 100644 --- a/ide/deploy/deploy.js +++ b/ide/deploy/deploy.js @@ -23,6 +23,7 @@ const MAINS = ["__main__.py", "index.js", "index.php", "main.go"]; const queue = []; const activeDeployments = new Map(); +let runtimeImages = new Map(); let dryRun = false; @@ -30,8 +31,22 @@ export function setDryRun(b) { dryRun = b; } +export function setRuntimeImages(images) { + runtimeImages = new Map(images || []); +} + +export function mergeRuntimeImages(images) { + for (const [action, image] of images || []) runtimeImages.set(action, image); +} + +export function appendRuntimeImage(args, image) { + if (!image || /(^|\s)--docker(?:\s|=)/.test(args)) return args; + return `${args} --docker ${image}`.trim(); +} + async function exec(cmd) { console.log("$", cmd); + if (dryRun) return; cmd = expandEnv(cmd); const cmdArgs = parse(cmd).filter((arg) => typeof arg === "string"); @@ -114,9 +129,9 @@ export async function deployAction(artifact) { typ = spData[1]; pkg = sp[1]; } catch(error) { - console.log("❌ cannot deploy", artifact, "Error:", error.message); - return; + activeDeployments.delete(artifact); + throw error; } await deployPackage(pkg); @@ -129,17 +144,20 @@ export async function deployAction(artifact) { toInspect = [artifact]; } - const args = (await extractArgs(toInspect)).join(" "); + let args = (await extractArgs(toInspect)).join(" "); const actionName = `${pkg}/${name}`; + const runtimeImage = runtimeImages.get(actionName); + args = appendRuntimeImage(args, runtimeImage); try { await exec(`ops action update ${actionName} ${artifact} ${args}`); } catch(error) { console.log("❌ cannot deploy", artifact, "Error:", error.message); + throw error; + } finally { + activeDeployments.delete(artifact); } - activeDeployments.delete(artifact); - if (queue.length > 0) { const nextArtifact = queue.shift(); console.debug(`📦 deploying from queue artifact ${nextArtifact}`); diff --git a/ide/deploy/index.js b/ide/deploy/index.js index 79c7b5a3..3cd0ede8 100644 --- a/ide/deploy/index.js +++ b/ide/deploy/index.js @@ -20,10 +20,11 @@ import {createServer} from 'net'; import process from 'process'; import {program} from 'commander'; import {scan} from './scan.js'; -import {watchAndDeploy, globalWatcher} from './watch.js'; -import {setDryRun, deploy} from './deploy.js'; +import {watchAndDeploy, globalWatcher, runtimeProfileWatcher} from './watch.js'; +import {setDryRun, setRuntimeImages, deploy} from './deploy.js'; import {undeploy, setDryRun as setUndeployDryRun} from './undeploy.js'; import {build} from './client.js'; +import {ensureRuntimeProfiles} from './builder.js'; @@ -35,6 +36,11 @@ async function signalHandler() { await globalWatcher.close(); } + if (runtimeProfileWatcher) { + console.log("Stopping runtime profile watcher"); + await runtimeProfileWatcher.close(); + } + const pidPath = expanduser('~/.ops/tmp/deploy.pid'); if (existsSync(pidPath)) { unlinkSync(pidPath); @@ -125,13 +131,13 @@ async function main() { } else if (options.watch) { checkPort(); if (!options.fast) { - await scan(); + await scan({dryRun: options.dryRun}); await build(); } await watchAndDeploy(); } else if (options.deploy) { - await scan(); + await scan({dryRun: options.dryRun}); await build(); process.exit(0); } else if (options.single !== '') { @@ -143,6 +149,9 @@ async function main() { console.log(`❌ action ${action} not found: must be either a file or a directory under packages`); return; } + const actionParts = action.split('/'); + const actionName = `${actionParts[1]}/${actionParts[2].split('.')[0]}`; + setRuntimeImages(await ensureRuntimeProfiles([actionName], {dryRun: options.dryRun})); console.log(`Deploying ${action}`); await deploy(action); process.exit(0); diff --git a/ide/deploy/scan.js b/ide/deploy/scan.js index 484089d3..930ae99c 100644 --- a/ide/deploy/scan.js +++ b/ide/deploy/scan.js @@ -16,10 +16,11 @@ // under the License. import {glob} from 'glob'; -import {buildAction, buildZip, deployAction, deployPackage, deployProject} from './deploy.js'; +import {buildAction, buildZip, deployAction, deployPackage, deployProject, setRuntimeImages} from './deploy.js'; import {getOpenServerlessConfig} from './client.js'; import {config} from "dotenv"; import {syncDeployInfo} from "./syncDeployInfo"; +import {ensureRuntimeProfiles} from "./builder.js"; /** * This function will prepare and deploy the functions in `packages` directory. @@ -37,12 +38,14 @@ import {syncDeployInfo} from "./syncDeployInfo"; * ``` * @returns {Promise} */ -export async function scan() { +export async function scan(options = {}) { const deployments = new Set(); const packages = new Set(); console.log("> Scan:"); + setRuntimeImages(await ensureRuntimeProfiles(null, options)); + // => REQUIREMENTS const defaultReqsGlobs = [ "packages/*/*/requirements.txt", diff --git a/ide/deploy/watch.js b/ide/deploy/watch.js index e8af88db..3bb72bcf 100644 --- a/ide/deploy/watch.js +++ b/ide/deploy/watch.js @@ -19,12 +19,14 @@ const SKIPDIR = ["virtualenv", "node_modules", "__pycache__"]; import {watch} from 'chokidar'; import {resolve} from 'path'; -import {deploy} from './deploy.js'; +import {deploy, mergeRuntimeImages} from './deploy.js'; import {logs, serve} from './client.js'; +import {ensureRuntimeProfiles, runtimeProfileWatchEntries} from './builder.js'; import process from 'process'; export let globalWatcher; +export let runtimeProfileWatcher; /** @@ -95,6 +97,29 @@ async function redeploy() { }); } +async function watchRuntimeProfiles() { + const entries = await runtimeProfileWatchEntries(); + if (entries.length === 0) return; + runtimeProfileWatcher = watch(entries.map((entry) => entry.path), { + persistent: true, + ignoreInitial: true, + atomic: 250, + }); + runtimeProfileWatcher.on('change', async (changedPath) => { + const resolved = resolve(changedPath); + const actions = entries + .filter((entry) => entry.path === resolved) + .flatMap((entry) => entry.actions); + if (actions.length === 0) return; + try { + mergeRuntimeImages(await ensureRuntimeProfiles(actions)); + for (const action of actions) await deploy(`packages/${action}`); + } catch (error) { + console.error(`runtime profile rebuild failed: ${error.message}`); + } + }); +} + /** * This function is the entry point and is called when * the program is started with the watch flag on @@ -104,6 +129,7 @@ export async function watchAndDeploy() { await logs(); try { + await watchRuntimeProfiles(); await redeploy(); } catch(error) { diff --git a/ide/opsfile.yml b/ide/opsfile.yml index 531153c6..09d8d529 100644 --- a/ide/opsfile.yml +++ b/ide/opsfile.yml @@ -102,10 +102,12 @@ tasks: desc: setup app cmds: - | + set -e echo ">>>" - ops action list setup | awk 'NR>1{print $1}' | while read action + ACTIONS=$(ops action list) + printf '%s\n' "$ACTIONS" | awk 'NR>1 && index($1, "/setup/"){print $1}' | while read action do echo "=== $action" - ops invoke "$action" || echo "error in $action" + ops invoke "$action" done echo "<<<" diff --git a/setup/kubernetes/crds/whisk-crd.yaml b/setup/kubernetes/crds/whisk-crd.yaml index 73dc3e79..94eb4cc8 100644 --- a/setup/kubernetes/crds/whisk-crd.yaml +++ b/setup/kubernetes/crds/whisk-crd.yaml @@ -825,6 +825,9 @@ spec: hostname: description: used to configure the repo hostname (if set to auto and mode=internal it will be img.) type: string + pull-hostname: + description: node-reachable registry hostname used in action image references + type: string ingress: description: configuration option for global REGISTRY ingresses exposure, will be taken into account only if deployment=interna type: object @@ -999,4 +1002,4 @@ spec: type: string priority: 0 jsonPath: .status.whisk_create.seaweedfs - description: Seaweedfs \ No newline at end of file + description: Seaweedfs diff --git a/setup/kubernetes/roles/nuvolaris-wsku-roles.yaml b/setup/kubernetes/roles/nuvolaris-wsku-roles.yaml index 101cff05..bdcf8bbd 100644 --- a/setup/kubernetes/roles/nuvolaris-wsku-roles.yaml +++ b/setup/kubernetes/roles/nuvolaris-wsku-roles.yaml @@ -33,6 +33,11 @@ rules: - apiGroups: [""] resources: ["configmaps"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +# allow the admin api to read registry credentials used by BuildKit +- apiGroups: [""] + resources: ["secrets"] + resourceNames: ["registry-pull-secret", "registry-pull-secret-int"] + verbs: ["get"] # assign the possibility to operate on jobs (admin api) - apiGroups: ["batch"] resources: ["jobs"] diff --git a/setup/kubernetes/whisk.yaml b/setup/kubernetes/whisk.yaml index 93beb408..5e9f475c 100644 --- a/setup/kubernetes/whisk.yaml +++ b/setup/kubernetes/whisk.yaml @@ -252,6 +252,7 @@ spec: username: ${REGISTRY_CONFIG_USERNAME:-opsuser} password: ${REGISTRY_CONFIG_SECRET_PUSH_PULL:-changeme-registry} hostname: ${REGISTRY_CONFIG_HOSTNAME:-auto} + pull-hostname: ${REGISTRY_CONFIG_PULL_HOSTNAME:-auto} ingress: enabled: ${REGISTRY_CONFIG_INGRESS_ENABLED:-false} seaweedfs: diff --git a/setup/nuvolaris/system-api/api-template.yaml b/setup/nuvolaris/system-api/api-template.yaml index 9a298b5b..6346c223 100644 --- a/setup/nuvolaris/system-api/api-template.yaml +++ b/setup/nuvolaris/system-api/api-template.yaml @@ -50,6 +50,8 @@ spec: failureThreshold: 3 successThreshold: 1 env: + - name: "BUILDER_CATALOG_FILE" + value: "/etc/openserverless/builders.json" - name: "APIHOST" value: "${SYS_API_HOSTNAME:-localhost}" - name: "COUCHDB_SERVICE_PORT" @@ -60,6 +62,15 @@ spec: value: "${SYS_API_CDB_USER}" - name: "COUCHDB_ADMIN_PASSWORD" value: "${SYS_API_CDB_PASSWORD}" + volumeMounts: + - name: builder-catalog + mountPath: /etc/openserverless/builders.json + subPath: builders.json + readOnly: true + volumes: + - name: builder-catalog + configMap: + name: nuvolaris-builder-catalog --- apiVersion: v1 kind: Service diff --git a/setup/nuvolaris/system-api/builders.json b/setup/nuvolaris/system-api/builders.json new file mode 100644 index 00000000..ac0f09a5 --- /dev/null +++ b/setup/nuvolaris/system-api/builders.json @@ -0,0 +1,7 @@ +{ + "builders": { + "python:3.13": { + "kind": "python" + } + } +} diff --git a/setup/nuvolaris/system-api/builders.test.ts b/setup/nuvolaris/system-api/builders.test.ts new file mode 100644 index 00000000..d6c1df76 --- /dev/null +++ b/setup/nuvolaris/system-api/builders.test.ts @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import {describe, expect, test} from "bun:test"; +import {buildCatalog} from "./builders"; + +describe("system API builder catalog", () => { + test("resolves a builder source from the versioned runtime catalog", () => { + const result = buildCatalog( + {runtimes: {python: [{ + kind: "python:3.13", + image: {prefix: "apache", name: "openserverless-runtime-python", tag: "v3.13-test"}, + }]}}, + {builders: {"python:3.13": {kind: "python"}}}, + ); + + expect(result.builders["python:3.13"]).toEqual({ + kind: "python", + source: "docker.io/apache/openserverless-runtime-python:v3.13-test", + }); + }); + + test("fails when a declared builder is missing from runtimes.json", () => { + expect(() => buildCatalog( + {runtimes: {python: []}}, + {builders: {"python:3.13": {kind: "python"}}}, + )).toThrow("is not present"); + }); +}); diff --git a/setup/nuvolaris/system-api/builders.ts b/setup/nuvolaris/system-api/builders.ts new file mode 100644 index 00000000..ec69b01f --- /dev/null +++ b/setup/nuvolaris/system-api/builders.ts @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const BUILDER_ID = /^[a-z0-9][a-z0-9._:-]{0,62}$/; +const IMAGE_PART = /^[A-Za-z0-9][A-Za-z0-9._/-]*$/; +const IMAGE_TAG = /^[A-Za-z0-9_][A-Za-z0-9._-]{0,127}$/; + +type RuntimeImage = {prefix?: string; name?: string; tag?: string}; +type Runtime = {kind?: string; image?: RuntimeImage}; +type RuntimeDocument = {runtimes?: Record}; +type BuilderDefinition = {kind?: string}; +type BuilderDocument = {builders?: Record}; + +export function buildCatalog( + runtimeDocument: RuntimeDocument, + builderDocument: BuilderDocument, +): {builders: Record} { + if (!runtimeDocument?.runtimes || typeof runtimeDocument.runtimes !== "object") { + throw new Error("runtime catalog must contain a runtimes object"); + } + if (!builderDocument?.builders || typeof builderDocument.builders !== "object") { + throw new Error("builder definition must contain a builders object"); + } + + const runtimes = Object.values(runtimeDocument.runtimes).flat(); + const builders: Record = {}; + for (const [builderId, definition] of Object.entries(builderDocument.builders)) { + if (!BUILDER_ID.test(builderId)) throw new Error(`invalid builder id: ${builderId}`); + if (!definition?.kind || !BUILDER_ID.test(definition.kind)) { + throw new Error(`invalid kind for builder ${builderId}`); + } + + const runtime = runtimes.find((entry) => entry.kind === builderId); + if (!runtime) throw new Error(`runtime ${builderId} is not present in runtimes.json`); + const image = runtime.image || {}; + if (!image.prefix || !IMAGE_PART.test(image.prefix) || + !image.name || !IMAGE_PART.test(image.name) || + !image.tag || !IMAGE_TAG.test(image.tag)) { + throw new Error(`runtime ${builderId} has an invalid image`); + } + const prefix = image.prefix.includes(".") || image.prefix.includes(":") + ? image.prefix + : `docker.io/${image.prefix}`; + builders[builderId] = { + kind: definition.kind, + source: `${prefix}/${image.name}:${image.tag}`, + }; + } + return {builders}; +} + +if (import.meta.main) { + const [runtimePath, builderPath] = Bun.argv.slice(2); + if (!runtimePath || !builderPath) { + throw new Error("usage: bun builders.ts "); + } + const runtimeDocument = await Bun.file(runtimePath).json(); + const builderDocument = await Bun.file(builderPath).json(); + console.log(JSON.stringify(buildCatalog(runtimeDocument, builderDocument), null, 2)); +} diff --git a/setup/nuvolaris/system-api/opsfile.yml b/setup/nuvolaris/system-api/opsfile.yml index f5e1767b..c53c43e0 100644 --- a/setup/nuvolaris/system-api/opsfile.yml +++ b/setup/nuvolaris/system-api/opsfile.yml @@ -55,6 +55,20 @@ env: sh: ops util ingress-type tasks: + install-builders: + desc: Install the versioned runtime builder catalog + silent: true + cmds: + - > + bun "$OPS_ROOT/setup/nuvolaris/system-api/builders.ts" + "$OPS_ROOT/runtimes.json" + "$OPS_ROOT/setup/nuvolaris/system-api/builders.json" + > "$OPS_TMP/builders.json" + - > + kubectl -n nuvolaris create configmap nuvolaris-builder-catalog + --from-file=builders.json="$OPS_TMP/builders.json" + --dry-run=client -o yaml | kubectl apply -f - + install-toml: desc: Update the buildkitd.toml file config map silent: true @@ -78,7 +92,7 @@ tasks: - task: remove-secret - | if ! kubectl -n nuvolaris get secret registry-pull-secret-int >/dev/null 2>&1; - then kubectl -n nuvolaris create secret docker-registry registry-pull-secret-int --docker-server=http://nuvolaris-registry-svc:5000 --docker-username=opsuser --docker-password=${REGISTRY_PASS} + then kubectl -n nuvolaris create secret docker-registry registry-pull-secret-int --docker-server=nuvolaris-registry-svc:5000 --docker-username=opsuser --docker-password=${REGISTRY_PASS} fi remove-secret: @@ -97,6 +111,7 @@ tasks: cmds: - test -e ${INGRESS_TYPE}-template.yaml || die "No avalable template for ingress type ${INGRESS_TYPE}." - test -n "$IMAGES_SYSTEMAPI" || die "IMAGES_SYSTEMAPI is not set. Please set it to the desired image version." + - task: install-builders - envsubst -i api-template.yaml -o _api.yaml > /dev/null 2>&1 - envsubst -i ${INGRESS_TYPE}-template.yaml -o _ingress.yaml > /dev/null 2>&1 - kubectl -n nuvolaris apply -f "$OPS_ROOT/setup/nuvolaris/system-api/_api.yaml" @@ -114,6 +129,7 @@ tasks: cmds: - task: remove-secret - task: remove-toml + - kubectl -n nuvolaris delete configmap nuvolaris-builder-catalog --ignore-not-found - kubectl -n nuvolaris delete sts/nuvolaris-system-api ing/nuvolaris-system-api-ingress svc/nuvolaris-system-api - | echo "System Admin API undeployed" @@ -131,4 +147,4 @@ tasks: env: CURRENT_API_VERSION: sh: | - echo $(kubectl -n nuvolaris get sts/nuvolaris-system-api -ojsonpath='{.spec.template.spec.containers[0].image}') \ No newline at end of file + echo $(kubectl -n nuvolaris get sts/nuvolaris-system-api -ojsonpath='{.spec.template.spec.containers[0].image}')