Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/scripts/run-agent-task/execute-native-agent-task.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -283,7 +283,7 @@ function validPublication(value, targetRepo) {
&& publication.success === true
&& publication.status === "published"
&& /^https:\/\/github\.com\/[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+\/pull\/\d+\/?$/.test(url)
&& url.startsWith(`https://github.com/${targetRepo}/pull/`)
&& url.toLowerCase().startsWith(`https://github.com/${targetRepo.toLowerCase()}/pull/`)
}

async function verifyPublishedPullRequest(publication, targetRepo, cwd) {
Expand All @@ -297,7 +297,7 @@ async function verifyPublishedPullRequest(publication, targetRepo, cwd) {
const pullRequest = JSON.parse(response.stdout)
return {
valid: pullRequest?.html_url === record(publication).pull_request?.url
&& pullRequest?.base?.repo?.full_name === targetRepo,
&& string(pullRequest?.base?.repo?.full_name).toLowerCase() === targetRepo.toLowerCase(),
error: "Published pull request did not resolve to the target repository.",
}
} catch {
Expand Down
29 changes: 21 additions & 8 deletions tests/execute-native-agent-task-lifecycle.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,10 @@ const execFileAsync = promisify(execFile)
async function run(mode = "success") {
const temp = await mkdtemp(join(tmpdir(), "wp-codebox-native-lifecycle-"))
const workspace = join(temp, "workspace")
const targetRepo = mode === "canonical-casing" ? "automattic/build-with-wordpress" : "owner/repo"
const publicationRepo = mode === "canonical-casing"
? "Automattic/build-with-wordpress"
: mode === "different-repo" ? "other/repo" : targetRepo
await mkdir(join(workspace, ".codebox"), { recursive: true })
await writeFile(join(workspace, "README.md"), "OPENAI_API_KEY\nbefore\n")

Expand All @@ -28,12 +32,12 @@ async function run(mode = "success") {
},
runtime_sources: [],
workload: { id: "run-1", label: "Update" },
target_repo: "owner/repo",
target_repo: targetRepo,
prompt: "Edit README using workspace_read and workspace_edit.",
writable_paths: mode === "deny" ? "src/**" : "README.md",
runner_workspace: {
enabled: true,
repo: "owner/repo",
repo: targetRepo,
base: "main",
branch_prefix: "wp-codebox/agent-task/",
},
Expand All @@ -44,9 +48,9 @@ async function run(mode = "success") {
drift_checks: [{ command: "echo drift >> .codebox/order" }],
success: { requires_pr: mode !== "no-op-maintenance" },
access: {
caller_repo: "owner/repo",
allowed_repos: ["owner/repo"],
access_token_repos: ["owner/repo"],
caller_repo: targetRepo,
allowed_repos: [targetRepo],
access_token_repos: [targetRepo],
},
limits: { max_turns: 1, time_budget_ms: 1000 },
artifacts: { expected: [], declarations: [] },
Expand Down Expand Up @@ -127,16 +131,16 @@ export async function publishRunnerWorkspace({ testHook }) {
success: true,
status: "published",
backend: "fixture",
pull_request: { url: "https://github.com/owner/repo/pull/1", reused: false, opened: true },
pull_request: { url: ${JSON.stringify(`https://github.com/${publicationRepo}/pull/1`)}, reused: false, opened: true },
}
}
`)

const gh = join(temp, "gh")
await writeFile(gh, `#!/usr/bin/env node
process.stdout.write(JSON.stringify({
html_url: "https://github.com/owner/repo/pull/1",
base: { repo: { full_name: "owner/repo" } },
html_url: ${JSON.stringify(`https://github.com/${publicationRepo}/pull/1`)},
base: { repo: { full_name: ${JSON.stringify(publicationRepo)} } },
}))
`)
await chmod(gh, 0o755)
Expand Down Expand Up @@ -190,6 +194,15 @@ assert.equal(success.result.runtime_result.agent_runtime_diagnostics.prepared_pa
assert.equal(success.result.runtime_result.agent_runtime_diagnostics.sandbox_workspace.mounts[0].target, "/workspace")
assert.equal(success.result.runtime_result.agent_runtime_diagnostics.local_executor_root, "/workspace")

const canonicalCasing = await run("canonical-casing")
assert.equal(canonicalCasing.code, 0, `${canonicalCasing.stderr}\n${JSON.stringify(canonicalCasing.result)}`)
assert.equal(canonicalCasing.result.publication_verification.valid, true)

const differentRepository = await run("different-repo")
assert.equal(differentRepository.code, 1)
assert.equal(differentRepository.result.publication_verification.valid, false)
assert.match(differentRepository.result.failure.message, /valid canonical pull-request result/)

const failedVerification = await run("verify-fail")
assert.equal(failedVerification.code, 1)
assert(!failedVerification.order.includes("publish"))
Expand Down
Loading