diff --git a/.agents/skills/buildkite-get-results/scripts/get_buildkite_results.py b/.agents/skills/buildkite-get-results/scripts/get_buildkite_results.py index e1fc635cf6..06117fbd7c 100755 --- a/.agents/skills/buildkite-get-results/scripts/get_buildkite_results.py +++ b/.agents/skills/buildkite-get-results/scripts/get_buildkite_results.py @@ -94,24 +94,32 @@ def fetch_buildkite_data(build_url): def download_log(job_url, output_path): - # Construct raw log URL: job_url + "/raw" (Buildkite convention) - # job_url e.g. https://buildkite.com/org/pipeline/builds/14394#job-id - # Wait, the job['path'] gives /org/pipeline/builds/14394#job-id - # We want /org/pipeline/builds/14394/jobs/job-id/raw? No - # The clean URL for a job is https://buildkite.com/org/pipeline/builds/14394/jobs/job-id - # And raw log is https://buildkite.com/org/pipeline/builds/14394/jobs/job-id/raw - - # We have full_url e.g. https://buildkite.com/bazel/rules-python-python/builds/14394#019c5cf9-e3cf-468f-a7b1-8f9f5ad4b08c - # We need to transform it. + # job_url looks like: + # https://buildkite.com/bazel/rules-python-python/builds/15594#019e879b-... + # We need to transform it to: + # https://buildkite.com/organizations/bazel/pipelines/rules-python-python/builds/15594/jobs/{job_id}/download.txt if "#" in job_url: base, job_id = job_url.split("#") - # Ensure base doesn't end with / - if base.endswith("/"): - base = base[:-1] - - # Build raw URL - raw_url = f"{base}/jobs/{job_id}/raw" + base = base.rstrip("/") + + # Parse the path segments: https://buildkite.com/org/pipeline/builds/N + # Rebuild with the /organizations/org/pipelines/pipeline/ format which + # supports the /jobs/{id}/download.txt log URL without auth. + parts = base.split("/") + # parts = ["https:", "", "buildkite.com", "org", "pipeline", "builds", "N"] + if len(parts) >= 7 and parts[2] == "buildkite.com": + org = parts[3] + pipeline = parts[4] + build_num = parts[6] if len(parts) >= 7 else "" + raw_url = ( + f"https://buildkite.com/organizations/{org}" + f"/pipelines/{pipeline}" + f"/builds/{build_num}" + f"/jobs/{job_id}/download.txt" + ) + else: + raw_url = f"{base}/jobs/{job_id}/download.txt" else: print(f"Could not parse job URL for download: {job_url}", file=sys.stderr) return False diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 6366741313..0f7952c590 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -140,7 +140,6 @@ tasks: test_targets: - "//..." - # Keep in sync with .bcr/gazelle/presubmit.yml gazelle_bcr_ubuntu: <<: *gazelle_common_bcr name: "Gazelle: BCR, Bazel {bazel}" @@ -203,7 +202,6 @@ tasks: <<: *common_workspace_flags_min_bazel name: "Default: Ubuntu, workspace, minimum Bazel" platform: ubuntu2204 - ubuntu_min_bzlmod: <<: *minimum_supported_version <<: *reusable_config @@ -220,8 +218,6 @@ tasks: name: "Default: Ubuntu, upcoming Bazel" platform: ubuntu2204 bazel: last_rc - # This is an advisory job; doesn't block merges - # RCs may have regressions, so don't fail our CI on them soft_fail: - exit_status: 1 - exit_status: 3 @@ -229,7 +225,6 @@ tasks: name: "Default: Ubuntu, rolling Bazel" platform: ubuntu2204 bazel: rolling - # This is an advisory job; doesn't block merges soft_fail: - exit_status: 1 - exit_status: 3 @@ -299,25 +294,14 @@ tasks: platform: rbe_ubuntu2404 build_flags: - "--experimental_repository_cache_hardlinks=false" - # BazelCI sets --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1, - # which prevents cc toolchain autodetection from working correctly - # on Bazel 5.4 and earlier. To workaround this, manually specify the - # build kite cc toolchain. - "--extra_toolchains=@buildkite_config//config:cc-toolchain" - - "--build_tag_filters=-docs" test_flags: - "--test_tag_filters=-integration-test,-acceptance-test,-docs" - # BazelCI sets --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1, - # which prevents cc toolchain autodetection from working correctly - # on Bazel 5.4 and earlier. To workaround this, manually specify the - # build kite cc toolchain. - "--extra_toolchains=@buildkite_config//config:cc-toolchain" rbe: <<: *reusable_config name: "RBE: Ubuntu" platform: rbe_ubuntu2404 - # TODO @aignas 2024-12-11: get the RBE working in CI for bazel 8.0 - # See https://github.com/bazelbuild/rules_python/issues/2499 bazel: 8.x test_flags: - "--test_tag_filters=-integration-test,-acceptance-test" @@ -465,9 +449,6 @@ tasks: <<: *common_bazelinbazel_config name: "tests/integration bazel-in-bazel: Debian" platform: debian11 - # The bazelinbazel tests were disabled on Mac to save CI jobs slots, and - # have bitrotted a bit. For now, just run a subset of what we're most - # interested in. integration_test_bazelinbazel_macos: <<: *common_bazelinbazel_config name: "tests/integration bazel-in-bazel: macOS (subset)" @@ -490,14 +471,10 @@ tasks: working_directory: tests/integration/compile_pip_requirements platform: ubuntu2204 shell_commands: - # Make a change to the locked requirements and then assert that //:requirements.update does the - # right thing. - "echo '' > requirements_lock.txt" - "! git diff --exit-code" - "bazel run //:requirements.update" - "git diff --exit-code" - # Make a change to the locked requirements and then assert that //:os_specific_requirements.update does the - # right thing. - "echo '' > requirements_lock_linux.txt" - "! git diff --exit-code" - "bazel run //:os_specific_requirements.update" @@ -508,14 +485,10 @@ tasks: working_directory: tests/integration/compile_pip_requirements platform: debian11 shell_commands: - # Make a change to the locked requirements and then assert that //:requirements.update does the - # right thing. - "echo '' > requirements_lock.txt" - "! git diff --exit-code" - "bazel run //:requirements.update" - "git diff --exit-code" - # Make a change to the locked requirements and then assert that //:os_specific_requirements.update does the - # right thing. - "echo '' > requirements_lock_linux.txt" - "! git diff --exit-code" - "bazel run //:os_specific_requirements.update" @@ -526,14 +499,10 @@ tasks: working_directory: tests/integration/compile_pip_requirements platform: macos_arm64 shell_commands: - # Make a change to the locked requirements and then assert that //:requirements.update does the - # right thing. - "echo '' > requirements_lock.txt" - "! git diff --exit-code" - "bazel run //:requirements.update" - "git diff --exit-code" - # Make a change to the locked requirements and then assert that //:os_specific_requirements.update does the - # right thing. - "echo '' > requirements_lock_darwin.txt" - "! git diff --exit-code" - "bazel run //:os_specific_requirements.update" @@ -564,7 +533,6 @@ tasks: working_directory: tests/integration/compile_pip_requirements_test_from_external_repo platform: ubuntu2204 shell_commands: - # Assert that @compile_pip_requirements//:requirements_test does the right thing. - "bazel test @compile_pip_requirements//..." integration_compile_pip_requirements_test_from_external_repo_ubuntu_min_bzlmod: <<: *minimum_supported_version @@ -573,28 +541,24 @@ tasks: platform: ubuntu2204 bazel: 7.x shell_commands: - # Assert that @compile_pip_requirements//:requirements_test does the right thing. - "bazel test @compile_pip_requirements//..." integration_compile_pip_requirements_test_from_external_repo_ubuntu: name: "compile_pip_requirements_test_from_external_repo: Ubuntu" working_directory: tests/integration/compile_pip_requirements_test_from_external_repo platform: ubuntu2204 shell_commands: - # Assert that @compile_pip_requirements//:requirements_test does the right thing. - "bazel test @compile_pip_requirements//..." integration_compile_pip_requirements_test_from_external_repo_debian: name: "compile_pip_requirements_test_from_external_repo: Debian" working_directory: tests/integration/compile_pip_requirements_test_from_external_repo platform: debian11 shell_commands: - # Assert that @compile_pip_requirements//:requirements_test does the right thing. - "bazel test @compile_pip_requirements//..." integration_compile_pip_requirements_test_from_external_repo_macos: name: "compile_pip_requirements_test_from_external_repo: macOS" working_directory: tests/integration/compile_pip_requirements_test_from_external_repo platform: macos_arm64 shell_commands: - # Assert that @compile_pip_requirements//:requirements_test does the right thing. - "bazel test @compile_pip_requirements//..." integration_compile_pip_requirements_test_from_external_repo_windows: name: "compile_pip_requirements_test_from_external_repo: Windows" diff --git a/.bazelignore b/.bazelignore index afd162998a..2cf1523aef 100644 --- a/.bazelignore +++ b/.bazelignore @@ -35,3 +35,4 @@ tests/integration/compile_pip_requirements/bazel-compile_pip_requirements tests/integration/local_toolchains/bazel-local_toolchains tests/integration/py_cc_toolchain_registered/bazel-py_cc_toolchain_registered tests/integration/toolchain_target_settings/bazel-module_under_test +tests/integration/uv_lock/bazel-uv_lock diff --git a/.bazelrc.deleted_packages b/.bazelrc.deleted_packages index f4ea8527f3..7256937a87 100644 --- a/.bazelrc.deleted_packages +++ b/.bazelrc.deleted_packages @@ -39,6 +39,7 @@ common --deleted_packages=tests/integration/pip_parse/empty common --deleted_packages=tests/integration/pip_parse_isolated common --deleted_packages=tests/integration/py_cc_toolchain_registered common --deleted_packages=tests/integration/toolchain_target_settings +common --deleted_packages=tests/integration/uv_lock common --deleted_packages=tests/modules/another_module common --deleted_packages=tests/modules/other common --deleted_packages=tests/modules/other/nspkg_delta diff --git a/.gitattributes b/.gitattributes index eae260e931..9905cbfacb 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,3 @@ python/features.bzl export-subst tools/publish/*.txt linguist-generated=true +tests/uv/lock/testdata/requirements.txt text eol=lf diff --git a/.gitignore b/.gitignore index fb1b17e466..efce592aa0 100644 --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,7 @@ user.bazelrc # MODULE.bazel.lock is ignored for now as per recommendation from upstream. # See https://github.com/bazelbuild/bazel/issues/20369 MODULE.bazel.lock + +# Buildkite logs +*Windows*.log + diff --git a/CHANGELOG.md b/CHANGELOG.md index e57e0969d3..a616aea591 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -119,6 +119,9 @@ END_UNRELEASED_TEMPLATE * (pypi) `package_metadata` support, fixes [#2054](https://github.com/bazel-contrib/rules_python/issues/2054). * (coverage) Add support for python 3.14 and bump `coverage.py` to 7.10.7. +* (uv) allow user overwrite the build environment using `--action_env` to allow + setting authentication for the index URL. + ([#3045](https://github.com/bazel-contrib/rules_python/issues/3405)) {#v2-0-2} ## [2.0.2] - 2026-05-14 diff --git a/python/uv/private/lock.bat b/python/uv/private/lock.bat index 3954c10347..5190ddf9eb 100755 --- a/python/uv/private/lock.bat +++ b/python/uv/private/lock.bat @@ -1,7 +1,7 @@ -if defined BUILD_WORKSPACE_DIRECTORY ( - set "out=%BUILD_WORKSPACE_DIRECTORY%\{{src_out}}" -) else ( - exit /b 1 -) - -"{{args}}" --output-file "%out%" %* +if defined BUILD_WORKSPACE_DIRECTORY ( + set "out=%BUILD_WORKSPACE_DIRECTORY%\{{src_out}}" +) else ( + exit /b 1 +) + +"{{args}}" --output-file "%out%" %* diff --git a/python/uv/private/lock.bzl b/python/uv/private/lock.bzl index b007baf9c1..6f0b80af89 100644 --- a/python/uv/private/lock.bzl +++ b/python/uv/private/lock.bzl @@ -117,31 +117,96 @@ def _lock_impl(ctx): args.run_shell.add("--no-progress") args.run_shell.add("--quiet") + # Generate a wrapper script that copies the existing output (if any) and + # then runs uv. On POSIX, args are forwarded via exec "$@". On Windows, + # the full command line is embedded in the .bat file with backslash paths + # (CMD doesn't recognize forward slashes in executable paths). + if ctx.attr.is_windows: + ext = ".bat" + lines = ["@echo off"] + else: + ext = ".sh" + lines = ["#!/usr/bin/env bash", "set -euo pipefail"] + + python_path = getattr(python, "path", python) + if ctx.files.existing_output: - command = '{python} -c {python_cmd} && "$@"'.format( - python = getattr(python, "path", python), - python_cmd = shell.quote( - "from shutil import copy; copy(\"{src}\", \"{dst}\")".format( + python_cmd = "from shutil import copy; copy(\"{src}\", \"{dst}\")".format( + src = ctx.files.existing_output[0].path, + dst = output.path, + ) + if ctx.attr.is_windows: + # In batch files, use "" to escape internal double quotes. + lines.append( + "\"{py}\" -c \"from shutil import copy; copy(\"\"{src}\"\", \"\"{dst}\"\")\"".format( + py = python_path, src = ctx.files.existing_output[0].path, dst = output.path, ), + ) + else: + lines.append("{py} -c '{cmd}'".format( + py = python_path, + cmd = python_cmd, + )) + + if ctx.attr.is_windows: + # Build the command line with backslash paths for CMD. + # args.run_info has most args; add the output/progress/quiet + # args that were only added directly to args.run_shell. + def _quote(arg): + if hasattr(arg, "path"): + arg = arg.path.replace("/", "\\") + else: + arg = str(arg) + return '"' + arg.replace('"', '""') + '"' + + bat_args = args.run_info + [ + "--output-file", + output, + "--no-progress", + "--quiet", + ] + lines.append(" ".join([_quote(a) for a in bat_args])) + + # Normalize CRLF line endings in the output on Windows. + lines.append( + "\"{py}\" -c \"import pathlib;p=pathlib.Path(r\"\"{dst}\"\");p.write_bytes(p.read_bytes().replace(b'\\r\\n', b'\\n'))\"".format( + py = python_path, + dst = output.path, ), ) else: - command = '"$@"' + lines.append('exec "$@"') + + script = ctx.actions.declare_file(ctx.label.name + "_lock" + ext) + if ctx.attr.is_windows: + content = "\r\n".join(lines) + "\r\n" + else: + content = "\n".join(lines) + "\n" + ctx.actions.write(output = script, content = content, is_executable = True) srcs = srcs + ctx.files.build_constraints + ctx.files.constraints - ctx.actions.run_shell( - command = command, + ctx.actions.run( + executable = script, inputs = srcs + ctx.files.existing_output, mnemonic = "PyRequirementsLockUv", outputs = [output], - arguments = [args.run_shell], + # On Windows, the command line is embedded directly in the .bat + # script (with backslash paths). On POSIX, args are forwarded via + # exec "$@" in the .sh script. + arguments = [args.run_shell] if not ctx.attr.is_windows else [], tools = [ uv, python_files, + script, ], + # User reported being unable to add `--action_env` and get it to work. + # Without this flag. + # + # Ref: https://app.slack.com/client/TA4K1KQ87/CA306CEV6 + use_default_shell_env = True, progress_message = "Creating a requirements.txt with uv: %{label}", env = ctx.attr.env, ) @@ -205,6 +270,7 @@ modifications and the locking is not done from scratch. doc = "Public, see the docs in the macro.", default = True, ), + "is_windows": attr.bool(mandatory = True), "output": attr.string( doc = "Public, see the docs in the macro.", mandatory = True, @@ -241,7 +307,7 @@ The string to input for the 'uv pip compile'. def _lock_run_impl(ctx): if ctx.attr.is_windows: path_sep = "\\" - ext = ".exe" + ext = ".bat" else: path_sep = "/" ext = "" @@ -250,7 +316,12 @@ def _lock_run_impl(ctx): if hasattr(arg, "short_path"): arg = arg.short_path - return shell.quote(arg.replace("/", path_sep)) + arg = arg.replace("/", path_sep) + if ctx.attr.is_windows: + # On Windows, CMD uses double quotes for quoting, and internal + # double quotes are escaped by doubling them. + return '"' + arg.replace('"', '""') + '"' + return shell.quote(arg) info = ctx.attr.lock[_RunLockInfo] executable = ctx.actions.declare_file(ctx.label.name + ext) @@ -438,6 +509,10 @@ def lock( env = env, existing_output = maybe_out, generate_hashes = generate_hashes, + is_windows = select({ + "@platforms//os:windows": True, + "//conditions:default": False, + }), python_version = python_version, srcs = srcs, strip_extras = strip_extras, diff --git a/python/uv/private/lock_copier.py b/python/uv/private/lock_copier.py index bcc64c1661..8756fc4de6 100644 --- a/python/uv/private/lock_copier.py +++ b/python/uv/private/lock_copier.py @@ -55,7 +55,7 @@ def main(): "This must be either run as `bazel test` via a `native_test` or similar or via `bazel run`" ) - print(f"cp /{src} /{dst}") + print(f"cp /{src.as_posix()} /{dst}") build_workspace = Path(environ["BUILD_WORKSPACE_DIRECTORY"]) dst_real_path = build_workspace / dst diff --git a/tests/integration/BUILD.bazel b/tests/integration/BUILD.bazel index 9295cbb22f..caebbf01d8 100644 --- a/tests/integration/BUILD.bazel +++ b/tests/integration/BUILD.bazel @@ -13,6 +13,7 @@ # limitations under the License. load("@rules_bazel_integration_test//bazel_integration_test:defs.bzl", "default_test_runner") +load("//python:py_binary.bzl", "py_binary") load("//python:py_library.bzl", "py_library") load(":integration_test.bzl", "rules_python_integration_test") @@ -48,6 +49,7 @@ test_suite( tests = [ "bzlmod_lockfile_test_bazel_9.1.0", "local_toolchains_test_bazel_self", + "uv_lock_test_bazel_self", ], ) @@ -111,8 +113,30 @@ rules_python_integration_test( py_main = "toolchain_target_settings_test.py", ) +rules_python_integration_test( + name = "uv_lock_test", + py_deps = [ + "@pypiserver//pypiserver", + ":uv_lock_pypi_server_lib", + ], + py_main = "uv_lock_test.py", +) + py_library( name = "runner_lib", srcs = ["runner.py"], imports = ["../../"], ) + +py_library( + name = "uv_lock_pypi_server_lib", + srcs = ["uv_lock_pypi_server.py"], + imports = ["../../"], + deps = ["@pypiserver//pypiserver"], +) + +py_binary( + name = "uv_lock_pypi_server", + srcs = ["uv_lock_pypi_server.py"], + deps = [":uv_lock_pypi_server_lib"], +) diff --git a/tests/integration/integration_test.bzl b/tests/integration/integration_test.bzl index 771976d037..f3d5cb6967 100644 --- a/tests/integration/integration_test.bzl +++ b/tests/integration/integration_test.bzl @@ -21,14 +21,14 @@ load( ) load("//python:py_test.bzl", "py_test") -def _test_runner(*, name, bazel_version, py_main, bzlmod): +def _test_runner(*, name, bazel_version, py_main, bzlmod, py_deps): if py_main: test_runner = "{}_bazel_{}_py_runner".format(name, bazel_version) py_test( name = test_runner, srcs = [py_main], main = py_main, - deps = [":runner_lib"], + deps = [":runner_lib"] + py_deps, # Hide from ... patterns; should only be run as part # of the bazel integration test tags = ["manual"], @@ -46,6 +46,7 @@ def rules_python_integration_test( bzlmod = True, tags = None, py_main = None, + py_deps = None, bazel_versions = None, **kwargs): """Runs a bazel-in-bazel integration test. @@ -60,6 +61,7 @@ def rules_python_integration_test( py_main: Optional `.py` file to run tests using. When specified, a python based test runner is used, and this source file is the main entry point and responsible for executing tests. + py_deps: Optional test runner deps to use for setup. bazel_versions: `list[str] | None`, the bazel versions to test. I not specified, defaults to all configured bazel versions. **kwargs: Passed to the upstream `bazel_integration_tests` rule. @@ -91,6 +93,7 @@ def rules_python_integration_test( name = name, bazel_version = bazel_version, py_main = py_main, + py_deps = py_deps or [], bzlmod = bzlmod, ) bazel_integration_test( diff --git a/tests/integration/uv_lock/.bazelrc b/tests/integration/uv_lock/.bazelrc new file mode 100644 index 0000000000..511f8a2413 --- /dev/null +++ b/tests/integration/uv_lock/.bazelrc @@ -0,0 +1,5 @@ +build --enable_runfiles +common --experimental_isolated_extension_usages +common --action_env=UV_EXTRA_INDEX_URL + +try-import %workspace%/user.bazelrc diff --git a/tests/integration/uv_lock/.bazelversion b/tests/integration/uv_lock/.bazelversion new file mode 100644 index 0000000000..47da986f86 --- /dev/null +++ b/tests/integration/uv_lock/.bazelversion @@ -0,0 +1 @@ +9.1.0 diff --git a/tests/integration/uv_lock/BUILD.bazel b/tests/integration/uv_lock/BUILD.bazel new file mode 100644 index 0000000000..da6482ba08 --- /dev/null +++ b/tests/integration/uv_lock/BUILD.bazel @@ -0,0 +1,26 @@ +load("@bazel_skylib//rules:diff_test.bzl", "diff_test") +load("@rules_python//python/uv:lock.bzl", "lock") +load(":uv_runner.bzl", "uv_runner") + +lock( + name = "requirements", + srcs = ["requirements.in"], + out = "requirements.txt", + tags = ["no-remote-exec"], +) + +uv_runner( + name = "uv", + is_windows = select({ + "@platforms//os:windows": True, + "//conditions:default": False, + }), + tags = ["manual"], +) + +diff_test( + name = "requirements_diff_test", + timeout = "short", + file1 = ":requirements", + file2 = ":requirements.txt", +) diff --git a/tests/integration/uv_lock/MODULE.bazel b/tests/integration/uv_lock/MODULE.bazel new file mode 100644 index 0000000000..0e0978ed36 --- /dev/null +++ b/tests/integration/uv_lock/MODULE.bazel @@ -0,0 +1,18 @@ +module(name = "uv_lock") + +bazel_dep(name = "bazel_skylib", version = "1.7.1") +bazel_dep(name = "platforms", version = "0.0.10") +bazel_dep(name = "rules_python") +local_path_override( + module_name = "rules_python", + path = "../../..", +) + +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +python.toolchain(python_version = "3.13") + +uv = use_extension("@rules_python//python/uv:uv.bzl", "uv") +uv.configure(version = "0.11.2") +use_repo(uv, "uv") + +register_toolchains("@uv//:all") diff --git a/tests/integration/uv_lock/README.md b/tests/integration/uv_lock/README.md new file mode 100644 index 0000000000..cce142852c --- /dev/null +++ b/tests/integration/uv_lock/README.md @@ -0,0 +1,64 @@ +# uv_lock integration test workspace + +This directory is a self-contained Bazel workspace used by the +`//tests/integration:uv_lock_test` integration test. + +It demonstrates how to use the `lock()` macro from `@rules_python//python/uv` +to pin requirements with `uv pip compile`. + +## Targets + +| Target | Description | +|--------|-------------| +| `//:requirements` | Build action that produces the locked requirements file | +| `//:requirements.update` | Update the in-source `requirements.txt` via `bazel run` | +| `//:requirements.run` | Run `uv pip compile` with extra command-line args | +| `//:requirements_diff_test` | Diff test comparing the lock output to the in-source file | +| `//:uv` | The `uv` binary from the registered toolchain | + +## Workflow for debugging + +If you want to debug and play around, you can start the server and then run the uv lock command +manually. + +### Start the local PyPI server + +In a separate terminal, start the pypiserver that serves the `my-local-pkg` test wheel: + +```shell +bazel run //tests/integration:uv_lock_pypi_server [-- --no-auth] +``` + +The server prints the URL to use (with and without authentication) and the +SHA256 of the wheel. Pass `--no-auth` to allow anonymous access. + +### Lock the requirements + +With the server running, lock the requirements from this directory: + +```shell +cd tests/integration/uv_lock +bazel run //:requirements.update \ + --action_env=UV_EXTRA_INDEX_URL="" \ + --action_env=UV_CREDENTIALS_DIR= +``` + +The `` and `` values are printed by the pypi-server. + +### Verify the lock output matches the in-source file + +```shell +bazel test //:requirements_diff_test +``` + +## bazel-in-bazel Testing + +When iterating on changes to `lock.bzl`, the integration test can be run +directly from rules_python: + +```shell +bazel test //tests/integration:uv_lock_test_bazel_self \ + --config=fast-tests \ + --test_output=streamed \ + --test_filter= +``` diff --git a/tests/integration/uv_lock/WORKSPACE b/tests/integration/uv_lock/WORKSPACE new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/uv_lock/requirements.in b/tests/integration/uv_lock/requirements.in new file mode 100644 index 0000000000..fba55a7329 --- /dev/null +++ b/tests/integration/uv_lock/requirements.in @@ -0,0 +1 @@ +my-local-pkg==1.0.0 diff --git a/tests/integration/uv_lock/requirements.txt b/tests/integration/uv_lock/requirements.txt new file mode 100644 index 0000000000..52f5c757f5 --- /dev/null +++ b/tests/integration/uv_lock/requirements.txt @@ -0,0 +1,5 @@ +# This file was autogenerated by uv via the following command: +# bazel run //:requirements.update +my-local-pkg==1.0.0 \ + --hash=sha256:be24d5183a182e8da4465ae1b7e60324864d1a72866ec9aefa6aaf80a4529eb1 + # via -r requirements.in diff --git a/tests/integration/uv_lock/uv_runner.bzl b/tests/integration/uv_lock/uv_runner.bzl new file mode 100644 index 0000000000..3faa3c6fc0 --- /dev/null +++ b/tests/integration/uv_lock/uv_runner.bzl @@ -0,0 +1,30 @@ +"""A rule exposing the uv binary from the registered toolchain as an executable target. + +This allows running ``bazel run //:uv -- `` from the test workspace. +""" + +def _uv_runner_impl(ctx): + toolchain_info = ctx.toolchains["@rules_python//python/uv:uv_toolchain_type"] + original_uv_executable = toolchain_info.uv_toolchain_info.uv[DefaultInfo].files_to_run.executable + + ext = "" + if ctx.attr.is_windows: + ext = ".exe" + + uv_exe = ctx.actions.declare_file("uv" + ext) + ctx.actions.symlink(output = uv_exe, target_file = original_uv_executable) + + return DefaultInfo( + files = depset([uv_exe]), + executable = uv_exe, + runfiles = toolchain_info.default_info.default_runfiles, + ) + +uv_runner = rule( + implementation = _uv_runner_impl, + executable = True, + attrs = { + "is_windows": attr.bool(mandatory = True), + }, + toolchains = ["@rules_python//python/uv:uv_toolchain_type"], +) diff --git a/tests/integration/uv_lock_pypi_server.py b/tests/integration/uv_lock_pypi_server.py new file mode 100644 index 0000000000..0d940e7569 --- /dev/null +++ b/tests/integration/uv_lock_pypi_server.py @@ -0,0 +1,148 @@ +import argparse +import hashlib +import io +import os +import sys +import uuid +import zipfile +from wsgiref.simple_server import make_server + +from pypiserver import app_from_config, setup_routes_from_config +from pypiserver.config import Config + + +def _create_wheel_bytes(name, version): + pkg_name_normalized = name.replace("-", "_") + wheel_name = "{}-{}-py3-none-any.whl".format(pkg_name_normalized, version) + dist_info = "{}-{}.dist-info".format(pkg_name_normalized, version) + + metadata = ( + "Metadata-Version: 2.1\n" + "Name: {name}\n" + "Version: {version}\n" + "Summary: A test package\n" + ).format(name=pkg_name_normalized, version=version) + + wheel_file = ( + "Wheel-Version: 1.0\n" + "Generator: test\n" + "Root-Is-Purelib: true\n" + "Tag: py3-none-any\n" + ) + + record_entries = [ + "{}/__init__.py,".format(pkg_name_normalized), + "{}/METADATA,".format(dist_info), + "{}/WHEEL,".format(dist_info), + "{}/RECORD,".format(dist_info), + ] + + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf: + zf.writestr("{}/__init__.py".format(pkg_name_normalized), "# empty\n") + zf.writestr("{}/METADATA".format(dist_info), metadata) + zf.writestr("{}/WHEEL".format(dist_info), wheel_file) + zf.writestr("{}/RECORD".format(dist_info), "\n".join(record_entries)) + + wheel_data = buf.getvalue() + sha256 = hashlib.sha256(wheel_data).hexdigest() + return wheel_data, sha256, wheel_name + + +def main(): + parser = argparse.ArgumentParser( + description="Standalone pypiserver for uv_lock integration tests" + ) + parser.add_argument( + "--packages-dir", + type=str, + default=None, + help="Directory for the test wheels (default: $TEST_TMPDIR/pypi-server-packages)", + ) + parser.add_argument( + "--no-auth", + action="store_true", + default=False, + help="Disable authentication (allows anonymous access)", + ) + parser.add_argument( + "--port", + type=int, + default=0, + help="Port to listen on (0 = find free port)", + ) + parser.add_argument( + "--host", + default="localhost", + help="Host to bind to", + ) + args = parser.parse_args() + + if args.packages_dir is None: + sandbox_root = ( + os.environ.get("TEST_TMPDIR") or os.environ.get("TMPDIR") or "/tmp" + ) + args.packages_dir = os.path.join(sandbox_root, "pypi-server-packages") + packages_dir = args.packages_dir + os.makedirs(packages_dir, exist_ok=True) + + wheel_data, sha256, wheel_name = _create_wheel_bytes("my-local-pkg", "1.0.0") + wheel_path = os.path.join(packages_dir, wheel_name) + with open(wheel_path, "wb") as f: + f.write(wheel_data) + + print("Wheel: {}".format(wheel_path), flush=True) + print("SHA256: {}".format(sha256), flush=True) + + password = uuid.uuid4().hex + username = "testuser" + + if args.no_auth: + authenticate = [] + else: + authenticate = ["download", "list", "update"] + + config = Config.default_with_overrides( + roots=[packages_dir], + port=args.port, + host=args.host, + authenticate=authenticate, + password_file=None, + auther=lambda u, p: u == username and p == password, + disable_fallback=True, + fallback_url="", + server_method="wsgiref", + verbosity=0, + log_stream=None, + ) + app = app_from_config(config) + app = setup_routes_from_config(app, config) + + server = make_server(args.host, args.port, app) + port = server.server_address[1] + + base_url = "http://{}:{}".format(args.host, port) + auth_url = "http://{}:{}@{}:{}".format(username, password, args.host, port) + + print("\npypiserver listening on:\n", flush=True) + print(" URL (no auth): {}".format(base_url), flush=True) + print(" URL (auth): {}".format(auth_url), flush=True) + print("\nRequired dependency in requirements.in:", flush=True) + print(" my-local-pkg==1.0.0", flush=True) + print("\nTo use from the uv_lock test workspace, run:", flush=True) + print(" cd tests/integration/uv_lock", flush=True) + print(" bazel run //:requirements.update \\", flush=True) + print(' --action_env=UV_EXTRA_INDEX_URL="{}" \\'.format(auth_url), flush=True) + print(" --action_env=UV_CREDENTIALS_DIR=", flush=True) + print("\nPress Ctrl+C to stop the server.\n", flush=True) + + try: + server.serve_forever() + except KeyboardInterrupt: + print("\nShutting down...", flush=True) + server.shutdown() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/integration/uv_lock_test.py b/tests/integration/uv_lock_test.py new file mode 100644 index 0000000000..8efd3cb84f --- /dev/null +++ b/tests/integration/uv_lock_test.py @@ -0,0 +1,288 @@ +import base64 +import hashlib +import os +import re +import threading +import time +import unittest +import uuid +from pathlib import Path +from urllib.error import URLError +from urllib.request import Request, urlopen +from wsgiref.simple_server import make_server + +from pypiserver import app_from_config, setup_routes_from_config +from pypiserver.config import Config + +from tests.integration import runner +from tests.integration.uv_lock_pypi_server import _create_wheel_bytes + + +def _make_server_on_free_port(app): + server = make_server("localhost", 0, app) + port = server.server_address[1] + return server, port + + +class UvLockIntegrationTest(runner.TestCase): + def setUp(self): + super().setUp() + + self.username = "testuser" + self.password = uuid.uuid4().hex + + self.dir = Path(os.environ["TEST_TMPDIR"]) + self.docroot = self.dir / "simple" + self.docroot.mkdir(exist_ok=True) + + self.wheel_data, self.wheel_sha256, wheel_name = _create_wheel_bytes( + "my-local-pkg", + "1.0.0", + ) + + packages_dir = self.docroot / "packages" + packages_dir.mkdir(exist_ok=True) + self.wheel_path = packages_dir / wheel_name + self.wheel_path.write_bytes(self.wheel_data) + + config = Config.default_with_overrides( + roots=[packages_dir], + port=0, + host="localhost", + authenticate=["download", "list", "update"], + password_file=None, + auther=lambda u, p: u == self.username and p == self.password, + disable_fallback=True, + fallback_url="", + server_method="wsgiref", + verbosity=0, + log_stream=None, + ) + app = app_from_config(config) + app = setup_routes_from_config(app, config) + + self._server, self.port = _make_server_on_free_port(app) + self.server_url = "http://localhost:{port}".format(port=self.port) + self.auth_url = "http://{user}:{passwd}@localhost:{port}".format( + user=self.username, + passwd=self.password, + port=self.port, + ) + + self._thread = threading.Thread(target=self._server.serve_forever) + self._thread.daemon = True + self._thread.start() + + interval = 0.1 + wait_seconds = 40 + for _ in range(int(wait_seconds / interval)): + try: + req = Request(self.server_url) + with urlopen(req, timeout=1) as response: + if response.status in (200, 401): + break + except (URLError, OSError): + pass + time.sleep(interval) + else: + raise RuntimeError( + "Could not start the server, waited for {}s".format(wait_seconds) + ) + + # Set a default value for UV_EXTRA_INDEX_URL in the bazel env so that + # the workspace .bazelrc `--action_env=UV_EXTRA_INDEX_URL` doesn't + # fail on Windows when the variable is unset in the client env. + self.bazel_env.setdefault("UV_EXTRA_INDEX_URL", "") + + # Use a sandbox-local credential store so credentials don't leak + # to the host system. + self.creds_dir = self.repo_root / ".uv-creds" + self.creds_dir.mkdir(parents=True, exist_ok=True) + self.bazel_env["UV_CREDENTIALS_DIR"] = str(self.creds_dir) + + # Log in to uv's credential store so `uv auth helper` can later + # serve the credentials to Bazel or uv itself. + self.run_bazel( + "run", + "//:uv", + "--", + "auth", + "login", + f"--username={self.username}", + f"--password={self.password}", + self.server_url, + ) + + def tearDown(self): + # Clear credentials from uv's credential store to ensure we are not + # logged into the service after the test. + self.run_bazel( + "run", + "//:uv", + "--", + "auth", + "logout", + self.server_url, + check=False, + ) + self._server.shutdown() + + def _assert_server_requires_auth(self): + req = Request(self.server_url + "/my-local-pkg/") + try: + urlopen(req, timeout=5) + self.fail("Expected 401 without auth") + except URLError: + pass + + def _auth_header(self): + return "Basic " + base64.b64encode( + "{user}:{passwd}".format( + user=self.username, + passwd=self.password, + ).encode("utf-8") + ).decode("utf-8") + + def _assert_simple_api_sha256(self): + auth_header = self._auth_header() + req = Request(self.server_url + "/simple/my-local-pkg/") + req.add_header("Authorization", auth_header) + resp = urlopen(req, timeout=5) + html = resp.read().decode("utf-8") + + match = re.search(r"#sha256=([a-f0-9]+)", html) + self.assertIsNotNone(match, "No sha256 found in simple API: {}".format(html)) + pypiserver_sha256 = match.group(1) + disk_sha256 = hashlib.sha256(self.wheel_path.read_bytes()).hexdigest() + self.assertEqual( + pypiserver_sha256, + disk_sha256, + "pypiserver hash {} != disk hash {}".format(pypiserver_sha256, disk_sha256), + ) + + def _creds_auth_args(self): + return [ + "--strategy=PyRequirementsLockUv=local", + "--action_env={key}={value}".format( + key="UV_CREDENTIALS_DIR", + value=str(self.creds_dir), + ), + "--action_env={key}={value}".format( + key="UV_EXTRA_INDEX_URL", + value=self.server_url, + ), + ] + + def _assert_lock_file(self, result): + self.assertEqual( + result.exit_code, + 0, + "Lock update failed:\n{}".format(result.describe()), + ) + lock_file = self.repo_root / "requirements.txt" + self.assertTrue(lock_file.exists(), "Lock file was not created") + contents = lock_file.read_text() + self.assertIn("my-local-pkg", contents) + self.assertIn("--hash=sha256:", contents) + + def test_lock_update_with_custom_index(self): + self._assert_server_requires_auth() + self._assert_simple_api_sha256() + + result = self.run_bazel( + "run", + "--action_env={key}={value}".format( + key="UV_EXTRA_INDEX_URL", + value=self.auth_url, + ), + "//:requirements.update", + ) + self._assert_lock_file(result) + + def test_update_with_credential_helper(self): + """Use a credential helper for authentication.""" + self._assert_server_requires_auth() + result = self.run_bazel( + "run", + *self._creds_auth_args(), + "//:requirements.update", + ) + self._assert_lock_file(result) + + def test_update_with_uv_auth_helper(self): + """Use the uv auth helper for authentication.""" + self._assert_server_requires_auth() + result = self.run_bazel( + "run", + *self._creds_auth_args(), + "//:requirements.update", + ) + self._assert_lock_file(result) + + def test_diff_test_with_requirements(self): + """Verify that ``diff_test`` can verify the generated lock file.""" + self._assert_server_requires_auth() + + # First generate the lock file + result = self.run_bazel( + "run", + *self._creds_auth_args(), + "//:requirements.update", + ) + self._assert_lock_file(result) + + # Copy the generated lock file to the expected location. The inner + # Bazel workspace is writable because it is a temporary copy created + # by the integration test framework. + generated = self.repo_root / "requirements.txt" + expected = self.repo_root / "requirements_expected.txt" + expected.write_text(generated.read_text()) + + # Run the diff_test: it builds the lock action, then compares the + # output to our expected file. + result = self.run_bazel( + "test", + *self._creds_auth_args(), + "//:requirements_diff_test", + ) + self.assertEqual( + result.exit_code, + 0, + "diff_test failed:\n{}".format(result.describe()), + ) + + def test_no_existing_requirements(self): + """Verify that ``bazel run`` and ``diff_test`` work when + ``requirements.txt`` does not yet exist.""" + self._assert_server_requires_auth() + + # Remove the existing lock file to simulate a fresh checkout + existing = self.repo_root / "requirements.txt" + existing.unlink() + self.assertFalse(existing.exists()) + + # Run ``requirements.update`` to generate the lock from scratch. The + # underlying lock rule will have no ``existing_output`` to copy, but + # ``uv pip compile`` should still produce the output. + result = self.run_bazel( + "run", + *self._creds_auth_args(), + "//:requirements.update", + ) + self._assert_lock_file(result) + + # diff_test should pass now that ``requirements.txt`` exists again + result = self.run_bazel( + "test", + *self._creds_auth_args(), + "//:requirements_diff_test", + ) + self.assertEqual( + result.exit_code, + 0, + "diff_test failed:\n{}".format(result.describe()), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/uv/lock/lock_run_test.py b/tests/uv/lock/lock_run_test.py index f64cbdccec..e2508161d5 100644 --- a/tests/uv/lock/lock_run_test.py +++ b/tests/uv/lock/lock_run_test.py @@ -1,3 +1,4 @@ +import os import subprocess import tempfile import unittest @@ -9,15 +10,59 @@ def _relative_rpath(path: str) -> Path: - p = (Path("_main") / "tests" / "uv" / "lock" / path).as_posix() - rpath = rfiles.Rlocation(p) - if not rpath: - raise ValueError(f"Could not find file: {p}") - - return Path(rpath) + """Find file in runfiles, handling Windows .bat/.exe wrappers.""" + # On Windows, try executable extensions first to avoid matching symlink + # entries in the runfiles manifest that point to non-executable files + # (e.g. a Python source file instead of the .exe launcher). + exts = (".exe", ".bat", "") if os.name == "nt" else ("", ".exe", ".bat") + for ext in exts: + p = (Path("_main") / "tests" / "uv" / "lock" / (path + ext)).as_posix() + rpath = rfiles.Rlocation(p) + if rpath: + rp = Path(rpath) + if rp.exists(): + return rp + + # Fallback: look in runfiles directory directly (handles .bat wrappers on + # Windows where Rlocation may return a runfiles link that doesn't exist) + runfiles_dir = os.environ.get("RUNFILES_DIR") + if runfiles_dir: + exts = (".exe", ".bat", "") if os.name == "nt" else ("", ".bat", ".exe") + for ext in exts: + rp = Path(runfiles_dir, "_main", "tests", "uv", "lock", path + ext) + if rp.exists(): + return rp + + raise ValueError(f"Could not find file in runfiles: {path}") + + +def _run_binary(path: Path, **kwargs): + """Run a binary, handling Windows .bat files.""" + if os.name == "nt": + return subprocess.run( + ["cmd.exe", "/c", str(path)], + **kwargs, + ) + return subprocess.run(path, **kwargs) class LockTests(unittest.TestCase): + def _subprocess_env(self, workspace_dir: Path) -> dict[str, str]: + env = { + "BUILD_WORKSPACE_DIRECTORY": str(workspace_dir), + } + # Inherit specific env vars needed for finding runfiles on Windows + for key in ( + "PATH", + "RUNFILES_DIR", + "RUNFILES_MANIFEST_FILE", + "SYSTEMROOT", + "PATHEXT", + ): + if key in os.environ: + env[key] = os.environ[key] + return env + def test_requirements_updating_for_the_first_time(self): # Given copier_path = _relative_rpath("requirements_new_file.update") @@ -30,19 +75,18 @@ def test_requirements_updating_for_the_first_time(self): self.assertFalse( want_path.exists(), "The path should not exist after the test" ) - output = subprocess.run( + output = _run_binary( copier_path, capture_output=True, - env={ - "BUILD_WORKSPACE_DIRECTORY": f"{workspace_dir}", - }, + env=self._subprocess_env(workspace_dir), ) # Then self.assertEqual(0, output.returncode, output.stderr) + stdout = output.stdout.decode("utf-8").replace("\\", "/") self.assertIn( "cp /tests/uv/lock/requirements_new_file", - output.stdout.decode("utf-8"), + stdout, ) self.assertTrue(want_path.exists(), "The path should exist after the test") self.assertNotEqual(want_path.read_text(), "") @@ -69,19 +113,18 @@ def test_requirements_updating(self): want_text + "\n\n" ) # Write something else to see that it is restored - output = subprocess.run( + output = _run_binary( copier_path, capture_output=True, - env={ - "BUILD_WORKSPACE_DIRECTORY": f"{workspace_dir}", - }, + env=self._subprocess_env(workspace_dir), ) # Then self.assertEqual(0, output.returncode) + stdout = output.stdout.decode("utf-8").replace("\\", "/") self.assertIn( "cp /tests/uv/lock/requirements", - output.stdout.decode("utf-8"), + stdout, ) self.assertEqual(want_path.read_text(), want_text) @@ -100,12 +143,10 @@ def test_requirements_run_on_the_first_time(self): self.assertFalse( want_path.exists(), "The path should not exist after the test" ) - output = subprocess.run( + output = _run_binary( copier_path, capture_output=True, - env={ - "BUILD_WORKSPACE_DIRECTORY": f"{workspace_dir}", - }, + env=self._subprocess_env(workspace_dir), ) # Then @@ -141,12 +182,10 @@ def test_requirements_run(self): want_text + "\n\n" ) # Write something else to see that it is restored - output = subprocess.run( + output = _run_binary( copier_path, capture_output=True, - env={ - "BUILD_WORKSPACE_DIRECTORY": f"{workspace_dir}", - }, + env=self._subprocess_env(workspace_dir), ) # Then diff --git a/tests/uv/lock/lock_tests.bzl b/tests/uv/lock/lock_tests.bzl index 1eb5b1d903..bcaed95b53 100644 --- a/tests/uv/lock/lock_tests.bzl +++ b/tests/uv/lock/lock_tests.bzl @@ -14,7 +14,7 @@ "" -load("@bazel_skylib//rules:native_binary.bzl", "native_test") +load("@bazel_skylib//rules:diff_test.bzl", "diff_test") load("//python/uv:lock.bzl", "lock") load("//tests/support:py_reconfig.bzl", "py_reconfig_test") @@ -77,23 +77,14 @@ def lock_test_suite(name): # `--index-url`. "no-remote-exec", ], - # FIXME @aignas 2025-03-19: It seems that currently: - # 1. The Windows runners are not compatible with the `uv` Windows binaries. - # 2. The Python launcher is having trouble launching scripts from within the Python test. - target_compatible_with = select({ - "@platforms//os:windows": ["@platforms//:incompatible"], - "//conditions:default": [], - }), ) - # document and check that this actually works - native_test( + # Document and check that the action output matches the in-source file. + diff_test( name = "requirements_test", - src = ":requirements.update", - target_compatible_with = select({ - "@platforms//os:windows": ["@platforms//:incompatible"], - "//conditions:default": [], - }), + timeout = "short", + file1 = ":requirements", + file2 = "testdata/requirements.txt", ) native.test_suite(