Add support for relative requirements in pip_install (bazel-contrib#433)

aaliddell · alexeagle · web-flow · commit aee9704a654a · 2021-10-25T19:04:34.000-07:00
* Run pip within the directory containing the requirements.txt file

This allows for relative requirements to be resolved in the way
that standalone pip would without patching or parsing the
requirements file manually

* Print test errors of sub-workspaces in bazel_integration_test

Prior to this, errors would be written to logs in the temp
directory that would promptly be deleted on parent test
teardown. This means at least the test logs will be printed
on failure

* Add example of relative requirement as test

* Explicitly cast Path to str to support Python 3.5

* Fix multiple declarations of pip_args

* Apply buildifier fix

Co-authored-by: Alex Eagle &lt;eagle@post.harvard.edu&gt;
diff --git a/.bazelrc b/.bazelrc
@@ -3,7 +3,7 @@
 # This lets us glob() up all the files inside the examples to make them inputs to tests
 # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
 # To update these lines, run tools/bazel_integration_test/update_deleted_packages.sh
-build --deleted_packages=examples/legacy_pip_import/boto,examples/legacy_pip_import/extras,examples/legacy_pip_import/helloworld,examples/pip_install,examples/pip_parse,examples/py_import
-query --deleted_packages=examples/legacy_pip_import/boto,examples/legacy_pip_import/extras,examples/legacy_pip_import/helloworld,examples/pip_install,examples/pip_parse,examples/py_import
+build --deleted_packages=examples/legacy_pip_import/boto,examples/legacy_pip_import/extras,examples/legacy_pip_import/helloworld,examples/pip_install,examples/pip_parse,examples/py_import,examples/relative_requirements
+query --deleted_packages=examples/legacy_pip_import/boto,examples/legacy_pip_import/extras,examples/legacy_pip_import/helloworld,examples/pip_install,examples/pip_parse,examples/py_import,examples/relative_requirements
 
 test --test_output=errors
diff --git a/examples/BUILD b/examples/BUILD
@@ -36,3 +36,8 @@ bazel_integration_test(
     name = "py_import_example",
     timeout = "long",
 )
+
+bazel_integration_test(
+    name = "relative_requirements_example",
+    timeout = "long",
+)
diff --git a/examples/relative_requirements/BUILD b/examples/relative_requirements/BUILD
@@ -0,0 +1,10 @@
+load("@pip//:requirements.bzl", "requirement")
+load("@rules_python//python:defs.bzl", "py_test")
+
+py_test(
+    name = "main",
+    srcs = ["main.py"],
+    deps = [
+        requirement("relative_package_name"),
+    ],
+)
diff --git a/examples/relative_requirements/README.md b/examples/relative_requirements/README.md
@@ -0,0 +1,4 @@
+# relative_requirements example
+
+This example shows how to use pip to fetch relative dependencies from a requirements.txt file,
+then use them in BUILD files as dependencies of Bazel targets.
diff --git a/examples/relative_requirements/WORKSPACE b/examples/relative_requirements/WORKSPACE
@@ -0,0 +1,15 @@
+workspace(name = "example_repo")
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+http_archive(
+    name = "rules_python",
+    sha256 = "b6d46438523a3ec0f3cead544190ee13223a52f6a6765a29eae7b7cc24cc83a0",
+    url = "https://github.com/bazelbuild/rules_python/releases/download/0.1.0/rules_python-0.1.0.tar.gz",
+)
+
+load("@rules_python//python:pip.bzl", "pip_install")
+
+pip_install(
+    requirements = "//:requirements.txt",
+)
diff --git a/examples/relative_requirements/main.py b/examples/relative_requirements/main.py
@@ -0,0 +1,5 @@
+import relative_package_name
+
+if __name__ == "__main__":
+    # Run a function from the relative package
+    print(relative_package_name.test())
diff --git a/examples/relative_requirements/relative_package/relative_package_name/__init__.py b/examples/relative_requirements/relative_package/relative_package_name/__init__.py
@@ -0,0 +1,2 @@
+def test():
+    return True
diff --git a/examples/relative_requirements/relative_package/setup.py b/examples/relative_requirements/relative_package/setup.py
@@ -0,0 +1,7 @@
+from setuptools import setup
+
+setup(
+    name='relative_package_name',
+    version='1.0.0',
+    packages=['relative_package_name'],
+)
diff --git a/examples/relative_requirements/requirements.txt b/examples/relative_requirements/requirements.txt
@@ -0,0 +1 @@
+./relative_package
diff --git a/python/pip_install/extract_wheels/__init__.py b/python/pip_install/extract_wheels/__init__.py
@@ -8,6 +8,7 @@
 import argparse
 import glob
 import os
+import pathlib
 import subprocess
 import sys
 import json
@@ -63,17 +64,22 @@ def main() -> None:
     deserialized_args = dict(vars(args))
     arguments.deserialize_structured_args(deserialized_args)
 
+    # Pip is run with the working directory changed to the folder containing the requirements.txt file, to allow for
+    # relative requirements to be correctly resolved. The --wheel-dir is therefore required to be repointed back to the
+    # current calling working directory (the repo root in .../external/name), where the wheel files should be written to
     pip_args = (
         [sys.executable, "-m", "pip"] + 
         (["--isolated"] if args.isolated else []) + 
         ["wheel", "-r", args.requirements] +
+        ["--wheel-dir", os.getcwd()] +
         deserialized_args["extra_pip_args"]
     )
 
     env = os.environ.copy()
     env.update(deserialized_args["environment"])
+
     # Assumes any errors are logged by pip so do nothing. This command will fail if pip fails
-    subprocess.run(pip_args, check=True, env=env)
+    subprocess.run(pip_args, check=True, env=env, cwd=str(pathlib.Path(args.requirements).parent.resolve()))
 
     extras = requirements.parse_extras(args.requirements)
 

Original file line number	Diff line number	Diff line change
`@@ -36,3 +36,8 @@ bazel_integration_test(`
`36`	`36`	`name = "py_import_example",`
`37`	`37`	`timeout = "long",`
`38`	`38`	`)`
	`39`	`+`
	`40`	`+bazel_integration_test(`
	`41`	`+ name = "relative_requirements_example",`
	`42`	`+ timeout = "long",`
	`43`	`+)`