bazel-contrib · aignas · Jan 31, 2024 · Jan 8, 2024 · Jan 8, 2024 · Jan 16, 2024
@@ -27,6 +27,12 @@ A brief description of the categories of changes:
   This fixes issues due to pyc files being created at runtime and affecting the
   definition of what files were considered part of the runtime.
 
+* (pip_parse) Added the `envsubst` parameter, which enables environment variable
+  substitutions in the `extra_pip_args` attribute.
+
+* (pip_repository) Added the `envsubst` parameter, which enables environment
+  variable substitutions in the `extra_pip_args` attribute.
+
 ### Fixed
 
 * (bzlmod) pip.parse now does not fail with an empty `requirements.txt`.

@@ -18,8 +18,14 @@ load("@rules_python//python:pip.bzl", "pip_parse")
 
 # This repository isn't referenced, except by our test that asserts the requirements.bzl is updated.
 # It also wouldn't be needed by users of this ruleset.
+# If you're using envsubst with extra_pip_args, as we do below, the value of the environment
+# variables at the time we generate requirements.bzl don't make it into the file, as you may
+# verify by inspection; the environment variables at a later time, when we download the
+# packages, will be the ones that take effect.
 pip_parse(
     name = "pip",
+    envsubst = ["PIP_RETRIES"],
+    extra_pip_args = ["--retries=${PIP_RETRIES:-5}"],
     python_interpreter_target = "@python39_host//:python",
     requirements_lock = "//:requirements.txt",
 )

@@ -16,7 +16,7 @@ all_whl_requirements = all_whl_requirements_by_package.values()
 all_data_requirements = ["@pip//certifi:data", "@pip//charset_normalizer:data", "@pip//idna:data", "@pip//requests:data", "@pip//urllib3:data"]
 
 _packages = [("pip_certifi", "certifi==2023.7.22     --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082     --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"), ("pip_charset_normalizer", "charset-normalizer==2.1.1     --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845     --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"), ("pip_idna", "idna==3.4     --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4     --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"), ("pip_requests", "requests==2.28.1     --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983     --hash=sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"), ("pip_urllib3", "urllib3==1.26.13     --hash=sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc     --hash=sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8")]
-_config = {"download_only": False, "enable_implicit_namespace_pkgs": False, "environment": {}, "extra_pip_args": [], "isolated": True, "pip_data_exclude": [], "python_interpreter": "python3", "python_interpreter_target": "@python39_host//:python", "quiet": True, "repo": "pip", "repo_prefix": "pip_", "timeout": 600}
+_config = {"download_only": False, "enable_implicit_namespace_pkgs": False, "environment": {}, "envsubst": ["PIP_RETRIES"], "extra_pip_args": ["--retries=${PIP_RETRIES:-5}"], "isolated": True, "pip_data_exclude": [], "python_interpreter": "python3", "python_interpreter_target": "@python39_host//:python", "quiet": True, "repo": "pip", "repo_prefix": "pip_", "timeout": 600}
 _annotations = {}
 
 def requirement(name):

@@ -30,6 +30,7 @@ bzl_library(
         "//python/pip_install/private:generate_whl_library_build_bazel_bzl",
         "//python/pip_install/private:srcs_bzl",
         "//python/private:bzlmod_enabled_bzl",
+        "//python/private:envsubst_bzl",
         "//python/private:normalize_name_bzl",
         "//python/private:parse_whl_name_bzl",
         "//python/private:patch_whl_bzl",

@@ -22,6 +22,7 @@ load("//python/pip_install:requirements_parser.bzl", parse_requirements = "parse
 load("//python/pip_install/private:generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel")
 load("//python/pip_install/private:generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel")
 load("//python/pip_install/private:srcs.bzl", "PIP_INSTALL_PY_SRCS")
+load("//python/private:envsubst.bzl", "envsubst")
 load("//python/private:normalize_name.bzl", "normalize_name")
 load("//python/private:parse_whl_name.bzl", "parse_whl_name")
 load("//python/private:patch_whl.bzl", "patch_whl")
@@ -195,12 +196,24 @@ def _parse_optional_attrs(rctx, args):
     if use_isolated(rctx, rctx.attr):
         args.append("--isolated")
 
+    # At the time of writing, the very latest Bazel, as in `USE_BAZEL_VERSION=last_green bazelisk`
+    # supports rctx.getenv(name, default): When building incrementally, any change to the value of
+    # the variable named by name will cause this repository to be re-fetched. That hasn't yet made
+    # its way into the official releases, though.
+    if "getenv" in dir(rctx):
+        getenv = rctx.getenv
+    else:
+        getenv = rctx.os.environ.get
+
     # Check for None so we use empty default types from our attrs.
     # Some args want to be list, and some want to be dict.
     if rctx.attr.extra_pip_args != None:
         args += [
             "--extra_pip_args",
-            json.encode(struct(arg = rctx.attr.extra_pip_args)),
+            json.encode(struct(arg = [
+                envsubst(pip_arg, rctx.attr.envsubst, getenv)
+                for pip_arg in rctx.attr.extra_pip_args
+            ])),
         ]
 
     if rctx.attr.download_only:
@@ -338,6 +351,7 @@ def _pip_repository_impl(rctx):
         "download_only": rctx.attr.download_only,
         "enable_implicit_namespace_pkgs": rctx.attr.enable_implicit_namespace_pkgs,
         "environment": rctx.attr.environment,
+        "envsubst": rctx.attr.envsubst,
         "extra_pip_args": options,
         "isolated": use_isolated(rctx, rctx.attr),
         "pip_data_exclude": rctx.attr.pip_data_exclude,
@@ -418,10 +432,23 @@ Environment variables to set in the pip subprocess.
 Can be used to set common variables such as `http_proxy`, `https_proxy` and `no_proxy`
 Note that pip is run with "--isolated" on the CLI so `PIP_<VAR>_<NAME>`
 style env vars are ignored, but env vars that control requests and urllib3
-can be passed.
+can be passed. If you need `PIP_<VAR>_<NAME>`, take a look at `extra_pip_args`
+and `envsubst`.
         """,
         default = {},
     ),
+    "envsubst": attr.string_list(
+        mandatory = False,
+        doc = """\
+A list of environment variables to substitute (e.g. `["PIP_INDEX_URL",
+"PIP_RETRIES"]`). The corresponding variables are expanded in `extra_pip_args`
+using the syntax `$VARNAME` or `${VARNAME}` (expanding to empty string if unset)
+or `${VARNAME:-default}` (expanding to default if the variable is unset or empty
+in the environment). Note: On Bazel 6 and Bazel 7 changes to the variables named
+here do not cause packages to be re-fetched. Don't fetch different things based
+on the value of these variables.
+""",
+    ),
     "experimental_requirement_cycles": attr.string_list_dict(
         default = {},
         doc = """\
@@ -509,7 +536,14 @@ NOTE: this is not for cross-compiling Python wheels but rather for parsing the `
 """,
     ),
     "extra_pip_args": attr.string_list(
-        doc = "Extra arguments to pass on to pip. Must not contain spaces.",
+        doc = """Extra arguments to pass on to pip. Must not contain spaces.
+
+Supports environment variables using the syntax `$VARNAME` or
+`${VARNAME}` (expanding to empty string if unset) or
+`${VARNAME:-default}` (expanding to default if the variable is unset
+or empty in the environment), if `"VARNAME"` is listed in the
+`envsubst` attribute. See also `envsubst`.
+""",
     ),
     "isolated": attr.bool(
         doc = """\

@@ -77,6 +77,11 @@ bzl_library(
     ],
 )
 
+bzl_library(
+    name = "envsubst_bzl",
+    srcs = ["envsubst.bzl"],
+)
+
 bzl_library(
     name = "full_version_bzl",
     srcs = ["full_version.bzl"],

@@ -200,6 +200,7 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides):
             pip_data_exclude = pip_attr.pip_data_exclude,
             enable_implicit_namespace_pkgs = pip_attr.enable_implicit_namespace_pkgs,
             environment = pip_attr.environment,
+            envsubst = pip_attr.envsubst,
             group_name = group_name,
             group_deps = group_deps,
         )

@@ -0,0 +1,65 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Substitute environment variables in shell format strings."""
+
+def envsubst(template_string, varnames, getenv):
+    """Helper function to substitute environment variables.
+
+    Supports `$VARNAME`, `${VARNAME}` and `${VARNAME:-default}`
+    syntaxes in the `template_string`, looking up each `VARNAME`
+    listed in the `varnames` list in the environment defined by the
+    `getenv` function. Typically called with `getenv = rctx.getenv`
+    (if it is available) or `getenv = rctx.os.environ.get` (on e.g.
+    Bazel 6 or Bazel 7, which don't have `rctx.getenv` yet).
+
+    Limitations: Unlike the shell, we don't support `${VARNAME}` and
+    `${VARNAME:-default}` in the default expression for a different
+    environment variable expansion. We do support the braceless syntax
+    in the default, so an expression such as `${HOME:-/home/$USER}` is
+    valid.
+
+    Args:
+      template_string: String that may contain variables to be expanded.
+      varnames: List of variable names of variables to expand in
+        `template_string`.
+      getenv: Callable mapping variable names (in the first argument)
+        to their values, or returns the default (provided in the
+        second argument to `getenv`) if a value wasn't found.
+
+    Returns:
+      `template_string` with environment variables expanded according
+      to their values as determined by `getenv`.
+    """
+
+    if not varnames:
+        return template_string
+
+    for varname in varnames:
+        value = getenv(varname, "")
+        template_string = template_string.replace("$%s" % varname, value)
+        template_string = template_string.replace("${%s}" % varname, value)
+        segments = template_string.split("${%s:-" % varname)
+        template_string = segments.pop(0)
+        for segment in segments:
+            default_value, separator, rest = segment.partition("}")
+            if "{" in default_value:
+                fail("Environment substitution expression " +
+                     "\"${%s:-\" has an opening \"{\" " % varname +
+                     "in default value \"%s\"." % default_value)
+            if not separator:
+                fail("Environment substitution expression " +
+                     "\"${%s:-\" is missing the final \"}\"" % varname)
+            template_string += (value if value else default_value) + rest
+    return template_string
@@ -0,0 +1,19 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for envsubsts."""
+
+load(":envsubst_tests.bzl", "envsubst_test_suite")
+
+envsubst_test_suite(name = "envsubst_tests")
@@ -0,0 +1,126 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Test for py_wheel."""
+
+load("@rules_testing//lib:analysis_test.bzl", "test_suite")
+load("//python/private:envsubst.bzl", "envsubst")  # buildifier: disable=bzl-visibility
+
+_basic_tests = []
+
+def _test_envsubst_braceless(env):
+    env.expect.that_str(
+        envsubst("--retries=$PIP_RETRIES", ["PIP_RETRIES"], {"PIP_RETRIES": "5"}.get),
+    ).equals("--retries=5")
+
+    env.expect.that_str(
+        envsubst("--retries=$PIP_RETRIES", [], {"PIP_RETRIES": "5"}.get),
+    ).equals("--retries=$PIP_RETRIES")
+
+    env.expect.that_str(
+        envsubst("--retries=$PIP_RETRIES", ["PIP_RETRIES"], {}.get),
+    ).equals("--retries=")
+
+_basic_tests.append(_test_envsubst_braceless)
+
+def _test_envsubst_braces_without_default(env):
+    env.expect.that_str(
+        envsubst("--retries=${PIP_RETRIES}", ["PIP_RETRIES"], {"PIP_RETRIES": "5"}.get),
+    ).equals("--retries=5")
+
+    env.expect.that_str(
+        envsubst("--retries=${PIP_RETRIES}", [], {"PIP_RETRIES": "5"}.get),
+    ).equals("--retries=${PIP_RETRIES}")
+
+    env.expect.that_str(
+        envsubst("--retries=${PIP_RETRIES}", ["PIP_RETRIES"], {}.get),
+    ).equals("--retries=")
+
+_basic_tests.append(_test_envsubst_braces_without_default)
+
+def _test_envsubst_braces_with_default(env):
+    env.expect.that_str(
+        envsubst("--retries=${PIP_RETRIES:-6}", ["PIP_RETRIES"], {"PIP_RETRIES": "5"}.get),
+    ).equals("--retries=5")
+
+    env.expect.that_str(
+        envsubst("--retries=${PIP_RETRIES:-6}", [], {"PIP_RETRIES": "5"}.get),
+    ).equals("--retries=${PIP_RETRIES:-6}")
+
+    env.expect.that_str(
+        envsubst("--retries=${PIP_RETRIES:-6}", ["PIP_RETRIES"], {}.get),
+    ).equals("--retries=6")
+
+_basic_tests.append(_test_envsubst_braces_with_default)
+
+def _test_envsubst_nested_both_vars(env):
+    env.expect.that_str(
+        envsubst(
+            "${HOME:-/home/$USER}",
+            ["HOME", "USER"],
+            {"HOME": "/home/testuser", "USER": "mockuser"}.get,
+        ),
+    ).equals("/home/testuser")
+
+_basic_tests.append(_test_envsubst_nested_both_vars)
+
+def _test_envsubst_nested_outer_var(env):
+    env.expect.that_str(
+        envsubst(
+            "${HOME:-/home/$USER}",
+            ["HOME"],
+            {"HOME": "/home/testuser", "USER": "mockuser"}.get,
+        ),
+    ).equals("/home/testuser")
+
+_basic_tests.append(_test_envsubst_nested_outer_var)
+
+def _test_envsubst_nested_no_vars(env):
+    env.expect.that_str(
+        envsubst(
+            "${HOME:-/home/$USER}",
+            [],
+            {"HOME": "/home/testuser", "USER": "mockuser"}.get,
+        ),
+    ).equals("${HOME:-/home/$USER}")
+
+    env.expect.that_str(
+        envsubst("${HOME:-/home/$USER}", ["HOME", "USER"], {}.get),
+    ).equals("/home/")
+
+_basic_tests.append(_test_envsubst_nested_no_vars)
+
+def _test_envsubst_nested_braces_inner_var(env):
+    env.expect.that_str(
+        envsubst(
+            "Home directory is ${HOME:-/home/$USER}.",
+            ["HOME", "USER"],
+            {"USER": "mockuser"}.get,
+        ),
+    ).equals("Home directory is /home/mockuser.")
+
+    env.expect.that_str(
+        envsubst(
+            "Home directory is ${HOME:-/home/$USER}.",
+            ["USER"],
+            {"USER": "mockuser"}.get,
+        ),
+    ).equals("Home directory is ${HOME:-/home/mockuser}.")
+
+_basic_tests.append(_test_envsubst_nested_braces_inner_var)
+
+def envsubst_test_suite(name):
+    test_suite(
+        name = name,
+        basic_tests = _basic_tests,
+    )