feat: wheel publishing

alexeagle · alexeagle · commit eb55280a0e24 · 2023-01-25T17:41:40.000-08:00
Tested manually with: ``` $ bazel run --stamp --embed_label=0.17.0 //python/runfiles:wheel.publish -- --repository testpypi ``` That result is here: https://test.pypi.org/project/bazel-runfiles/0.17.0/ Note, I'd also like to add this to the examples/wheel, see #1017 for a pre-requisite.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -14,16 +14,14 @@ jobs:
         uses: actions/checkout@v2
       - name: Prepare workspace snippet
         run: .github/workflows/workspace_snippet.sh > release_notes.txt
-      - name: Build wheel dist
-        run: bazel build --stamp --embed_label=${{ env.GITHUB_REF_NAME }} //python/runfiles:wheel
-      - name: Publish runfiles package to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
+      - name: Publish wheel dist
+        env:
+          # This special value tells pypi that the user identity is supplied within the token
+          TWINE_USERNAME: __token__
           # Note, the PYPI_API_TOKEN was added on
           # https://github.com/bazelbuild/rules_python/settings/secrets/actions 
-          # and currently uses a token which authenticates as https://pypi.org/user/alexeagle/
-          password: ${{ secrets.PYPI_API_TOKEN }}
-          packages_dir: bazel-bin/python/runfiles
+          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+        run: bazel run --stamp --embed_label=${{ env.GITHUB_REF_NAME }} //python/runfiles:wheel.publish
       - name: Release
         uses: softprops/action-gh-release@v1
         with:
diff --git a/WORKSPACE b/WORKSPACE
@@ -67,3 +67,21 @@ load("@rules_python_gazelle_plugin//:deps.bzl", _py_gazelle_deps = "gazelle_deps
 # This rule loads and compiles various go dependencies that running gazelle
 # for python requirements.
 _py_gazelle_deps()
+
+load("@python//3.11.1:defs.bzl", "interpreter")
+
+#####################
+# Install twine for our own runfiles wheel publishing
+# Eventually we might want to install twine automatically for users too, see:
+# See https://github.com/bazelbuild/rules_python/issues/1016
+load("@rules_python//python:pip.bzl", "pip_parse")
+
+pip_parse(
+    name = "publish_deps",
+    python_interpreter_target = interpreter,
+    requirements_lock = "//python/runfiles:requirements.txt",
+)
+
+load("@publish_deps//:requirements.bzl", "install_deps")
+
+install_deps()
diff --git a/python/packaging.bzl b/python/packaging.bzl
@@ -15,6 +15,7 @@
 """Public API for for building wheels."""
 
 load("//python/private:py_package.bzl", "py_package_lib")
+load("//python/private:py_twine.bzl", "py_twine_lib")
 load("//python/private:py_wheel.bzl", _PyWheelInfo = "PyWheelInfo", _py_wheel = "py_wheel")
 
 # Re-export as public API
@@ -80,15 +81,22 @@ def py_wheel(name, **kwargs):
         name:  A unique name for this target.
         **kwargs: other named parameters passed to the underlying [py_wheel rule](#py_wheel_rule)
     """
-    _py_wheel(name = name, **kwargs)
+    py_twine(
+        name = "{}.publish".format(name),
+        wheel = name,
+        twine_bin = kwargs.pop("twine_bin", None),
+    )
 
-    # TODO(alexeagle): produce an executable target like this:
-    # py_publish_wheel(
-    #     name = "{}.publish".format(name),
-    #     wheel = name,
-    #     # Optional: override the label for a py_binary that runs twine
-    #     # https://twine.readthedocs.io/en/stable/
-    #     twine_bin = "//path/to:twine",
-    # )
+    _py_wheel(name = name, **kwargs)
 
 py_wheel_rule = _py_wheel
+
+py_twine = rule(
+    doc = """\
+The py_twine rule executes the twine CLI to upload packages.
+https://packaging.python.org/en/latest/tutorials/packaging-projects/#uploading-the-distribution-archives
+""",
+    implementation = py_twine_lib.implementation,
+    attrs = py_twine_lib.attrs,
+    executable = True,
+)
diff --git a/python/private/py_twine.bzl b/python/private/py_twine.bzl
@@ -0,0 +1,106 @@
+"""Implementation of py_twine rule.
+
+Simply wraps the tool with a bash script and a runfiles manifest.
+See https://twine.readthedocs.io/
+"""
+
+load(":py_wheel.bzl", "PyWheelInfo")
+
+_attrs = {
+    "twine_bin": attr.label(
+        doc = """\
+A py_binary that runs the twine tool.
+The default value assumes you have `twine` listed in your own requirements.txt, and have run
+`pip_parse` with the default name of `pypi`.
+
+If these don't apply, you might use the `entry_point` helper to supply your own twine binary:
+```starlark
+load("@my_pip_parse_name//:requirements.bzl", "entry_point")
+py_twine(
+    ...
+    twine_bin = entry_point("twine"),
+)
+```
+
+Or of course you can supply a py_binary by some other means which is CLI-compatible with twine.
+
+Currently rules_python doesn't supply twine itself.
+Follow https://github.com/bazelbuild/rules_python/issues/1016
+""",
+        default = "@pypi_twine//:rules_python_wheel_entry_point_twine",
+        executable = True,
+        cfg = "exec",
+    ),
+    "wheel": attr.label(providers = [PyWheelInfo]),
+    "_runfiles_lib": attr.label(default = "@bazel_tools//tools/bash/runfiles"),
+}
+
+# Bash helper function for looking up runfiles.
+# Vendored from
+# https://github.com/bazelbuild/bazel/blob/master/tools/bash/runfiles/runfiles.bash
+BASH_RLOCATION_FUNCTION = r"""
+# --- begin runfiles.bash initialization v2 ---
+set -uo pipefail; f=bazel_tools/tools/bash/runfiles/runfiles.bash
+source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \
+source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \
+source "$0.runfiles/$f" 2>/dev/null || \
+source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \
+source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \
+{ echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e
+# --- end runfiles.bash initialization v2 ---
+"""
+
+# Copied from https://github.com/aspect-build/bazel-lib/blob/main/lib/private/paths.bzl
+# to avoid a dependency from bazelbuild -> aspect-build
+def _to_manifest_path(ctx, file):
+    """The runfiles manifest entry path for a file.
+
+    This is the full runfiles path of a file including its workspace name as
+    the first segment. We refert to it as the manifest path as it is the path
+    flavor that is used for in the runfiles MANIFEST file.
+    We must avoid using non-normalized paths (workspace/../other_workspace/path)
+    in order to locate entries by their key.
+    Args:
+        ctx: starlark rule execution context
+        file: a File object
+    Returns:
+        The runfiles manifest entry path for a file
+    """
+
+    if file.short_path.startswith("../"):
+        return file.short_path[3:]
+    else:
+        return ctx.workspace_name + "/" + file.short_path
+
+_exec_tmpl = """\
+#!/usr/bin/env bash
+{rlocation}
+tmp=$(mktemp -d)
+# The namefile is just a file with one line, containing the real filename for the wheel.
+wheel_filename=$tmp/$(cat "$(rlocation {wheel_namefile})")
+cp $(rlocation {wheel}) $wheel_filename
+$(rlocation {twine_bin}) upload $wheel_filename "$@"
+"""
+
+def _implementation(ctx):
+    exec = ctx.actions.declare_file(ctx.label.name + ".sh")
+
+    ctx.actions.write(exec, content = _exec_tmpl.format(
+        rlocation = BASH_RLOCATION_FUNCTION,
+        twine_bin = _to_manifest_path(ctx, ctx.executable.twine_bin),
+        wheel = _to_manifest_path(ctx, ctx.files.wheel[0]),
+        wheel_namefile = _to_manifest_path(ctx, ctx.attr.wheel[PyWheelInfo].name_file),
+    ), is_executable = True)
+
+    runfiles = ctx.runfiles(ctx.files.twine_bin + ctx.files.wheel + ctx.files._runfiles_lib + [
+        ctx.attr.wheel[PyWheelInfo].name_file,
+    ])
+    runfiles = runfiles.merge(ctx.attr.twine_bin[DefaultInfo].default_runfiles)
+    return [
+        DefaultInfo(executable = exec, runfiles = runfiles),
+    ]
+
+py_twine_lib = struct(
+    implementation = _implementation,
+    attrs = _attrs,
+)
diff --git a/python/runfiles/BUILD.bazel b/python/runfiles/BUILD.bazel
@@ -14,6 +14,11 @@
 
 load("//python:defs.bzl", "py_library")
 load("//python:packaging.bzl", "py_wheel")
+load("//python:pip.bzl", "compile_pip_requirements")
+
+compile_pip_requirements(
+    name = "requirements",
+)
 
 filegroup(
     name = "distribution",
@@ -40,10 +45,12 @@ py_wheel(
         "Development Status :: 5 - Production/Stable",
         "License :: OSI Approved :: Apache Software License",
     ],
-    description_file = "README.md",
+    description_file = "README.rst",
     distribution = "bazel_runfiles",
     homepage = "https://github.com/bazelbuild/rules_python",
     strip_path_prefixes = ["python"],
+    # This attribute is for the "wheel.publish" target.
+    twine_bin = "@publish_deps_twine//:rules_python_wheel_entry_point_twine",
     version = "{BUILD_EMBED_LABEL}",
     visibility = ["//visibility:public"],
     deps = [":runfiles"],
diff --git a/python/runfiles/README.rst b/python/runfiles/README.rst
@@ -1,26 +1,27 @@
-# bazel-runfiles library
+bazel-runfiles library
+======================
 
 This is a Bazel Runfiles lookup library for Bazel-built Python binaries and tests.
 
 Typical Usage
 -------------
 
 1.  Add the 'runfiles' dependency along with other third-party dependencies, for example in your
-    `requirements.txt` file.
+    ``requirements.txt`` file.
 
-2.  Depend on this runfiles library from your build rule, like you would other third-party libraries.
+2.  Depend on this runfiles library from your build rule, like you would other third-party libraries::
 
       py_binary(
           name = "my_binary",
           ...
           deps = [requirement("runfiles")],
       )
 
-3.  Import the runfiles library.
+3.  Import the runfiles library::
 
       import runfiles  # not "from runfiles import runfiles"
 
-4.  Create a Runfiles object and use rlocation to look up runfile paths:
+4.  Create a Runfiles object and use rlocation to look up runfile paths::
 
       r = runfiles.Create()
       ...
@@ -32,15 +33,15 @@ Typical Usage
     on the environment variables in os.environ. See `Create()` for more info.
 
     If you want to explicitly create a manifest- or directory-based
-    implementations, you can do so as follows:
+    implementations, you can do so as follows::
 
       r1 = runfiles.CreateManifestBased("path/to/foo.runfiles_manifest")
 
       r2 = runfiles.CreateDirectoryBased("path/to/foo.runfiles/")
 
-    If you wnat to start subprocesses, and the subprocess can't automatically
+    If you want to start subprocesses, and the subprocess can't automatically
     find the correct runfiles directory, you can explicitly set the right
-    environment variables for them:
+    environment variables for them::
 
       import subprocess
       import runfiles
diff --git a/python/runfiles/requirements.in b/python/runfiles/requirements.in
@@ -0,0 +1 @@
+twine
diff --git a/python/runfiles/requirements.txt b/python/runfiles/requirements.txt
