From 2fb6c07f211430665fefd15be47eab9a930012cf Mon Sep 17 00:00:00 2001 From: opbro Date: Thu, 28 May 2020 20:41:40 -0400 Subject: [PATCH 1/5] Added new rule for putting package files path into a single file to avoid long commands --- experimental/examples/wheel/BUILD | 18 +++++++++- experimental/python/wheel.bzl | 45 +++++++++++++++++++++++-- experimental/rules_python/wheelmaker.py | 12 +++++++ 3 files changed, 71 insertions(+), 4 deletions(-) diff --git a/experimental/examples/wheel/BUILD b/experimental/examples/wheel/BUILD index 8916423907..809d8c49ee 100644 --- a/experimental/examples/wheel/BUILD +++ b/experimental/examples/wheel/BUILD @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -load("//experimental/python:wheel.bzl", "py_package", "py_wheel") +load("//experimental/python:wheel.bzl", "py_package", "py_wheel", "py_wrapped_input_files") load("//python:defs.bzl", "py_library", "py_test") package(default_visibility = ["//visibility:public"]) @@ -53,6 +53,11 @@ py_package( deps = [":main"], ) +py_wrapped_input_files( + name = "wrapped_example_package", + deps = [":example_pkg"], +) + py_wheel( name = "minimal_with_py_package", # Package data. We're building "example_minimal_package-0.0.1-py3-none-any.whl" @@ -62,6 +67,17 @@ py_wheel( deps = [":example_pkg"], ) +py_wheel( + name = "minimal_with_py_package_wrapped", + # Package data. We're building "example_minimal_package-0.0.1-py3-none-any.whl" + distribution = "example_minimal_package", + python_tag = "py3", + version = "0.0.1", + wrapped_package_lists = [":wrapped_example_package"], + deps = [":example_pkg", ":wrapped_example_package"], +) + + # An example that uses all features provided by py_wheel. py_wheel( name = "customized", diff --git a/experimental/python/wheel.bzl b/experimental/python/wheel.bzl index 9cd6534e78..df9a1a7b9f 100644 --- a/experimental/python/wheel.bzl +++ b/experimental/python/wheel.bzl @@ -39,7 +39,6 @@ def _py_package_impl(ctx): transitive = [dep[DefaultInfo].data_runfiles.files for dep in ctx.attr.deps] + [dep[DefaultInfo].default_runfiles.files for dep in ctx.attr.deps], ) - # TODO: '/' is wrong on windows, but the path separator is not available in skylark. # Fix this once ctx.configuration has directory separator information. packages = [p.replace(".", "/") for p in ctx.attr.packages] @@ -54,8 +53,9 @@ def _py_package_impl(ctx): for package in packages: if wheel_path.startswith(package): filtered_files.append(input_file) + filtered_inputs = depset(direct = filtered_files) - + return [DefaultInfo( files = filtered_inputs, )] @@ -81,6 +81,37 @@ Sub-packages are automatically included. }, ) +def _py_wrapped_input_files_impl(ctx): + inputs_to_package = depset( + direct = ctx.files.deps + ) + packageinputfile = ctx.actions.declare_file(ctx.attr.name + '_target_wrapped_inputs.txt') + content = '' + for input_file in inputs_to_package.to_list(): + content += _input_file_to_arg(input_file) + '\n' + ctx.actions.write(output = packageinputfile, content=content) + + return [ + DefaultInfo( + files = depset(direct = [packageinputfile]) + ) + ] + + + +py_wrapped_input_files = rule( + implementation = _py_wrapped_input_files_impl, + doc = """ + This was created to wrap the input files from py_package and use as in input to py_wheel, + by setting the attribute wrapped_input_files to avoid long command lines for systems that cannot + handle them. + """, + attrs = { + "deps": attr.label_list() + }, +) + + def _py_wheel_impl(ctx): outfile = ctx.actions.declare_file("-".join([ ctx.attr.distribution, @@ -107,7 +138,14 @@ def _py_wheel_impl(ctx): args.add("--out", outfile.path) args.add_all(ctx.attr.strip_path_prefixes, format_each = "--strip_path_prefix=%s") - args.add_all(inputs_to_package, format_each = "--input_file=%s", map_each = _input_file_to_arg) + if ctx.attr.wrapped_package_lists: + wrapped_inputs = depset( + direct = ctx.files.wrapped_package_lists + ) + print(wrapped_inputs.to_list()) + args.add_all(wrapped_inputs, format_each = "--input_file_list=%s") + else: + args.add_all(inputs_to_package, format_each = "--input_file=%s", map_each = _input_file_to_arg) extra_headers = [] if ctx.attr.author: @@ -216,6 +254,7 @@ _other_attrs = { default = [], doc = "path prefixes to strip from files added to the generated package", ), + "wrapped_package_lists": attr.label_list(), } py_wheel = rule( diff --git a/experimental/rules_python/wheelmaker.py b/experimental/rules_python/wheelmaker.py index 6be5d3847f..1b3261d018 100644 --- a/experimental/rules_python/wheelmaker.py +++ b/experimental/rules_python/wheelmaker.py @@ -242,6 +242,10 @@ def main(): help="'package_path;real_path' pairs listing " "files to be included in the wheel. " "Can be supplied multiple times.") + contents_group.add_argument( + '--input_file_list', action='append', + help='A file that has all the input files defined as a list to avoid the long command' + ) contents_group.add_argument( '--console_script', action='append', help="Defines a 'console_script' entry point. " @@ -264,6 +268,14 @@ def main(): input_files = [i.split(';') for i in arguments.input_file] else: input_files = [] + + if arguments.input_file_list: + for input_file in arguments.input_file_list: + with open(input_file) as _file: + input_file_list = _file.read().splitlines() + for _input_file in input_file_list: + input_files.append(_input_file.split(';')) + all_files = get_files_to_package(input_files) # Sort the files for reproducible order in the archive. all_files = sorted(all_files.items()) From 03023aebda8832e95957f0f532d746083e977e76 Mon Sep 17 00:00:00 2001 From: opbro Date: Thu, 28 May 2020 21:12:55 -0400 Subject: [PATCH 2/5] removed debugging and changed wheel name --- experimental/examples/wheel/BUILD | 2 +- experimental/python/wheel.bzl | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/experimental/examples/wheel/BUILD b/experimental/examples/wheel/BUILD index 809d8c49ee..8e3f9164c8 100644 --- a/experimental/examples/wheel/BUILD +++ b/experimental/examples/wheel/BUILD @@ -70,7 +70,7 @@ py_wheel( py_wheel( name = "minimal_with_py_package_wrapped", # Package data. We're building "example_minimal_package-0.0.1-py3-none-any.whl" - distribution = "example_minimal_package", + distribution = "example_minimal_package_wrapped_inputs", python_tag = "py3", version = "0.0.1", wrapped_package_lists = [":wrapped_example_package"], diff --git a/experimental/python/wheel.bzl b/experimental/python/wheel.bzl index df9a1a7b9f..cb3d17de3f 100644 --- a/experimental/python/wheel.bzl +++ b/experimental/python/wheel.bzl @@ -142,7 +142,6 @@ def _py_wheel_impl(ctx): wrapped_inputs = depset( direct = ctx.files.wrapped_package_lists ) - print(wrapped_inputs.to_list()) args.add_all(wrapped_inputs, format_each = "--input_file_list=%s") else: args.add_all(inputs_to_package, format_each = "--input_file=%s", map_each = _input_file_to_arg) From 68f5cdce1a964371d6bb5070ad3f276d349b19ba Mon Sep 17 00:00:00 2001 From: opbro Date: Mon, 1 Jun 2020 21:44:34 -0400 Subject: [PATCH 3/5] Removed the need to add the extra step --- experimental/examples/wheel/BUILD | 18 +----------- experimental/python/wheel.bzl | 49 +++++++------------------------ 2 files changed, 11 insertions(+), 56 deletions(-) diff --git a/experimental/examples/wheel/BUILD b/experimental/examples/wheel/BUILD index 8e3f9164c8..8916423907 100644 --- a/experimental/examples/wheel/BUILD +++ b/experimental/examples/wheel/BUILD @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -load("//experimental/python:wheel.bzl", "py_package", "py_wheel", "py_wrapped_input_files") +load("//experimental/python:wheel.bzl", "py_package", "py_wheel") load("//python:defs.bzl", "py_library", "py_test") package(default_visibility = ["//visibility:public"]) @@ -53,11 +53,6 @@ py_package( deps = [":main"], ) -py_wrapped_input_files( - name = "wrapped_example_package", - deps = [":example_pkg"], -) - py_wheel( name = "minimal_with_py_package", # Package data. We're building "example_minimal_package-0.0.1-py3-none-any.whl" @@ -67,17 +62,6 @@ py_wheel( deps = [":example_pkg"], ) -py_wheel( - name = "minimal_with_py_package_wrapped", - # Package data. We're building "example_minimal_package-0.0.1-py3-none-any.whl" - distribution = "example_minimal_package_wrapped_inputs", - python_tag = "py3", - version = "0.0.1", - wrapped_package_lists = [":wrapped_example_package"], - deps = [":example_pkg", ":wrapped_example_package"], -) - - # An example that uses all features provided by py_wheel. py_wheel( name = "customized", diff --git a/experimental/python/wheel.bzl b/experimental/python/wheel.bzl index cb3d17de3f..3ae5f7c218 100644 --- a/experimental/python/wheel.bzl +++ b/experimental/python/wheel.bzl @@ -81,37 +81,6 @@ Sub-packages are automatically included. }, ) -def _py_wrapped_input_files_impl(ctx): - inputs_to_package = depset( - direct = ctx.files.deps - ) - packageinputfile = ctx.actions.declare_file(ctx.attr.name + '_target_wrapped_inputs.txt') - content = '' - for input_file in inputs_to_package.to_list(): - content += _input_file_to_arg(input_file) + '\n' - ctx.actions.write(output = packageinputfile, content=content) - - return [ - DefaultInfo( - files = depset(direct = [packageinputfile]) - ) - ] - - - -py_wrapped_input_files = rule( - implementation = _py_wrapped_input_files_impl, - doc = """ - This was created to wrap the input files from py_package and use as in input to py_wheel, - by setting the attribute wrapped_input_files to avoid long command lines for systems that cannot - handle them. - """, - attrs = { - "deps": attr.label_list() - }, -) - - def _py_wheel_impl(ctx): outfile = ctx.actions.declare_file("-".join([ ctx.attr.distribution, @@ -129,6 +98,14 @@ def _py_wheel_impl(ctx): # Currently this is only the description file (if used). other_inputs = [] + # Wrap the inputs into a file to reduce command line length. + packageinputfile = ctx.actions.declare_file(ctx.attr.name + '_target_wrapped_inputs.txt') + content = '' + for input_file in inputs_to_package.to_list(): + content += _input_file_to_arg(input_file) + '\n' + ctx.actions.write(output = packageinputfile, content=content) + other_inputs.append(packageinputfile) + args = ctx.actions.args() args.add("--name", ctx.attr.distribution) args.add("--version", ctx.attr.version) @@ -138,13 +115,7 @@ def _py_wheel_impl(ctx): args.add("--out", outfile.path) args.add_all(ctx.attr.strip_path_prefixes, format_each = "--strip_path_prefix=%s") - if ctx.attr.wrapped_package_lists: - wrapped_inputs = depset( - direct = ctx.files.wrapped_package_lists - ) - args.add_all(wrapped_inputs, format_each = "--input_file_list=%s") - else: - args.add_all(inputs_to_package, format_each = "--input_file=%s", map_each = _input_file_to_arg) + args.add("--input_file_list", packageinputfile) extra_headers = [] if ctx.attr.author: @@ -177,6 +148,7 @@ def _py_wheel_impl(ctx): args.add("--description_file", description_file) other_inputs.append(description_file) + ctx.actions.run( inputs = depset(direct = other_inputs, transitive = [inputs_to_package]), outputs = [outfile], @@ -253,7 +225,6 @@ _other_attrs = { default = [], doc = "path prefixes to strip from files added to the generated package", ), - "wrapped_package_lists": attr.label_list(), } py_wheel = rule( From 7ad9db35ff58ee2f839170ff8b303c25ac3fd408 Mon Sep 17 00:00:00 2001 From: opbro Date: Mon, 1 Jun 2020 21:46:53 -0400 Subject: [PATCH 4/5] cleaned up extra lines --- experimental/python/wheel.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/experimental/python/wheel.bzl b/experimental/python/wheel.bzl index 3ae5f7c218..059b0a43d0 100644 --- a/experimental/python/wheel.bzl +++ b/experimental/python/wheel.bzl @@ -39,6 +39,7 @@ def _py_package_impl(ctx): transitive = [dep[DefaultInfo].data_runfiles.files for dep in ctx.attr.deps] + [dep[DefaultInfo].default_runfiles.files for dep in ctx.attr.deps], ) + # TODO: '/' is wrong on windows, but the path separator is not available in skylark. # Fix this once ctx.configuration has directory separator information. packages = [p.replace(".", "/") for p in ctx.attr.packages] @@ -53,9 +54,8 @@ def _py_package_impl(ctx): for package in packages: if wheel_path.startswith(package): filtered_files.append(input_file) - filtered_inputs = depset(direct = filtered_files) - + return [DefaultInfo( files = filtered_inputs, )] From 370378e496a6102d381a81735d1c3a955ed8ec5d Mon Sep 17 00:00:00 2001 From: opbro Date: Mon, 1 Jun 2020 21:47:22 -0400 Subject: [PATCH 5/5] cleaned up extra lines --- experimental/python/wheel.bzl | 1 - 1 file changed, 1 deletion(-) diff --git a/experimental/python/wheel.bzl b/experimental/python/wheel.bzl index 059b0a43d0..71bde94e1f 100644 --- a/experimental/python/wheel.bzl +++ b/experimental/python/wheel.bzl @@ -148,7 +148,6 @@ def _py_wheel_impl(ctx): args.add("--description_file", description_file) other_inputs.append(description_file) - ctx.actions.run( inputs = depset(direct = other_inputs, transitive = [inputs_to_package]), outputs = [outfile],