From d8719e4d044fb68f2f9367e941622f851c8879e0 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Wed, 29 Nov 2023 17:40:31 -0600 Subject: [PATCH 1/5] Get package name from settings.toml, allow arbitrary files in packages This is more dependable, and when we know the package name we can glob inside it to get all files such as bin or ttf files. This will allow e.g., 5x8.bin & ov5640_autofocus.bin within bundles. the behavior of bundlefly and circup when encountering .bin files needs to be checked. Tested by building modified pycamera bundle and the autofocus.bin file appears in the generated zip files: ``` pycamera-py-ec67bde/lib/adafruit_pycamera/ov5640_autofocus.bin 4077 4096 pycamera-8.x-mpy-ec67bde/lib/adafruit_pycamera/ov5640_autofocus.bin 4077 4096 pycamera-9.x-mpy-ec67bde/lib/adafruit_pycamera/ov5640_autofocus.bin 4077 4096 ``` There's at least one library in the bundle that has incorrect metadata and that leads to an error: https://github.com/adafruit/Adafruit_CircuitPython_Colorsys/pull/29 --- circuitpython_build_tools/build.py | 260 ++++++++++++++--------------- requirements.txt | 1 + 2 files changed, 122 insertions(+), 139 deletions(-) diff --git a/circuitpython_build_tools/build.py b/circuitpython_build_tools/build.py index 06bcfad..21f7849 100644 --- a/circuitpython_build_tools/build.py +++ b/circuitpython_build_tools/build.py @@ -36,8 +36,29 @@ import subprocess import tempfile +if sys.version_info >= (3, 11): + from tomllib import loads as load_toml +else: + from tomli import loads as load_toml + +def load_settings_toml(lib_path: pathlib.Path): + try: + return load_toml((lib_path / "pyproject.toml") .read_text(encoding="utf-8")) + except FileNotFoundError: + print(f"No settings.toml in {lib_path}") + return {} + +def get_nested(doc, *args, default=None): + for a in args: + if doc is None: return default + try: + doc = doc[a] + except (KeyError, IndexError) as e: + return default + return doc + IGNORE_PY = ["setup.py", "conf.py", "__init__.py"] -GLOB_PATTERNS = ["*.py", "font5x8.bin"] +GLOB_PATTERNS = ["*.py", "*.bin"] S3_MPY_PREFIX = "https://adafruit-circuit-python.s3.amazonaws.com/bin/mpy-cross" def version_string(path=None, *, valid_semver=False): @@ -131,17 +152,13 @@ def mpy_cross(mpy_cross_filename, circuitpython_tag, quiet=False): shutil.copy("build_deps/circuitpython/mpy-cross/mpy-cross", mpy_cross_filename) def _munge_to_temp(original_path, temp_file, library_version): - with open(original_path, "rb") as original_file: + with open(original_path, "r", encoding="utf-8") as original_file: for line in original_file: - if original_path.endswith(".bin"): - # this is solely for adafruit_framebuf/examples/font5x8.bin - temp_file.write(line) - else: - line = line.decode("utf-8").strip("\n") - if line.startswith("__version__"): - line = line.replace("0.0.0-auto.0", library_version) - line = line.replace("0.0.0+auto.0", library_version) - temp_file.write(line.encode("utf-8") + b"\r\n") + line = line.strip("\n") + if line.startswith("__version__"): + line = line.replace("0.0.0-auto.0", library_version) + line = line.replace("0.0.0+auto.0", library_version) + print(line, file=temp_file) temp_file.flush() def get_package_info(library_path, package_folder_prefix): @@ -154,61 +171,46 @@ def get_package_info(library_path, package_folder_prefix): for pattern in GLOB_PATTERNS: glob_search.extend(list(lib_path.rglob(pattern))) - package_info["is_package"] = False - for file in glob_search: - if file.parts[parent_idx] != "examples": - if len(file.parts) > parent_idx + 1: - for prefix in package_folder_prefix: - if file.parts[parent_idx].startswith(prefix): - package_info["is_package"] = True - if package_info["is_package"]: - package_files.append(file) - else: - if file.name in IGNORE_PY: - #print("Ignoring:", file.resolve()) - continue - if file.parent == lib_path: - py_files.append(file) - - if package_files: - package_info["module_name"] = package_files[0].relative_to(library_path).parent.name - elif py_files: - package_info["module_name"] = py_files[0].relative_to(library_path).name[:-3] - else: - package_info["module_name"] = None - - try: - package_info["version"] = version_string(library_path, valid_semver=True) - except ValueError as e: - package_info["version"] = version_string(library_path) - - return package_info - -def library(library_path, output_directory, package_folder_prefix, - mpy_cross=None, example_bundle=False): - py_files = [] - package_files = [] - example_files = [] - total_size = 512 - - lib_path = pathlib.Path(library_path) - parent_idx = len(lib_path.parts) - glob_search = [] - for pattern in GLOB_PATTERNS: - glob_search.extend(list(lib_path.rglob(pattern))) - - for file in glob_search: - if file.parts[parent_idx] == "examples": - example_files.append(file) - else: - if not example_bundle: - is_package = False + settings_toml = load_settings_toml(lib_path) + py_modules = get_nested(settings_toml, "tool", "setuptools", "py-modules", default=[]) + packages = get_nested(settings_toml, "tool", "setuptools", "packages", default=[]) + + example_files = [sub_path for sub_path in (lib_path / "examples").rglob("*") + if sub_path.is_file()] + + if packages and py_modules: + raise ValueError("Cannot specify both tool.setuptools.py-modules and .packages") + + elif packages: + if len(packages) > 1: + raise ValueError("Only a single package is supported") + package_name = packages[0] + print(f"Using package name from settings.toml: {package_name}") + package_info["is_package"] = True + package_info["module_name"] = package_name + package_files = [sub_path for sub_path in (lib_path / package_name).rglob("*") + if sub_path.is_file()] + + elif py_modules: + if len(py_modules) > 1: + raise ValueError("Only a single module is supported") + print("Using module name from settings.toml") + py_module = py_modules[0] + package_name = py_module + package_info["is_package"] = False + package_info["module_name"] = py_module + py_files = [lib_path / f"{py_module}.py"] + + if not packages and not py_modules: + print("Using legacy autodetection") + package_info["is_package"] = False + for file in glob_search: + if file.parts[parent_idx] != "examples": if len(file.parts) > parent_idx + 1: for prefix in package_folder_prefix: if file.parts[parent_idx].startswith(prefix): - is_package = True - - if is_package: + package_info["is_package"] = True + if package_info["is_package"]: package_files.append(file) else: if file.name in IGNORE_PY: @@ -217,91 +219,78 @@ def library(library_path, output_directory, package_folder_prefix, if file.parent == lib_path: py_files.append(file) + if package_files: + package_info["module_name"] = package_files[0].relative_to(library_path).parent.name + elif py_files: + package_info["module_name"] = py_files[0].relative_to(library_path).name[:-3] + else: + package_info["module_name"] = None + if len(py_files) > 1: raise ValueError("Multiple top level py files not allowed. Please put " "them in a package or combine them into a single file.") - if package_files: - module_name = package_files[0].relative_to(library_path).parent.name - elif py_files: - module_name = py_files[0].relative_to(library_path).name[:-3] - else: - module_name = None + package_info["package_files"] = package_files + package_info["py_files"] = py_files + package_info["example_files"] = example_files + + try: + package_info["version"] = version_string(library_path, valid_semver=True) + except ValueError as e: + print(library_path + " has version that doesn't follow SemVer (semver.org)") + print(e) + package_info["version"] = version_string(library_path) + + return package_info + +def library(library_path, output_directory, package_folder_prefix, + mpy_cross=None, example_bundle=False): + lib_path = pathlib.Path(library_path) + package_info = get_package_info(library_path, package_folder_prefix) + py_package_files = package_info["package_files"] + package_info["py_files"] + example_files = package_info["example_files"] + module_name = package_info["module_name"] for fn in example_files: base_dir = os.path.join(output_directory.replace("/lib", "/"), fn.relative_to(library_path).parent) if not os.path.isdir(base_dir): os.makedirs(base_dir) - total_size += 512 - for fn in package_files: + for fn in py_package_files: base_dir = os.path.join(output_directory, fn.relative_to(library_path).parent) if not os.path.isdir(base_dir): os.makedirs(base_dir) - total_size += 512 - - new_extension = ".py" - if mpy_cross: - new_extension = ".mpy" - - try: - library_version = version_string(library_path, valid_semver=True) - except ValueError as e: - print(library_path + " has version that doesn't follow SemVer (semver.org)") - print(e) - library_version = version_string(library_path) - for filename in py_files: - full_path = os.path.join(library_path, filename) - output_file = os.path.join( - output_directory, - filename.relative_to(library_path).with_suffix(new_extension) - ) - with tempfile.NamedTemporaryFile(delete=False) as temp_file: - _munge_to_temp(full_path, temp_file, library_version) - temp_filename = temp_file.name - # Windows: close the temp file before it can be read or copied by name - if mpy_cross: - mpy_success = subprocess.call([ - mpy_cross, - "-o", output_file, - "-s", str(filename.relative_to(library_path)), - temp_filename - ]) - if mpy_success != 0: - raise RuntimeError("mpy-cross failed on", full_path) - else: - shutil.copyfile(temp_filename, output_file) - os.remove(temp_filename) + library_version = package_info['version'] - for filename in package_files: - full_path = os.path.join(library_path, filename) - output_file = "" - with tempfile.NamedTemporaryFile(delete=False) as temp_file: - _munge_to_temp(full_path, temp_file, library_version) - temp_filename = temp_file.name - # Windows: close the temp file before it can be read or copied by name - if not mpy_cross or os.stat(full_path).st_size == 0: - output_file = os.path.join(output_directory, - filename.relative_to(library_path)) - shutil.copyfile(temp_filename, output_file) - else: - output_file = os.path.join( - output_directory, - filename.relative_to(library_path).with_suffix(new_extension) - ) - - mpy_success = subprocess.call([ - mpy_cross, - "-o", output_file, - "-s", str(filename.relative_to(library_path)), - temp_filename - ]) - if mpy_success != 0: - raise RuntimeError("mpy-cross failed on", full_path) - os.remove(temp_filename) + if not example_bundle: + for filename in py_package_files: + full_path = os.path.join(library_path, filename) + output_file = output_directory / filename.relative_to(library_path) + if filename.suffix == ".py": + with tempfile.NamedTemporaryFile(delete=False, mode="w+") as temp_file: + temp_file_name = temp_file.name + try: + _munge_to_temp(full_path, temp_file, library_version) + temp_file.close() + if mpy_cross: + output_file = output_file.with_suffix(".mpy") + mpy_success = subprocess.call([ + mpy_cross, + "-o", output_file, + "-s", str(filename.relative_to(library_path)), + temp_file.name + ]) + if mpy_success != 0: + raise RuntimeError("mpy-cross failed on", full_path) + else: + shutil.copyfile(full_path, output_file) + finally: + os.remove(temp_file_name) + else: + shutil.copyfile(full_path, output_file) requirements_files = lib_path.glob("requirements.txt*") requirements_files = [f for f in requirements_files if f.stat().st_size > 0] @@ -314,11 +303,9 @@ def library(library_path, output_directory, package_folder_prefix, requirements_dir = pathlib.Path(output_directory).parent / "requirements" if not os.path.isdir(requirements_dir): os.makedirs(requirements_dir, exist_ok=True) - total_size += 512 requirements_subdir = f"{requirements_dir}/{module_name}" if not os.path.isdir(requirements_subdir): os.makedirs(requirements_subdir, exist_ok=True) - total_size += 512 for filename in requirements_files: full_path = os.path.join(library_path, filename) output_file = os.path.join(requirements_subdir, filename.name) @@ -328,9 +315,4 @@ def library(library_path, output_directory, package_folder_prefix, full_path = os.path.join(library_path, filename) output_file = os.path.join(output_directory.replace("/lib", "/"), filename.relative_to(library_path)) - temp_filename = "" - with tempfile.NamedTemporaryFile(delete=False) as temp_file: - _munge_to_temp(full_path, temp_file, library_version) - temp_filename = temp_file.name - shutil.copyfile(temp_filename, output_file) - os.remove(temp_filename) + shutil.copyfile(full_path, output_file) diff --git a/requirements.txt b/requirements.txt index 861b8da..8a3514c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ Click requests semver wheel +tomli; python_version < "3.11" From 0c58704d658b2c741d47a042c0ddeb3419cf0f2e Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Thu, 30 Nov 2023 09:08:41 -0600 Subject: [PATCH 2/5] Restore behavior of shipping 0-byte py files as .py, not .mpy (it's smaller on disk) --- circuitpython_build_tools/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circuitpython_build_tools/build.py b/circuitpython_build_tools/build.py index 21f7849..faecfd5 100644 --- a/circuitpython_build_tools/build.py +++ b/circuitpython_build_tools/build.py @@ -275,7 +275,7 @@ def library(library_path, output_directory, package_folder_prefix, try: _munge_to_temp(full_path, temp_file, library_version) temp_file.close() - if mpy_cross: + if mpy_cross and os.stat(temp_file.name).st_size != 0: output_file = output_file.with_suffix(".mpy") mpy_success = subprocess.call([ mpy_cross, From ca779868d854e5aa8b0814ef6d080d8d4bdbb5bb Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Thu, 30 Nov 2023 10:34:45 -0600 Subject: [PATCH 3/5] pyproject is the name of the file --- circuitpython_build_tools/build.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/circuitpython_build_tools/build.py b/circuitpython_build_tools/build.py index faecfd5..c80712e 100644 --- a/circuitpython_build_tools/build.py +++ b/circuitpython_build_tools/build.py @@ -41,11 +41,11 @@ else: from tomli import loads as load_toml -def load_settings_toml(lib_path: pathlib.Path): +def load_pyproject_toml(lib_path: pathlib.Path): try: return load_toml((lib_path / "pyproject.toml") .read_text(encoding="utf-8")) except FileNotFoundError: - print(f"No settings.toml in {lib_path}") + print(f"No pyproject.toml in {lib_path}") return {} def get_nested(doc, *args, default=None): @@ -171,9 +171,9 @@ def get_package_info(library_path, package_folder_prefix): for pattern in GLOB_PATTERNS: glob_search.extend(list(lib_path.rglob(pattern))) - settings_toml = load_settings_toml(lib_path) - py_modules = get_nested(settings_toml, "tool", "setuptools", "py-modules", default=[]) - packages = get_nested(settings_toml, "tool", "setuptools", "packages", default=[]) + pyproject_toml = load_pyproject.toml(lib_path) + py_modules = get_nested(pyproject_toml, "tool", "setuptools", "py-modules", default=[]) + packages = get_nested(pyproject_toml, "tool", "setuptools", "packages", default=[]) example_files = [sub_path for sub_path in (lib_path / "examples").rglob("*") if sub_path.is_file()] @@ -185,7 +185,7 @@ def get_package_info(library_path, package_folder_prefix): if len(packages) > 1: raise ValueError("Only a single package is supported") package_name = packages[0] - print(f"Using package name from settings.toml: {package_name}") + print(f"Using package name from pyproject.toml: {package_name}") package_info["is_package"] = True package_info["module_name"] = package_name package_files = [sub_path for sub_path in (lib_path / package_name).rglob("*") @@ -194,14 +194,14 @@ def get_package_info(library_path, package_folder_prefix): elif py_modules: if len(py_modules) > 1: raise ValueError("Only a single module is supported") - print("Using module name from settings.toml") + print("Using module name from pyproject.toml") py_module = py_modules[0] package_name = py_module package_info["is_package"] = False package_info["module_name"] = py_module py_files = [lib_path / f"{py_module}.py"] - if not packages and not py_modules: + else: print("Using legacy autodetection") package_info["is_package"] = False for file in glob_search: From 373b25644121b382b26117616b5c462407a2bda5 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sat, 2 Dec 2023 15:22:33 -0600 Subject: [PATCH 4/5] typo --- circuitpython_build_tools/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circuitpython_build_tools/build.py b/circuitpython_build_tools/build.py index c80712e..336e50a 100644 --- a/circuitpython_build_tools/build.py +++ b/circuitpython_build_tools/build.py @@ -171,7 +171,7 @@ def get_package_info(library_path, package_folder_prefix): for pattern in GLOB_PATTERNS: glob_search.extend(list(lib_path.rglob(pattern))) - pyproject_toml = load_pyproject.toml(lib_path) + pyproject_toml = load_pyproject_toml(lib_path) py_modules = get_nested(pyproject_toml, "tool", "setuptools", "py-modules", default=[]) packages = get_nested(pyproject_toml, "tool", "setuptools", "packages", default=[]) From b51d905d9d7ad5d04ff6b63e191708ec75a02e56 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sat, 2 Dec 2023 15:23:00 -0600 Subject: [PATCH 5/5] blacklist should let this pass CI --- circuitpython_build_tools/build.py | 31 +++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/circuitpython_build_tools/build.py b/circuitpython_build_tools/build.py index 336e50a..f884a1c 100644 --- a/circuitpython_build_tools/build.py +++ b/circuitpython_build_tools/build.py @@ -36,6 +36,25 @@ import subprocess import tempfile +# pyproject.toml `py_modules` values that are incorrect. These should all have PRs filed! +# and should be removed when the fixed version is incorporated in its respective bundle. + +pyproject_py_modules_blacklist = set(( + # adafruit bundle + "adafruit_colorsys", + + # community bundle + "at24mac_eeprom", + "circuitpython_Candlesticks", + "CircuitPython_Color_Picker", + "CircuitPython_Equalizer", + "CircuitPython_Scales", + "circuitPython_Slider", + "circuitpython_uboxplot", + "P1AM", + "p1am_200_helpers", +)) + if sys.version_info >= (3, 11): from tomllib import loads as load_toml else: @@ -175,6 +194,12 @@ def get_package_info(library_path, package_folder_prefix): py_modules = get_nested(pyproject_toml, "tool", "setuptools", "py-modules", default=[]) packages = get_nested(pyproject_toml, "tool", "setuptools", "packages", default=[]) + blacklisted = [name for name in py_modules if name in pyproject_py_modules_blacklist] + + if blacklisted: + print(f"{lib_path}/settings.toml:1: {blacklisted[0]} blacklisted: not using metadata from pyproject.toml") + py_modules = packages = () + example_files = [sub_path for sub_path in (lib_path / "examples").rglob("*") if sub_path.is_file()] @@ -185,7 +210,7 @@ def get_package_info(library_path, package_folder_prefix): if len(packages) > 1: raise ValueError("Only a single package is supported") package_name = packages[0] - print(f"Using package name from pyproject.toml: {package_name}") + #print(f"Using package name from pyproject.toml: {package_name}") package_info["is_package"] = True package_info["module_name"] = package_name package_files = [sub_path for sub_path in (lib_path / package_name).rglob("*") @@ -194,15 +219,15 @@ def get_package_info(library_path, package_folder_prefix): elif py_modules: if len(py_modules) > 1: raise ValueError("Only a single module is supported") - print("Using module name from pyproject.toml") py_module = py_modules[0] + #print(f"Using module name from pyproject.toml: {py_module}") package_name = py_module package_info["is_package"] = False package_info["module_name"] = py_module py_files = [lib_path / f"{py_module}.py"] else: - print("Using legacy autodetection") + print(f"{lib_path}: Using legacy autodetection") package_info["is_package"] = False for file in glob_search: if file.parts[parent_idx] != "examples":