Skip to content

Replace setup.py bdist_wheel with python -m build --wheel #156712

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 7 commits into
base: gh/zklaus/11/base
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .ci/aarch64_linux/aarch64_wheel_ci_build.py
Original file line number Diff line number Diff line change
Expand Up @@ -250,7 +250,7 @@ def parse_arguments():
else:
print("build pytorch without mkldnn backend")

os.system(f"cd /pytorch; {build_vars} python3 setup.py bdist_wheel")
os.system(f"cd /pytorch; {build_vars} python3 -m build --wheel --no-isolation")
if enable_cuda:
print("Updating Cuda Dependency")
filename = os.listdir("/pytorch/dist/")
Expand Down
15 changes: 8 additions & 7 deletions .ci/aarch64_linux/build_aarch64_wheel.py
Original file line number Diff line number Diff line change
Expand Up @@ -442,7 +442,7 @@ def build_torchvision(
if host.using_docker():
build_vars += " CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000"

host.run_cmd(f"cd vision && {build_vars} python3 setup.py bdist_wheel")
host.run_cmd(f"cd vision && {build_vars} python3 -m build --wheel --no-isolation")
vision_wheel_name = host.list_dir("vision/dist")[0]
embed_libgomp(host, use_conda, os.path.join("vision", "dist", vision_wheel_name))

Expand Down Expand Up @@ -497,7 +497,7 @@ def build_torchdata(
if host.using_docker():
build_vars += " CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000"

host.run_cmd(f"cd data && {build_vars} python3 setup.py bdist_wheel")
host.run_cmd(f"cd data && {build_vars} python3 -m build --wheel --no-isolation")
wheel_name = host.list_dir("data/dist")[0]
embed_libgomp(host, use_conda, os.path.join("data", "dist", wheel_name))

Expand Down Expand Up @@ -553,7 +553,7 @@ def build_torchtext(
if host.using_docker():
build_vars += " CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000"

host.run_cmd(f"cd text && {build_vars} python3 setup.py bdist_wheel")
host.run_cmd(f"cd text && {build_vars} python3 -m build --wheel --no-isolation")
wheel_name = host.list_dir("text/dist")[0]
embed_libgomp(host, use_conda, os.path.join("text", "dist", wheel_name))

Expand Down Expand Up @@ -614,7 +614,7 @@ def build_torchaudio(
host.run_cmd(
f"cd audio && export FFMPEG_ROOT=$(pwd)/third_party/ffmpeg && export USE_FFMPEG=1 \
&& ./packaging/ffmpeg/build.sh \
&& {build_vars} python3 setup.py bdist_wheel"
&& {build_vars} python3 -m build --wheel --no-isolation"
)

wheel_name = host.list_dir("audio/dist")[0]
Expand Down Expand Up @@ -726,7 +726,7 @@ def start_build(
print("Building PyTorch wheel")
build_opts = ""
if pytorch_build_number is not None:
build_opts += f" --build-number {pytorch_build_number}"
build_opts += f" -C--build-option=--build-number={pytorch_build_number}"
# Breakpad build fails on aarch64
build_vars = "USE_BREAKPAD=0 "
if branch == "nightly":
Expand All @@ -747,7 +747,8 @@ def start_build(
print("build pytorch with mkldnn+acl backend")
build_vars += " USE_MKLDNN=ON USE_MKLDNN_ACL=ON"
host.run_cmd(
f"cd $HOME/pytorch && export ACL_ROOT_DIR=$HOME/ComputeLibrary && {build_vars} python3 setup.py bdist_wheel{build_opts}"
f"cd $HOME/pytorch && export ACL_ROOT_DIR=$HOME/ComputeLibrary && "
f"{build_vars} python3 -m build --wheel --no-isolation{build_opts}"
)
print("Repair the wheel")
pytorch_wheel_name = host.list_dir("pytorch/dist")[0]
Expand All @@ -763,7 +764,7 @@ def start_build(
else:
print("build pytorch without mkldnn backend")
host.run_cmd(
f"cd pytorch && {build_vars} python3 setup.py bdist_wheel{build_opts}"
f"cd pytorch && {build_vars} python3 -m build --wheel --no-isolation{build_opts}"
)

print("Deleting build folder")
Expand Down
9 changes: 0 additions & 9 deletions .ci/docker/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -456,12 +456,3 @@ elif [ "$HAS_TRITON" = "yes" ]; then
echo "expecting triton to not be installed, but it is"
exit 1
fi

# Sanity check cmake version. Executorch reinstalls cmake and I'm not sure if
# they support 4.0.0 yet, so exclude them from this check.
CMAKE_VERSION=$(drun cmake --version)
if [[ "$EXECUTORCH" != *yes* && "$CMAKE_VERSION" != *4.* ]]; then
echo "CMake version is not 4.0.0:"
drun cmake --version
exit 1
fi
6 changes: 3 additions & 3 deletions .ci/docker/common/install_triton.sh
Original file line number Diff line number Diff line change
Expand Up @@ -66,15 +66,15 @@ if [ -n "${UBUNTU_VERSION}" ] && [ -n "${GCC_VERSION}" ] && [[ "${GCC_VERSION}"
# Triton needs at least gcc-9 to build
apt-get install -y g++-9

CXX=g++-9 conda_run python setup.py bdist_wheel
CXX=g++-9 conda_run python -m build --wheel --no-isolation
elif [ -n "${UBUNTU_VERSION}" ] && [ -n "${CLANG_VERSION}" ]; then
# Triton needs <filesystem> which surprisingly is not available with clang-9 toolchain
add-apt-repository -y ppa:ubuntu-toolchain-r/test
apt-get install -y g++-9

CXX=g++-9 conda_run python setup.py bdist_wheel
CXX=g++-9 conda_run python -m build --wheel --no-isolation
else
conda_run python setup.py bdist_wheel
conda_run python -m build --wheel --no-isolation
fi

# Copy the wheel to /opt for multi stage docker builds
Expand Down
7 changes: 6 additions & 1 deletion .ci/docker/requirements-ci.txt
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,11 @@ boto3==1.35.42
#Pinned versions: 1.19.12, 1.16.34
#test that import:

build
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
build
build==1.3.0

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As long as we stick with --config-setting/-C, I think we can leave this pin open.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is used to keep our CI reproducible.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That makes sense and I have no objection to pinning. However, not all packages in the file are pinned, c.f. click, psutil, pyyaml, and others. How is it decided which ones to pin and which ones to leave open? And when should the actual pin deviate from the comment as with fbscribelogger and others?

#Description: A simple, correct Python build frontend.
#Pinned versions:
#test that import:

click
#Description: Command Line Interface Creation Kit
#Pinned versions:
Expand Down Expand Up @@ -381,7 +386,7 @@ dataclasses_json==0.6.7
#Pinned versions: 0.6.7
#test that import:

cmake==4.0.0
cmake==3.31.6
#Description: required for building

tlparse==0.3.30
Expand Down
2 changes: 1 addition & 1 deletion .ci/manywheel/build_common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ time CMAKE_ARGS=${CMAKE_ARGS[@]} \
EXTRA_CAFFE2_CMAKE_FLAGS=${EXTRA_CAFFE2_CMAKE_FLAGS[@]} \
BUILD_LIBTORCH_CPU_WITH_DEBUG=$BUILD_DEBUG_INFO \
USE_NCCL=${USE_NCCL} USE_RCCL=${USE_RCCL} USE_KINETO=${USE_KINETO} \
python setup.py bdist_wheel -d /tmp/$WHEELHOUSE_DIR
python -m build --wheel --no-isolation --outdir /tmp/$WHEELHOUSE_DIR
echo "Finished setup.py bdist at $(date)"

# Build libtorch packages
Expand Down
4 changes: 2 additions & 2 deletions .ci/pytorch/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -261,13 +261,13 @@ else

WERROR=1 python setup.py clean

WERROR=1 python setup.py bdist_wheel
WERROR=1 python -m build --wheel --no-isolation
else
python setup.py clean
if [[ "$BUILD_ENVIRONMENT" == *xla* ]]; then
source .ci/pytorch/install_cache_xla.sh
fi
python setup.py bdist_wheel
python -m build --wheel --no-isolation
fi
pip_install_whl "$(echo dist/*.whl)"

Expand Down
4 changes: 2 additions & 2 deletions .ci/pytorch/macos-build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,11 @@ fi
print_cmake_info
if [[ ${BUILD_ENVIRONMENT} == *"distributed"* ]]; then
# Needed for inductor benchmarks, as lots of HF networks make `torch.distribtued` calls
USE_DISTRIBUTED=1 USE_OPENMP=1 WERROR=1 python setup.py bdist_wheel
USE_DISTRIBUTED=1 USE_OPENMP=1 WERROR=1 python -m build --wheel --no-isolation
else
# Explicitly set USE_DISTRIBUTED=0 to align with the default build config on mac. This also serves as the sole CI config that tests
# that building with USE_DISTRIBUTED=0 works at all. See https://github.com/pytorch/pytorch/issues/86448
USE_DISTRIBUTED=0 USE_OPENMP=1 MACOSX_DEPLOYMENT_TARGET=11.0 WERROR=1 BUILD_TEST=OFF USE_PYTORCH_METAL=1 python setup.py bdist_wheel --plat-name macosx_11_0_arm64
USE_DISTRIBUTED=0 USE_OPENMP=1 MACOSX_DEPLOYMENT_TARGET=11.0 WERROR=1 BUILD_TEST=OFF USE_PYTORCH_METAL=1 python -m build --wheel --no-isolation -C--build-option=--plat-name=macosx_11_0_arm64
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

FYI, build 1.3.0 was released last week, which added a new --config-json option. That might be better than --config-setting / -C.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks, that's an interesting new facility. It seems clear that we could move to --config-json="{'--build-option': '--platform-name=macosx_11_0_arm64'}", but given the support in setuptools, I am afraid we can't go all the way to --config-json="{'build-option': {'platform-name': 'macosx_11_0_arm64'}}". With the lack of support in older build versions, I think the best way forward is to stick with --config-setting for now and to reevaluate with a larger build system overhaul.

fi
if which sccache > /dev/null; then
print_sccache_stats
Expand Down
2 changes: 1 addition & 1 deletion .ci/pytorch/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -1383,7 +1383,7 @@ EOF
pip3 install -r requirements.txt
# shellcheck source=./common-build.sh
source "$(dirname "${BASH_SOURCE[0]}")/common-build.sh"
python setup.py bdist_wheel --bdist-dir="base_bdist_tmp" --dist-dir="base_dist"
python -m build --wheel --no-isolation -C--build-option=--bdist-dir="base_bdist_tmp" --outdir "base_dist"
python -mpip install base_dist/*.whl
echo "::endgroup::"

Expand Down
2 changes: 1 addition & 1 deletion .ci/pytorch/win-test-helpers/build_pytorch.bat
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ if "%USE_CUDA%"=="1" (
:: Print all existing environment variable for debugging
set

python setup.py bdist_wheel
python -m build --wheel --no-isolation
if errorlevel 1 goto fail
if not errorlevel 0 goto fail
sccache --show-stats
Expand Down
2 changes: 1 addition & 1 deletion .ci/pytorch/windows/arm64/build_pytorch.bat
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ sccache --zero-stats
sccache --show-stats

:: Call PyTorch build script
python setup.py bdist_wheel -d "%PYTORCH_FINAL_PACKAGE_DIR%"
python -m build --wheel --no-isolation --outdir "%PYTORCH_FINAL_PACKAGE_DIR%"

:: show sccache stats
sccache --show-stats
Expand Down
2 changes: 1 addition & 1 deletion .ci/pytorch/windows/internal/setup.bat
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ copy /Y "%LIBTORCH_PREFIX%-%PYTORCH_BUILD_VERSION%.zip" "%PYTORCH_FINAL_PACKAGE_
goto build_end

:pytorch
%PYTHON_EXEC% setup.py bdist_wheel -d "%PYTORCH_FINAL_PACKAGE_DIR%"
%PYTHON_EXEC% -m build --wheel --no-isolation --outdir "%PYTORCH_FINAL_PACKAGE_DIR%"

:build_end
IF ERRORLEVEL 1 exit /b 1
Expand Down
6 changes: 3 additions & 3 deletions .ci/wheel/build_wheel.sh
Original file line number Diff line number Diff line change
Expand Up @@ -197,11 +197,11 @@ export USE_QNNPACK=OFF
export BUILD_TEST=OFF

pushd "$pytorch_rootdir"
echo "Calling setup.py bdist_wheel at $(date)"
echo "Calling -m build --wheel --no-isolation at $(date)"

python setup.py bdist_wheel -d "$whl_tmp_dir"
python -m build --wheel --no-isolation --outdir "$whl_tmp_dir"

echo "Finished setup.py bdist_wheel at $(date)"
echo "Finished -m build --wheel --no-isolation at $(date)"

if [[ $package_type != 'libtorch' ]]; then
echo "delocating wheel dependencies"
Expand Down
1 change: 1 addition & 0 deletions .github/requirements/pip-requirements-macOS.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
boto3==1.35.42
build==1.2.2.post1
cmake==3.27.*
expecttest==0.3.0
fbscribelogger==0.1.7
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,15 +24,15 @@ def setUpClass(cls):
shutil.rmtree(cls.dist_dir)

# Build the wheel
wheel_cmd = [sys.executable, "setup.py", "bdist_wheel"]
wheel_cmd = [sys.executable, "-m", "build", "--wheel", "--no-isolation"]
return_code = shell(wheel_cmd, cwd=cls.extension_root, env=os.environ)
if return_code != 0:
raise RuntimeError("python_agnostic bdist_wheel failed to build")

@onlyCUDA
@unittest.skipIf(not IS_LINUX, "test requires linux tools ldd and nm")
def test_extension_is_python_agnostic(self, device):
# For this test, run_test.py will call `python setup.py bdist_wheel` in the
# For this test, run_test.py will call `python -m build --wheel --no-isolation` in the
# cpp_extensions/python_agnostic_extension folder, where the extension and
# setup calls specify py_limited_api to `True`. To approximate that the
# extension is indeed python agnostic, we test
Expand Down
2 changes: 1 addition & 1 deletion test/run_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -842,7 +842,7 @@ def _test_cpp_extensions_aot(test_directory, options, use_ninja):
"--root",
"./install",
]
wheel_cmd = [sys.executable, "setup.py", "bdist_wheel"]
wheel_cmd = [sys.executable, "-m", "build", "--wheel", "--no-isolation"]
return_code = shell(install_cmd, cwd=cpp_extensions_test_dir, env=shell_env)
if return_code != 0:
return return_code
Expand Down
26 changes: 17 additions & 9 deletions tools/packaging/build_wheel.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/usr/bin/env python3

Check warning on line 1 in tools/packaging/build_wheel.py

View workflow job for this annotation

GitHub Actions / bc_linter

Function Builder.bdist_wheel: function deleted

Check warning on line 1 in tools/packaging/build_wheel.py

View workflow job for this annotation

GitHub Actions / bc_linter

Function Builder.setup_py: function deleted

import argparse
import contextlib
Expand All @@ -20,7 +20,6 @@
logger.setLevel(logging.INFO)

ROOT_PATH = Path(__file__).absolute().parent.parent.parent
SETUP_PY_PATH = ROOT_PATH / "setup.py"
REQUIREMENTS_PATH = ROOT_PATH / "requirements.txt"
PYPROJECT_TOML_PATH = ROOT_PATH / "pyproject.toml"

Expand Down Expand Up @@ -143,18 +142,27 @@
def __init__(self, interpreter: str) -> None:
self.interpreter = interpreter

def setup_py(self, cmd_args: list[str]) -> bool:
def build_wheel(self, destination: str) -> bool:
logger.info("Running bdist_wheel -d %s", destination)
return (
run_cmd([self.interpreter, str(SETUP_PY_PATH), *cmd_args]).returncode == 0
run_cmd(
[
self.interpreter,
"-m",
"build",
"--wheel",
"--no-isolation",
"--outdir",
destination,
str(ROOT_PATH),
]
).returncode
== 0
)

def bdist_wheel(self, destination: str) -> bool:
logger.info("Running bdist_wheel -d %s", destination)
return self.setup_py(["bdist_wheel", "-d", destination])

def clean(self) -> bool:
logger.info("Running clean")
return self.setup_py(["clean"])
return run_cmd([self.interpreter, "setup.py", "clean"]).returncode == 0

def install_requirements(self) -> None:
logger.info("Installing requirements")
Expand Down Expand Up @@ -234,7 +242,7 @@

start_time = time.time()

builder.bdist_wheel(args.destination)
builder.build_wheel(args.destination)

end_time = time.time()

Expand Down
Loading