Skip to content

Commit 36a5a47

Browse files
moksiucmoksiuc
committed
Merge branch 'refs/heads/main' into moksiucik_torchrun_xpu
2 parents a6e379c + 1c2cba1 commit 36a5a47

File tree

487 files changed

+10557
-43032
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

487 files changed

+10557
-43032
lines changed

‎.ci/aarch64_linux/build_aarch64_wheel.py

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -438,9 +438,7 @@ def build_torchvision(
438438
)
439439
build_vars += f"BUILD_VERSION={version}.dev{build_date}"
440440
elif build_version is not None:
441-
build_vars += (
442-
f"BUILD_VERSION={build_version} PYTORCH_VERSION={branch[1:].split('-')[0]}"
443-
)
441+
build_vars += f"BUILD_VERSION={build_version} PYTORCH_VERSION={branch[1:].split('-', maxsplit=1)[0]}"
444442
if host.using_docker():
445443
build_vars += " CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000"
446444

@@ -495,9 +493,7 @@ def build_torchdata(
495493
)
496494
build_vars += f"BUILD_VERSION={version}.dev{build_date}"
497495
elif build_version is not None:
498-
build_vars += (
499-
f"BUILD_VERSION={build_version} PYTORCH_VERSION={branch[1:].split('-')[0]}"
500-
)
496+
build_vars += f"BUILD_VERSION={build_version} PYTORCH_VERSION={branch[1:].split('-', maxsplit=1)[0]}"
501497
if host.using_docker():
502498
build_vars += " CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000"
503499

@@ -553,9 +549,7 @@ def build_torchtext(
553549
)
554550
build_vars += f"BUILD_VERSION={version}.dev{build_date}"
555551
elif build_version is not None:
556-
build_vars += (
557-
f"BUILD_VERSION={build_version} PYTORCH_VERSION={branch[1:].split('-')[0]}"
558-
)
552+
build_vars += f"BUILD_VERSION={build_version} PYTORCH_VERSION={branch[1:].split('-', maxsplit=1)[0]}"
559553
if host.using_docker():
560554
build_vars += " CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000"
561555

@@ -613,9 +607,7 @@ def build_torchaudio(
613607
)
614608
build_vars += f"BUILD_VERSION={version}.dev{build_date}"
615609
elif build_version is not None:
616-
build_vars += (
617-
f"BUILD_VERSION={build_version} PYTORCH_VERSION={branch[1:].split('-')[0]}"
618-
)
610+
build_vars += f"BUILD_VERSION={build_version} PYTORCH_VERSION={branch[1:].split('-', maxsplit=1)[0]}"
619611
if host.using_docker():
620612
build_vars += " CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000"
621613

‎.ci/docker/build.sh

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,7 @@ case "$tag" in
176176
VISION=yes
177177
TRITON=yes
178178
;;
179-
pytorch-linux-jammy-rocm-n-py3 | pytorch-linux-noble-rocm-n-py3)
179+
pytorch-linux-jammy-rocm-n-py3 | pytorch-linux-jammy-rocm-n-py3-benchmarks | pytorch-linux-noble-rocm-n-py3)
180180
if [[ $tag =~ "jammy" ]]; then
181181
ANACONDA_PYTHON_VERSION=3.10
182182
else
@@ -190,7 +190,9 @@ case "$tag" in
190190
KATEX=yes
191191
UCX_COMMIT=${_UCX_COMMIT}
192192
UCC_COMMIT=${_UCC_COMMIT}
193-
INDUCTOR_BENCHMARKS=yes
193+
if [[ $tag =~ "benchmarks" ]]; then
194+
INDUCTOR_BENCHMARKS=yes
195+
fi
194196
;;
195197
pytorch-linux-noble-rocm-alpha-py3)
196198
ANACONDA_PYTHON_VERSION=3.12
@@ -202,7 +204,6 @@ case "$tag" in
202204
KATEX=yes
203205
UCX_COMMIT=${_UCX_COMMIT}
204206
UCC_COMMIT=${_UCC_COMMIT}
205-
INDUCTOR_BENCHMARKS=yes
206207
PYTORCH_ROCM_ARCH="gfx90a;gfx942;gfx950"
207208
;;
208209
pytorch-linux-jammy-xpu-2025.0-py3)

‎.ci/docker/common/install_cpython.sh

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,8 +66,9 @@ function do_cpython_build {
6666
ln -s pip3 ${prefix}/bin/pip
6767
fi
6868
# install setuptools since python 3.12 is required to use distutils
69-
${prefix}/bin/pip install wheel==0.45.1 setuptools==80.9.0
70-
local abi_tag=$(${prefix}/bin/python -c "from wheel.pep425tags import get_abbr_impl, get_impl_ver, get_abi_tag; print('{0}{1}-{2}'.format(get_abbr_impl(), get_impl_ver(), get_abi_tag()))")
69+
# packaging is needed to create symlink since wheel no longer provides needed information
70+
${prefix}/bin/pip install packaging==25.0 wheel==0.45.1 setuptools==80.9.0
71+
local abi_tag=$(${prefix}/bin/python -c "from packaging.tags import interpreter_name, interpreter_version; import sysconfig ; from sysconfig import get_config_var; print('{0}{1}-{0}{1}{2}'.format(interpreter_name(), interpreter_version(), 't' if sysconfig.get_config_var('Py_GIL_DISABLED') else ''))")
7172
ln -sf ${prefix} /opt/python/${abi_tag}
7273
}
7374

‎.ci/docker/common/install_cuda.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,8 +68,8 @@ function install_nvshmem {
6868
# download, unpack, install
6969
wget -q "${url}"
7070
tar xf "${filename}.tar.gz"
71-
cp -a "libnvshmem/include/"* /usr/local/include/
72-
cp -a "libnvshmem/lib/"* /usr/local/lib/
71+
cp -a "libnvshmem/include/"* /usr/local/cuda/include/
72+
cp -a "libnvshmem/lib/"* /usr/local/cuda/lib64/
7373

7474
# cleanup
7575
cd ..

‎.ci/docker/common/install_inductor_benchmark_deps.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,4 +48,4 @@ install_huggingface
4848
install_timm
4949

5050
# Clean up
51-
conda_run pip uninstall -y torch torchvision torchaudio triton
51+
conda_run pip uninstall -y torch torchvision torchaudio triton torchao

‎.ci/docker/common/install_xpu.sh

Lines changed: 25 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -34,18 +34,27 @@ function install_ubuntu() {
3434

3535
# The xpu-smi packages
3636
apt-get install -y flex bison xpu-smi
37-
# Compute and Media Runtimes
38-
apt-get install -y \
39-
intel-opencl-icd intel-level-zero-gpu level-zero \
40-
intel-media-va-driver-non-free libmfx1 libmfxgen1 libvpl2 \
41-
libegl-mesa0 libegl1-mesa libegl1-mesa-dev libgbm1 libgl1-mesa-dev libgl1-mesa-dri \
42-
libglapi-mesa libgles2-mesa-dev libglx-mesa0 libigdgmm12 libxatracker2 mesa-va-drivers \
43-
mesa-vdpau-drivers mesa-vulkan-drivers va-driver-all vainfo hwinfo clinfo
44-
if [[ "${XPU_DRIVER_TYPE,,}" == "rolling" ]]; then
45-
apt-get install -y intel-ocloc
37+
38+
if [[ "${XPU_DRIVER_TYPE,,}" == "lts" ]]; then
39+
# Compute and Media Runtimes
40+
apt-get install -y \
41+
intel-opencl-icd intel-level-zero-gpu level-zero \
42+
intel-media-va-driver-non-free libmfx1 libmfxgen1 libvpl2 \
43+
libegl-mesa0 libegl1-mesa libegl1-mesa-dev libgbm1 libgl1-mesa-dev libgl1-mesa-dri \
44+
libglapi-mesa libgles2-mesa-dev libglx-mesa0 libigdgmm12 libxatracker2 mesa-va-drivers \
45+
mesa-vdpau-drivers mesa-vulkan-drivers va-driver-all vainfo hwinfo clinfo
46+
# Development Packages
47+
apt-get install -y libigc-dev intel-igc-cm libigdfcl-dev libigfxcmrt-dev level-zero-dev
48+
else # rolling driver
49+
apt-get install -y \
50+
intel-opencl-icd libze-intel-gpu1 libze1 \
51+
intel-media-va-driver-non-free libmfx-gen1 libvpl2 \
52+
libegl-mesa0 libegl1-mesa libegl1-mesa-dev libgbm1 libgl1-mesa-dev libgl1-mesa-dri \
53+
libglapi-mesa libglx-mesa0 libigdgmm12 libxatracker2 mesa-va-drivers \
54+
mesa-vdpau-drivers mesa-vulkan-drivers va-driver-all vainfo hwinfo clinfo intel-ocloc
55+
apt-get install -y libigc-dev intel-igc-cm libigdfcl-dev libigfxcmrt-dev libze-dev
4656
fi
47-
# Development Packages
48-
apt-get install -y libigc-dev intel-igc-cm libigdfcl-dev libigfxcmrt-dev level-zero-dev
57+
4958
# Install Intel Support Packages
5059
apt-get install -y ${XPU_PACKAGES}
5160

@@ -130,11 +139,11 @@ function install_sles() {
130139

131140
}
132141

133-
# Default use GPU driver LTS releases
134-
XPU_DRIVER_VERSION="/lts/2350"
135-
if [[ "${XPU_DRIVER_TYPE,,}" == "rolling" ]]; then
136-
# Use GPU driver rolling releases
137-
XPU_DRIVER_VERSION=""
142+
# Default use GPU driver rolling releases
143+
XPU_DRIVER_VERSION=""
144+
if [[ "${XPU_DRIVER_TYPE,,}" == "lts" ]]; then
145+
# Use GPU driver LTS releases
146+
XPU_DRIVER_VERSION="/lts/2350"
138147
fi
139148

140149
# Default use Intel® oneAPI Deep Learning Essentials 2025.0

‎.ci/docker/requirements-ci.txt

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -63,11 +63,12 @@ lark==0.12.0
6363
#Pinned versions: 0.12.0
6464
#test that import:
6565

66-
librosa>=0.6.2 ; python_version < "3.11"
67-
librosa==0.10.2 ; python_version == "3.12"
66+
librosa>=0.6.2 ; python_version < "3.11" and platform_machine != "s390x"
67+
librosa==0.10.2 ; python_version == "3.12" and platform_machine != "s390x"
6868
#Description: A python package for music and audio analysis
6969
#Pinned versions: >=0.6.2
7070
#test that import: test_spectral_ops.py
71+
#librosa depends on numba; disable it for s390x while numba is disabled too
7172

7273
#mkl #this breaks linux-bionic-rocm4.5-py3.7
7374
#Description: Intel oneAPI Math Kernel Library
@@ -110,14 +111,15 @@ ninja==1.11.1.3
110111
#Pinned versions: 1.11.1.3
111112
#test that import: run_test.py, test_cpp_extensions_aot.py,test_determination.py
112113

113-
numba==0.49.0 ; python_version < "3.9"
114-
numba==0.55.2 ; python_version == "3.9"
115-
numba==0.55.2 ; python_version == "3.10"
116-
numba==0.60.0 ; python_version == "3.12"
114+
numba==0.49.0 ; python_version < "3.9" and platform_machine != "s390x"
115+
numba==0.55.2 ; python_version == "3.9" and platform_machine != "s390x"
116+
numba==0.55.2 ; python_version == "3.10" and platform_machine != "s390x"
117+
numba==0.60.0 ; python_version == "3.12" and platform_machine != "s390x"
117118
#Description: Just-In-Time Compiler for Numerical Functions
118119
#Pinned versions: 0.54.1, 0.49.0, <=0.49.1
119120
#test that import: test_numba_integration.py
120121
#For numba issue see https://github.com/pytorch/pytorch/issues/51511
122+
#Need release > 0.61.2 for s390x due to https://github.com/numba/numba/pull/10073
121123

122124
#numpy
123125
#Description: Provides N-dimensional arrays and linear algebra
@@ -307,7 +309,7 @@ pytest-cpp==2.3.0
307309
#Pinned versions: 2.3.0
308310
#test that import:
309311

310-
z3-solver==4.15.1.0
312+
z3-solver==4.15.1.0 ; platform_machine != "s390x"
311313
#Description: The Z3 Theorem Prover Project
312314
#Pinned versions:
313315
#test that import:

‎.ci/manywheel/build_common.sh

Lines changed: 2 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -138,28 +138,11 @@ fi
138138

139139
echo "Calling setup.py bdist at $(date)"
140140

141-
if [[ "$USE_SPLIT_BUILD" == "true" ]]; then
142-
echo "Calling setup.py bdist_wheel for split build (BUILD_LIBTORCH_WHL)"
143-
time EXTRA_CAFFE2_CMAKE_FLAGS=${EXTRA_CAFFE2_CMAKE_FLAGS[@]} \
144-
BUILD_LIBTORCH_WHL=1 BUILD_PYTHON_ONLY=0 \
141+
time CMAKE_ARGS=${CMAKE_ARGS[@]} \
142+
EXTRA_CAFFE2_CMAKE_FLAGS=${EXTRA_CAFFE2_CMAKE_FLAGS[@]} \
145143
BUILD_LIBTORCH_CPU_WITH_DEBUG=$BUILD_DEBUG_INFO \
146144
USE_NCCL=${USE_NCCL} USE_RCCL=${USE_RCCL} USE_KINETO=${USE_KINETO} \
147145
python setup.py bdist_wheel -d /tmp/$WHEELHOUSE_DIR
148-
echo "Finished setup.py bdist_wheel for split build (BUILD_LIBTORCH_WHL)"
149-
echo "Calling setup.py bdist_wheel for split build (BUILD_PYTHON_ONLY)"
150-
time EXTRA_CAFFE2_CMAKE_FLAGS=${EXTRA_CAFFE2_CMAKE_FLAGS[@]} \
151-
BUILD_LIBTORCH_WHL=0 BUILD_PYTHON_ONLY=1 \
152-
BUILD_LIBTORCH_CPU_WITH_DEBUG=$BUILD_DEBUG_INFO \
153-
USE_NCCL=${USE_NCCL} USE_RCCL=${USE_RCCL} USE_KINETO=${USE_KINETO} \
154-
CMAKE_FRESH=1 python setup.py bdist_wheel -d /tmp/$WHEELHOUSE_DIR
155-
echo "Finished setup.py bdist_wheel for split build (BUILD_PYTHON_ONLY)"
156-
else
157-
time CMAKE_ARGS=${CMAKE_ARGS[@]} \
158-
EXTRA_CAFFE2_CMAKE_FLAGS=${EXTRA_CAFFE2_CMAKE_FLAGS[@]} \
159-
BUILD_LIBTORCH_CPU_WITH_DEBUG=$BUILD_DEBUG_INFO \
160-
USE_NCCL=${USE_NCCL} USE_RCCL=${USE_RCCL} USE_KINETO=${USE_KINETO} \
161-
python setup.py bdist_wheel -d /tmp/$WHEELHOUSE_DIR
162-
fi
163146
echo "Finished setup.py bdist at $(date)"
164147

165148
# Build libtorch packages
@@ -272,10 +255,6 @@ ls /tmp/$WHEELHOUSE_DIR
272255
mkdir -p "/$WHEELHOUSE_DIR"
273256
mv /tmp/$WHEELHOUSE_DIR/torch*linux*.whl /$WHEELHOUSE_DIR/
274257

275-
if [[ "$USE_SPLIT_BUILD" == "true" ]]; then
276-
mv /tmp/$WHEELHOUSE_DIR/torch_no_python*.whl /$WHEELHOUSE_DIR/ || true
277-
fi
278-
279258
if [[ -n "$BUILD_PYTHONLESS" ]]; then
280259
mkdir -p /$LIBTORCH_HOUSE_DIR
281260
mv /tmp/$LIBTORCH_HOUSE_DIR/*.zip /$LIBTORCH_HOUSE_DIR
@@ -452,16 +431,8 @@ if [[ -z "$BUILD_PYTHONLESS" ]]; then
452431
pushd $PYTORCH_ROOT/test
453432
454433
# Install the wheel for this Python version
455-
if [[ "$USE_SPLIT_BUILD" == "true" ]]; then
456-
pip uninstall -y "$TORCH_NO_PYTHON_PACKAGE_NAME" || true
457-
fi
458-
459434
pip uninstall -y "$TORCH_PACKAGE_NAME"
460435
461-
if [[ "$USE_SPLIT_BUILD" == "true" ]]; then
462-
pip install "$TORCH_NO_PYTHON_PACKAGE_NAME" --no-index -f /$WHEELHOUSE_DIR --no-dependencies -v
463-
fi
464-
465436
pip install "$TORCH_PACKAGE_NAME" --no-index -f /$WHEELHOUSE_DIR --no-dependencies -v
466437
467438
# Print info on the libraries installed in this wheel

‎.ci/pytorch/build.sh

Lines changed: 3 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -50,9 +50,6 @@ if [[ ${BUILD_ENVIRONMENT} == *"parallelnative"* ]]; then
5050
export ATEN_THREADING=NATIVE
5151
fi
5252

53-
# Enable LLVM dependency for TensorExpr testing
54-
export USE_LLVM=/opt/llvm
55-
export LLVM_DIR=/opt/llvm/lib/cmake/llvm
5653

5754
if ! which conda; then
5855
# In ROCm CIs, we are doing cross compilation on build machines with
@@ -176,7 +173,7 @@ fi
176173

177174
# We only build FlashAttention files for CUDA 8.0+, and they require large amounts of
178175
# memory to build and will OOM
179-
if [[ "$BUILD_ENVIRONMENT" == *cuda* ]] && [[ 1 -eq $(echo "${TORCH_CUDA_ARCH_LIST} >= 8.0" | bc) ]]; then
176+
if [[ "$BUILD_ENVIRONMENT" == *cuda* ]] && echo "${TORCH_CUDA_ARCH_LIST}" | tr ' ' '\n' | sed 's/$/>= 8.0/' | bc | grep -q 1; then
180177
export BUILD_CUSTOM_STEP="ninja -C build flash_attention -j 2"
181178
fi
182179

@@ -192,7 +189,6 @@ if [[ "$BUILD_ENVIRONMENT" == *-clang*-asan* ]]; then
192189
export USE_ASAN=1
193190
export REL_WITH_DEB_INFO=1
194191
export UBSAN_FLAGS="-fno-sanitize-recover=all"
195-
unset USE_LLVM
196192
fi
197193

198194
if [[ "${BUILD_ENVIRONMENT}" == *no-ops* ]]; then
@@ -265,22 +261,13 @@ else
265261

266262
WERROR=1 python setup.py clean
267263

268-
if [[ "$USE_SPLIT_BUILD" == "true" ]]; then
269-
python3 tools/packaging/split_wheel.py bdist_wheel
270-
else
271-
WERROR=1 python setup.py bdist_wheel
272-
fi
264+
WERROR=1 python setup.py bdist_wheel
273265
else
274266
python setup.py clean
275267
if [[ "$BUILD_ENVIRONMENT" == *xla* ]]; then
276268
source .ci/pytorch/install_cache_xla.sh
277269
fi
278-
if [[ "$USE_SPLIT_BUILD" == "true" ]]; then
279-
echo "USE_SPLIT_BUILD cannot be used with xla or rocm"
280-
exit 1
281-
else
282-
python setup.py bdist_wheel
283-
fi
270+
python setup.py bdist_wheel
284271
fi
285272
pip_install_whl "$(echo dist/*.whl)"
286273

‎.ci/pytorch/test.sh

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1051,20 +1051,10 @@ test_libtorch_api() {
10511051
mkdir -p $TEST_REPORTS_DIR
10521052

10531053
OMP_NUM_THREADS=2 TORCH_CPP_TEST_MNIST_PATH="${MNIST_DIR}" "$TORCH_BIN_DIR"/test_api --gtest_filter='-IMethodTest.*' --gtest_output=xml:$TEST_REPORTS_DIR/test_api.xml
1054-
"$TORCH_BIN_DIR"/test_tensorexpr --gtest_output=xml:$TEST_REPORTS_DIR/test_tensorexpr.xml
10551054
else
10561055
# Exclude IMethodTest that relies on torch::deploy, which will instead be ran in test_deploy
10571056
OMP_NUM_THREADS=2 TORCH_CPP_TEST_MNIST_PATH="${MNIST_DIR}" python test/run_test.py --cpp --verbose -i cpp/test_api -k "not IMethodTest"
10581057

1059-
# On s390x, pytorch is built without llvm.
1060-
# Even if it would be built with llvm, llvm currently doesn't support used features on s390x and
1061-
# test fails with errors like:
1062-
# JIT session error: Unsupported target machine architecture in ELF object pytorch-jitted-objectbuffer
1063-
# unknown file: Failure
1064-
# C++ exception with description "valOrErr INTERNAL ASSERT FAILED at "/var/lib/jenkins/workspace/torch/csrc/jit/tensorexpr/llvm_jit.h":34, please report a bug to PyTorch. Unexpected failure in LLVM JIT: Failed to materialize symbols: { (main, { func }) }
1065-
if [[ "${BUILD_ENVIRONMENT}" != *s390x* ]]; then
1066-
python test/run_test.py --cpp --verbose -i cpp/test_tensorexpr
1067-
fi
10681058
fi
10691059

10701060
# quantization is not fully supported on s390x yet

0 commit comments

Comments
 (0)