Skip to content

Commit 82d443a

Browse files
stas00sgugger
stas00
and
sgugger
authored
[core] implement support for run-time dependency version checking (huggingface#8645)
* implement support for run-time dependency version checking * try not escaping ! * use findall that works on py36 * small tweaks * autoformatter worship * simplify * shorter names * add support for non-versioned checks * add deps * revert * tokenizers not required, check version only if installed * make a proper distutils cmd and add make target * tqdm must be checked before tokenizers * workaround the DistributionNotFound peculiar setup * handle the rest of packages in setup.py * fully sync setup.py's install_requires - to check them all * nit * make install_requires more readable * typo * Update setup.py Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> * restyle * add types * simplify * simplify2 Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com>
1 parent a7d73cf commit 82d443a

File tree

8 files changed

+427
-77
lines changed

8 files changed

+427
-77
lines changed

‎Makefile

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
.PHONY: modified_only_fixup extra_quality_checks quality style fixup fix-copies test test-examples docs
1+
.PHONY: deps_table_update modified_only_fixup extra_quality_checks quality style fixup fix-copies test test-examples docs
22

33

44
check_dirs := examples tests src utils
@@ -14,9 +14,14 @@ modified_only_fixup:
1414
echo "No library .py files were modified"; \
1515
fi
1616

17+
# Update src/transformers/dependency_versions_table.py
18+
19+
deps_table_update:
20+
@python setup.py deps_table_update
21+
1722
# Check that source code meets quality standards
1823

19-
extra_quality_checks:
24+
extra_quality_checks: deps_table_update
2025
python utils/check_copies.py
2126
python utils/check_dummies.py
2227
python utils/check_repo.py
@@ -32,7 +37,7 @@ quality:
3237

3338
# Format source code automatically and check is there are any problems left that need manual fixing
3439

35-
style:
40+
style: deps_table_update
3641
black $(check_dirs)
3742
isort $(check_dirs)
3843
python utils/style_doc.py src/transformers docs/source --max_len 119

‎examples/lightning_base.py

Lines changed: 2 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,9 @@
44
from pathlib import Path
55
from typing import Any, Dict
66

7-
import packaging
87
import pytorch_lightning as pl
98
from pytorch_lightning.utilities import rank_zero_info
109

11-
import pkg_resources
1210
from transformers import (
1311
AdamW,
1412
AutoConfig,
@@ -30,21 +28,12 @@
3028
get_linear_schedule_with_warmup,
3129
get_polynomial_decay_schedule_with_warmup,
3230
)
31+
from transformers.utils.versions import require_version_examples
3332

3433

3534
logger = logging.getLogger(__name__)
3635

37-
38-
def require_min_ver(pkg, min_ver):
39-
got_ver = pkg_resources.get_distribution(pkg).version
40-
if packaging.version.parse(got_ver) < packaging.version.parse(min_ver):
41-
logger.warning(
42-
f"{pkg}>={min_ver} is required for a normal functioning of this module, but found {pkg}=={got_ver}. "
43-
"Try: pip install -r examples/requirements.txt"
44-
)
45-
46-
47-
require_min_ver("pytorch_lightning", "1.0.4")
36+
require_version_examples("pytorch_lightning>=1.0.4")
4837

4938
MODEL_MODES = {
5039
"base": AutoModel,

‎setup.py

Lines changed: 147 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,9 @@
4747
"""
4848

4949
import os
50+
import re
5051
import shutil
52+
from distutils.core import Command
5153
from pathlib import Path
5254

5355
from setuptools import find_packages, setup
@@ -69,54 +71,163 @@
6971
shutil.rmtree(stale_egg_info)
7072

7173

72-
extras = {}
73-
74-
extras["ja"] = ["fugashi>=1.0", "ipadic>=1.0.0,<2.0", "unidic_lite>=1.0.7", "unidic>=1.0.2"]
75-
extras["sklearn"] = ["scikit-learn"]
76-
77-
# keras2onnx and onnxconverter-common version is specific through a commit until 1.7.0 lands on pypi
78-
extras["tf"] = [
79-
"tensorflow>=2.0",
74+
# IMPORTANT:
75+
# 1. all dependencies should be listed here with their version requirements if any
76+
# 2. once modified, run: `make deps_table_update` to update src/transformers/dependency_versions_table.py
77+
_deps = [
78+
"black>=20.8b1",
79+
"cookiecutter==1.7.2",
80+
"dataclasses",
81+
"datasets",
82+
"faiss-cpu",
83+
"fastapi",
84+
"filelock",
85+
"flake8>=3.8.3",
86+
"flax==0.2.2",
87+
"fugashi>=1.0",
88+
"ipadic>=1.0.0,<2.0",
89+
"isort>=5.5.4",
90+
"jax>=0.2.0",
91+
"jaxlib==0.1.55",
92+
"keras2onnx",
93+
"numpy",
8094
"onnxconverter-common",
81-
"keras2onnx"
82-
# "onnxconverter-common @ git+git://github.com/microsoft/onnxconverter-common.git@f64ca15989b6dc95a1f3507ff6e4c395ba12dff5#egg=onnxconverter-common",
83-
# "keras2onnx @ git+git://github.com/onnx/keras-onnx.git@cbdc75cb950b16db7f0a67be96a278f8d2953b48#egg=keras2onnx",
84-
]
85-
extras["tf-cpu"] = [
95+
"onnxruntime-tools>=1.4.2",
96+
"onnxruntime>=1.4.0",
97+
"packaging",
98+
"parameterized",
99+
"protobuf",
100+
"psutil",
101+
"pydantic",
102+
"pytest",
103+
"pytest-xdist",
104+
"python>=3.6.0",
105+
"recommonmark",
106+
"regex!=2019.12.17",
107+
"requests",
108+
"sacremoses",
109+
"scikit-learn",
110+
"sentencepiece==0.1.91",
111+
"sphinx-copybutton",
112+
"sphinx-markdown-tables",
113+
"sphinx-rtd-theme==0.4.3", # sphinx-rtd-theme==0.5.0 introduced big changes in the style.
114+
"sphinx==3.2.1",
115+
"starlette",
86116
"tensorflow-cpu>=2.0",
87-
"onnxconverter-common",
88-
"keras2onnx"
89-
# "onnxconverter-common @ git+git://github.com/microsoft/onnxconverter-common.git@f64ca15989b6dc95a1f3507ff6e4c395ba12dff5#egg=onnxconverter-common",
90-
# "keras2onnx @ git+git://github.com/onnx/keras-onnx.git@cbdc75cb950b16db7f0a67be96a278f8d2953b48#egg=keras2onnx",
117+
"tensorflow>=2.0",
118+
"timeout-decorator",
119+
"tokenizers==0.9.4",
120+
"torch>=1.0",
121+
"tqdm>=4.27",
122+
"unidic>=1.0.2",
123+
"unidic_lite>=1.0.7",
124+
"uvicorn",
91125
]
92-
extras["torch"] = ["torch>=1.0"]
126+
127+
128+
# tokenizers: "tokenizers==0.9.4" lookup table
129+
# support non-versions file too so that they can be checked at run time
130+
deps = {b: a for a, b in (re.findall(r"^(([^!=<>]+)(?:[!=<>].*)?$)", x)[0] for x in _deps)}
131+
132+
133+
def deps_list(*pkgs):
134+
return [deps[pkg] for pkg in pkgs]
135+
136+
137+
class DepsTableUpdateCommand(Command):
138+
"""
139+
A custom distutils command that updates the dependency table.
140+
usage: python setup.py deps_table_update
141+
"""
142+
143+
description = "build runtime dependency table"
144+
user_options = [
145+
# format: (long option, short option, description).
146+
("dep-table-update", None, "updates src/transformers/dependency_versions_table.py"),
147+
]
148+
149+
def initialize_options(self):
150+
pass
151+
152+
def finalize_options(self):
153+
pass
154+
155+
def run(self):
156+
entries = "\n".join([f' "{k}": "{v}",' for k, v in deps.items()])
157+
content = [
158+
"# THIS FILE HAS BEEN AUTOGENERATED. To update:",
159+
"# 1. modify the `_deps` dict in setup.py",
160+
"# 2. run `make deps_table_update``",
161+
"deps = {",
162+
entries,
163+
"}",
164+
""
165+
]
166+
target = "src/transformers/dependency_versions_table.py"
167+
print(f"updating {target}")
168+
with open(target, "w") as f:
169+
f.write("\n".join(content))
170+
171+
172+
extras = {}
173+
174+
extras["ja"] = deps_list("fugashi", "ipadic", "unidic_lite", "unidic")
175+
extras["sklearn"] = deps_list("scikit-learn")
176+
177+
extras["tf"] = deps_list("tensorflow", "onnxconverter-common", "keras2onnx")
178+
extras["tf-cpu"] = deps_list("tensorflow-cpu", "onnxconverter-common", "keras2onnx")
179+
180+
extras["torch"] = deps_list("torch")
93181

94182
if os.name == "nt": # windows
95-
extras["retrieval"] = ["datasets"] # faiss is not supported on windows
96-
extras["flax"] = [] # jax is not supported on windows
183+
extras["retrieval"] = deps_list("datasets") # faiss is not supported on windows
184+
extras["flax"] = [] # jax is not supported on windows
97185
else:
98-
extras["retrieval"] = ["faiss-cpu", "datasets"]
99-
extras["flax"] = ["jaxlib==0.1.55", "jax>=0.2.0", "flax==0.2.2"]
100-
101-
extras["tokenizers"] = ["tokenizers==0.9.4"]
102-
extras["onnxruntime"] = ["onnxruntime>=1.4.0", "onnxruntime-tools>=1.4.2"]
103-
extras["modelcreation"] = ["cookiecutter==1.7.2"]
186+
extras["retrieval"] = deps_list("faiss-cpu", "datasets")
187+
extras["flax"] = deps_list("jax", "jaxlib", "flax")
104188

105-
extras["serving"] = ["pydantic", "uvicorn", "fastapi", "starlette"]
189+
extras["tokenizers"] = deps_list("tokenizers")
190+
extras["onnxruntime"] = deps_list("onnxruntime", "onnxruntime-tools")
191+
extras["modelcreation"] = deps_list("cookiecutter")
106192

107-
extras["sentencepiece"] = ["sentencepiece==0.1.91", "protobuf"]
108-
extras["retrieval"] = ["faiss-cpu", "datasets"]
109-
extras["testing"] = ["pytest", "pytest-xdist", "timeout-decorator", "parameterized", "psutil"] + extras["retrieval"] + extras["modelcreation"]
110-
# sphinx-rtd-theme==0.5.0 introduced big changes in the style.
111-
extras["docs"] = ["recommonmark", "sphinx==3.2.1", "sphinx-markdown-tables", "sphinx-rtd-theme==0.4.3", "sphinx-copybutton"]
112-
extras["quality"] = ["black >= 20.8b1", "isort >= 5.5.4", "flake8 >= 3.8.3"]
193+
extras["serving"] = deps_list("pydantic", "uvicorn", "fastapi", "starlette")
113194

195+
extras["sentencepiece"] = deps_list("sentencepiece", "protobuf")
196+
extras["retrieval"] = deps_list("faiss-cpu", "datasets")
197+
extras["testing"] = (
198+
deps_list("pytest", "pytest-xdist", "timeout-decorator", "parameterized", "psutil")
199+
+ extras["retrieval"]
200+
+ extras["modelcreation"]
201+
)
202+
extras["docs"] = deps_list("recommonmark", "sphinx", "sphinx-markdown-tables", "sphinx-rtd-theme", "sphinx-copybutton")
203+
extras["quality"] = deps_list("black", "isort", "flake8")
114204

115205
extras["all"] = extras["tf"] + extras["torch"] + extras["flax"] + extras["sentencepiece"] + extras["tokenizers"]
116206

117-
extras["dev"] = extras["all"] + extras["testing"] + extras["quality"] + extras["ja"] + extras["docs"] + extras["sklearn"] + extras["modelcreation"]
207+
extras["dev"] = (
208+
extras["all"]
209+
+ extras["testing"]
210+
+ extras["quality"]
211+
+ extras["ja"]
212+
+ extras["docs"]
213+
+ extras["sklearn"]
214+
+ extras["modelcreation"]
215+
)
118216

119217

218+
# when modifying the following list, make sure to update src/transformers/dependency_versions_check.py
219+
install_requires = [
220+
deps["dataclasses"] + ";python_version<'3.7'", # dataclasses for Python versions that don't have it
221+
deps["filelock"], # filesystem locks, e.g., to prevent parallel downloads
222+
deps["numpy"],
223+
deps["packaging"], # utilities from PyPA to e.g., compare versions
224+
deps["regex"], # for OpenAI GPT
225+
deps["requests"], # for downloading models over HTTPS
226+
deps["sacremoses"], # for XLM
227+
deps["tokenizers"],
228+
deps["tqdm"], # progress bars in model download and training scripts
229+
]
230+
120231
setup(
121232
name="transformers",
122233
version="4.0.0-rc-1",
@@ -130,27 +241,10 @@
130241
url="https://github.com/huggingface/transformers",
131242
package_dir={"": "src"},
132243
packages=find_packages("src"),
133-
install_requires=[
134-
"numpy",
135-
"tokenizers == 0.9.4",
136-
# dataclasses for Python versions that don't have it
137-
"dataclasses;python_version<'3.7'",
138-
# utilities from PyPA to e.g. compare versions
139-
"packaging",
140-
# filesystem locks e.g. to prevent parallel downloads
141-
"filelock",
142-
# for downloading models over HTTPS
143-
"requests",
144-
# progress bars in model download and training scripts
145-
"tqdm >= 4.27",
146-
# for OpenAI GPT
147-
"regex != 2019.12.17",
148-
# for XLM
149-
"sacremoses",
150-
],
151244
extras_require=extras,
152245
entry_points={"console_scripts": ["transformers-cli=transformers.commands.transformers_cli:main"]},
153246
python_requires=">=3.6.0",
247+
install_requires=install_requires,
154248
classifiers=[
155249
"Development Status :: 5 - Production/Stable",
156250
"Intended Audience :: Developers",
@@ -163,4 +257,5 @@
163257
"Programming Language :: Python :: 3.7",
164258
"Topic :: Scientific/Engineering :: Artificial Intelligence",
165259
],
260+
cmdclass={"deps_table_update": DepsTableUpdateCommand},
166261
)

‎src/transformers/__init__.py

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,7 @@
1717
absl.logging.set_stderrthreshold("info")
1818
absl.logging._warn_preinit_stderr = False
1919

20-
# Integrations: this needs to come before other ml imports
21-
# in order to allow any 3rd-party code to initialize properly
22-
from .integrations import ( # isort:skip
23-
is_comet_available,
24-
is_optuna_available,
25-
is_ray_available,
26-
is_tensorboard_available,
27-
is_wandb_available,
28-
)
20+
from . import dependency_versions_check
2921

3022
# Configuration
3123
from .configuration_utils import PretrainedConfig
@@ -203,6 +195,17 @@
203195
)
204196

205197

198+
# Integrations: this needs to come before other ml imports
199+
# in order to allow any 3rd-party code to initialize properly
200+
from .integrations import ( # isort:skip
201+
is_comet_available,
202+
is_optuna_available,
203+
is_ray_available,
204+
is_tensorboard_available,
205+
is_wandb_available,
206+
)
207+
208+
206209
if is_sentencepiece_available():
207210
from .models.albert import AlbertTokenizer
208211
from .models.bert_generation import BertGenerationTokenizer

‎src/transformers/dependency_versions_check.py

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
import sys
2+
3+
from .dependency_versions_table import deps
4+
from .utils.versions import require_version_core
5+
6+
7+
# define which module versions we always want to check at run time
8+
# (usually the ones defined in `install_requires` in setup.py)
9+
#
10+
# order specific notes:
11+
# - tqdm must be checked before tokenizers
12+
13+
pkgs_to_check_at_runtime = "python tqdm regex sacremoses requests packaging filelock numpy tokenizers".split()
14+
if sys.version_info < (3, 7):
15+
pkgs_to_check_at_runtime.append("dataclasses")
16+
17+
for pkg in pkgs_to_check_at_runtime:
18+
if pkg in deps:
19+
if pkg == "tokenizers":
20+
# must be loaded here, or else tqdm check may fail
21+
from .file_utils import is_tokenizers_available
22+
23+
if not is_tokenizers_available():
24+
continue # not required, check version only if installed
25+
26+
require_version_core(deps[pkg])
27+
else:
28+
raise ValueError(f"can't find {pkg} in {deps.keys()}, check dependency_versions_table.py")

0 commit comments

Comments
 (0)