From abed0a58929311475d74f48d6080c948661e0879 Mon Sep 17 00:00:00 2001 From: Victor Naumov Date: Mon, 6 Feb 2023 17:22:45 +0100 Subject: [PATCH 1/7] added support for sqlalchemy2 (#120) * added support for sqlalchemy2 * added declarative_base support for sqlalchemy 2.0 * fixed sqlalchemy.exc.MovedIn20Warning. using sqlalchemy.orm.declarative_base * make check happy * simplified the sqlalchemy check * making black happy * added support of deferred fields * yet another sqlalchemy check * made mypy happier --------- Co-authored-by: victor naumov --- devtools/prettier.py | 13 ++++++++++++- devtools/utils.py | 12 +++++++++++- tests/test_prettier.py | 6 +++++- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/devtools/prettier.py b/devtools/prettier.py index bb5974c..fe01128 100644 --- a/devtools/prettier.py +++ b/devtools/prettier.py @@ -12,6 +12,11 @@ cache = lru_cache() +try: + from sqlalchemy import inspect as sa_inspect # type: ignore +except ImportError: + sa_inspect = None + __all__ = 'PrettyFormat', 'pformat', 'pprint' MYPY = False if MYPY: @@ -239,8 +244,14 @@ def _format_dataclass(self, value: 'Any', _: str, indent_current: int, indent_ne self._format_fields(value, value.__dict__.items(), indent_current, indent_new) def _format_sqlalchemy_class(self, value: 'Any', _: str, indent_current: int, indent_new: int) -> None: + if sa_inspect is not None: + state = sa_inspect(value) + deferred = state.unloaded + else: + deferred = set() + fields = [ - (field, getattr(value, field)) + (field, getattr(value, field) if field not in deferred else "") for field in dir(value) if not (field.startswith('_') or field in ['metadata', 'registry']) ] diff --git a/devtools/utils.py b/devtools/utils.py index 994027e..c0ac1a3 100644 --- a/devtools/utils.py +++ b/devtools/utils.py @@ -149,13 +149,23 @@ class DataClassType(metaclass=MetaDataClassType): class MetaSQLAlchemyClassType(type): def __instancecheck__(self, instance: 'Any') -> bool: + try: + from sqlalchemy.orm import DeclarativeBase # type: ignore + except ImportError: + pass + else: + if isinstance(instance, DeclarativeBase): + return True + try: from sqlalchemy.ext.declarative import DeclarativeMeta # type: ignore except ImportError: - return False + pass else: return isinstance(instance.__class__, DeclarativeMeta) + return False + class SQLAlchemyClassType(metaclass=MetaSQLAlchemyClassType): pass diff --git a/tests/test_prettier.py b/tests/test_prettier.py index 6dea882..657e1b7 100644 --- a/tests/test_prettier.py +++ b/tests/test_prettier.py @@ -28,7 +28,11 @@ try: from sqlalchemy import Column, Integer, String - from sqlalchemy.ext.declarative import declarative_base + try: + from sqlalchemy.orm import declarative_base + except ImportError: + from sqlalchemy.ext.declarative import declarative_base + SQLAlchemyBase = declarative_base() except ImportError: SQLAlchemyBase = None From 8f087b210bc41a0a02be689c582050a796e19867 Mon Sep 17 00:00:00 2001 From: Samuel Colvin Date: Wed, 5 Apr 2023 12:41:16 +0100 Subject: [PATCH 2/7] switch to ruff (#124) * switch to ruff * revert tests/test_expr_render.py * fix pyproject.toml, etc. * switch to pinned dependencies * switch to 3.7 deps * skip some tests on 3.7, add pre-commit --- .github/workflows/ci.yml | 13 ++- .pre-commit-config.yaml | 25 +++++ HISTORY.md | 2 +- Makefile | 31 ++++-- README.md | 2 +- devtools/debug.py | 2 +- devtools/prettier.py | 6 +- devtools/utils.py | 6 +- docs/plugins.py | 2 +- docs/usage.md | 2 +- pyproject.toml | 19 ++-- requirements.txt | 3 - requirements/all.txt | 4 + requirements/docs.in | 7 ++ requirements/docs.txt | 67 ++++++++++++ requirements/linting.in | 5 + requirements/linting.txt | 34 +++++++ requirements/pyproject.txt | 12 +++ requirements/testing.in | 11 ++ requirements/testing.txt | 55 ++++++++++ tests/requirements-linting.txt | 6 -- tests/requirements.txt | 11 -- tests/test_custom_pretty.py | 5 +- tests/test_expr_render.py | 6 +- tests/test_main.py | 54 ++++------ tests/test_prettier.py | 181 ++++++++++++++------------------- 26 files changed, 378 insertions(+), 193 deletions(-) create mode 100644 .pre-commit-config.yaml delete mode 100644 requirements.txt create mode 100644 requirements/all.txt create mode 100644 requirements/docs.in create mode 100644 requirements/docs.txt create mode 100644 requirements/linting.in create mode 100644 requirements/linting.txt create mode 100644 requirements/pyproject.txt create mode 100644 requirements/testing.in create mode 100644 requirements/testing.txt delete mode 100644 tests/requirements-linting.txt delete mode 100644 tests/requirements.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7cd8862..ad2c060 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,10 +19,13 @@ jobs: with: python-version: '3.10' - - run: pip install -r tests/requirements-linting.txt - - run: pip install . + - run: pip install -r requirements/linting.txt -r requirements/pyproject.txt + + - run: mypy devtools - - run: make lint + - uses: pre-commit/action@v3.0.0 + with: + extra_args: --all-files --verbose test: name: test py${{ matrix.python-version }} on ${{ matrix.os }} @@ -47,7 +50,7 @@ jobs: with: python-version: ${{ matrix.python-version }} - - run: pip install -r tests/requirements.txt + - run: pip install -r requirements/testing.txt -r requirements/pyproject.txt - run: pip install . - run: pip freeze @@ -102,7 +105,7 @@ jobs: python-version: '3.10' - name: install - run: make install + run: pip install build twine - name: build run: python -m build diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..89b5ad4 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,25 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.3.0 + hooks: + - id: check-yaml + - id: check-toml + - id: end-of-file-fixer + - id: trailing-whitespace + +- repo: local + hooks: + - id: ruff + name: Ruff + entry: ruff + args: [--fix, --exit-non-zero-on-fix] + types: [python] + language: system + files: ^devtools/|^tests/ + - id: black + name: Black + entry: black + types: [python] + language: system + files: ^devtools/|^tests/ + exclude: test_expr_render.py diff --git a/HISTORY.md b/HISTORY.md index fcbebe3..2ebbb56 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -14,7 +14,7 @@ ## v0.7.0 (2021-09-03) -* switch to [`executing`](https://pypi.org/project/executing/) and [`asttokens`](https://pypi.org/project/asttokens/) +* switch to [`executing`](https://pypi.org/project/executing/) and [`asttokens`](https://pypi.org/project/asttokens/) for finding and printing debug arguments, #82, thanks @alexmojaki * correct changelog links, #76, thanks @Cielquan * return `debug()` arguments, #87 diff --git a/Makefile b/Makefile index 15225e9..3a464eb 100644 --- a/Makefile +++ b/Makefile @@ -1,23 +1,36 @@ .DEFAULT_GOAL := all -isort = isort devtools tests docs/plugins.py -black = black -S -l 120 --target-version py37 devtools docs/plugins.py +sources = devtools tests docs/plugins.py .PHONY: install install: - python -m pip install -U setuptools pip wheel twine build - pip install -U -r requirements.txt + python -m pip install -U pip pre-commit + pip install -U -r requirements/all.txt pip install -e . + pre-commit install + +.PHONY: refresh-lockfiles +refresh-lockfiles: + find requirements/ -name '*.txt' ! -name 'all.txt' -type f -delete + make update-lockfiles + +.PHONY: update-lockfiles +update-lockfiles: + @echo "Updating requirements/*.txt files using pip-compile" + pip-compile -q --resolver backtracking -o requirements/linting.txt requirements/linting.in + pip-compile -q --resolver backtracking -o requirements/testing.txt requirements/testing.in + pip-compile -q --resolver backtracking -o requirements/docs.txt requirements/docs.in + pip-compile -q --resolver backtracking -o requirements/pyproject.txt pyproject.toml + pip install --dry-run -r requirements/all.txt .PHONY: format format: - $(isort) - $(black) + black $(sources) + ruff $(sources) --fix --exit-zero .PHONY: lint lint: - flake8 --max-complexity 10 --max-line-length 120 --ignore E203,W503 devtools tests docs/plugins.py - $(isort) --check-only --df - $(black) --check --diff + black $(sources) --check --diff + ruff $(sources) mypy devtools .PHONY: test diff --git a/README.md b/README.md index 6a78b47..55e69e0 100644 --- a/README.md +++ b/README.md @@ -64,5 +64,5 @@ outputs: devtools can be used without `from devtools import debug` if you add `debug` into `__builtins__` in `sitecustomize.py`. -For instructions on adding `debug` to `__builtins__`, +For instructions on adding `debug` to `__builtins__`, see the [installation docs](https://python-devtools.helpmanual.io/usage/#usage-without-import). diff --git a/devtools/debug.py b/devtools/debug.py index 657859c..5ea836a 100644 --- a/devtools/debug.py +++ b/devtools/debug.py @@ -178,7 +178,7 @@ def _process(self, args: 'Any', kwargs: 'Any') -> DebugOutput: ex = source.executing(call_frame) function = ex.code_qualname() if not ex.node: - warning = "executing failed to find the calling node" + warning = 'executing failed to find the calling node' arguments = list(self._args_inspection_failed(args, kwargs)) else: arguments = list(self._process_args(ex, args, kwargs)) diff --git a/devtools/prettier.py b/devtools/prettier.py index fe01128..7489d33 100644 --- a/devtools/prettier.py +++ b/devtools/prettier.py @@ -13,9 +13,9 @@ cache = lru_cache() try: - from sqlalchemy import inspect as sa_inspect # type: ignore + from sqlalchemy import inspect as sa_inspect except ImportError: - sa_inspect = None + sa_inspect = None # type: ignore[assignment] __all__ = 'PrettyFormat', 'pformat', 'pprint' MYPY = False @@ -251,7 +251,7 @@ def _format_sqlalchemy_class(self, value: 'Any', _: str, indent_current: int, in deferred = set() fields = [ - (field, getattr(value, field) if field not in deferred else "") + (field, getattr(value, field) if field not in deferred else '') for field in dir(value) if not (field.startswith('_') or field in ['metadata', 'registry']) ] diff --git a/devtools/utils.py b/devtools/utils.py index c0ac1a3..2a96765 100644 --- a/devtools/utils.py +++ b/devtools/utils.py @@ -93,7 +93,7 @@ def _set_conout_mode(new_mode, mask=0xFFFFFFFF): mode = mask = ENABLE_VIRTUAL_TERMINAL_PROCESSING try: _set_conout_mode(mode, mask) - except WindowsError as e: # type: ignore + except OSError as e: if e.winerror == ERROR_INVALID_PARAMETER: return False raise @@ -150,7 +150,7 @@ class DataClassType(metaclass=MetaDataClassType): class MetaSQLAlchemyClassType(type): def __instancecheck__(self, instance: 'Any') -> bool: try: - from sqlalchemy.orm import DeclarativeBase # type: ignore + from sqlalchemy.orm import DeclarativeBase except ImportError: pass else: @@ -158,7 +158,7 @@ def __instancecheck__(self, instance: 'Any') -> bool: return True try: - from sqlalchemy.ext.declarative import DeclarativeMeta # type: ignore + from sqlalchemy.ext.declarative import DeclarativeMeta except ImportError: pass else: diff --git a/docs/plugins.py b/docs/plugins.py index aa46876..5f40bf1 100755 --- a/docs/plugins.py +++ b/docs/plugins.py @@ -55,7 +55,7 @@ def gen_examples_html(m: re.Match) -> str: conv = Ansi2HTMLConverter() name = THIS_DIR / Path(m.group(1)) - logger.info("running %s to generate HTML...", name) + logger.info('running %s to generate HTML...', name) p = subprocess.run((sys.executable, str(name)), stdout=subprocess.PIPE, check=True) html = conv.convert(p.stdout.decode(), full=False).strip('\r\n') html = html.replace('docs/build/../examples/', '') diff --git a/docs/usage.md b/docs/usage.md index 68bb6cb..0388a2b 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -96,7 +96,7 @@ Two ways to do this: ### Automatic install !!! warning - This is experimental, please [create an issue](https://github.com/samuelcolvin/python-devtools/issues) + This is experimental, please [create an issue](https://github.com/samuelcolvin/python-devtools/issues) if you encounter any problems. To install `debug` into `__builtins__` automatically, run: diff --git a/pyproject.toml b/pyproject.toml index 87bff20..193aabc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,18 +71,21 @@ exclude_lines = [ [tool.black] color = true line-length = 120 -target-version = ['py37', 'py38', 'py39', 'py310'] +target-version = ['py37', 'py38', 'py39', 'py310', 'py311'] skip-string-normalization = true +extend-exclude = ['tests/test_expr_render.py'] -[tool.isort] -line_length = 120 -multi_line_output = 3 -include_trailing_comma = true -force_grid_wrap = 0 -combine_as_imports = true -color_output = true +[tool.ruff] +line-length = 120 +exclude = ['cases_update'] +extend-select = ['Q', 'RUF100', 'C90', 'UP', 'I'] +flake8-quotes = {inline-quotes = 'single', multiline-quotes = 'double'} +mccabe = { max-complexity = 14 } +isort = { known-first-party = ['devtools'] } +target-version = 'py37' [tool.mypy] +show_error_codes = true strict = true warn_return_any = false diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 357dec2..0000000 --- a/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ --r docs/requirements.txt --r tests/requirements-linting.txt --r tests/requirements.txt diff --git a/requirements/all.txt b/requirements/all.txt new file mode 100644 index 0000000..3e6af75 --- /dev/null +++ b/requirements/all.txt @@ -0,0 +1,4 @@ +-r ./docs.txt +-r ./linting.txt +-r ./testing.txt +-r ./pyproject.txt diff --git a/requirements/docs.in b/requirements/docs.in new file mode 100644 index 0000000..f4c60e6 --- /dev/null +++ b/requirements/docs.in @@ -0,0 +1,7 @@ +ansi2html==1.8.0 +mkdocs==1.3.1 +mkdocs-exclude==1.0.2 +mkdocs-material==8.3.9 +mkdocs-simple-hooks==0.1.5 +markdown-include==0.7.0 +pygments diff --git a/requirements/docs.txt b/requirements/docs.txt new file mode 100644 index 0000000..d39c76c --- /dev/null +++ b/requirements/docs.txt @@ -0,0 +1,67 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --output-file=requirements/docs.txt --resolver=backtracking requirements/docs.in +# +ansi2html==1.8.0 + # via -r requirements/docs.in +click==8.1.3 + # via mkdocs +ghp-import==2.1.0 + # via mkdocs +importlib-metadata==6.1.0 + # via mkdocs +jinja2==3.1.2 + # via + # mkdocs + # mkdocs-material +markdown==3.3.7 + # via + # markdown-include + # mkdocs + # mkdocs-material + # pymdown-extensions +markdown-include==0.7.0 + # via -r requirements/docs.in +markupsafe==2.1.2 + # via jinja2 +mergedeep==1.3.4 + # via mkdocs +mkdocs==1.3.1 + # via + # -r requirements/docs.in + # mkdocs-exclude + # mkdocs-material + # mkdocs-simple-hooks +mkdocs-exclude==1.0.2 + # via -r requirements/docs.in +mkdocs-material==8.3.9 + # via -r requirements/docs.in +mkdocs-material-extensions==1.1.1 + # via mkdocs-material +mkdocs-simple-hooks==0.1.5 + # via -r requirements/docs.in +packaging==23.0 + # via mkdocs +pygments==2.14.0 + # via + # -r requirements/docs.in + # mkdocs-material +pymdown-extensions==9.10 + # via mkdocs-material +python-dateutil==2.8.2 + # via ghp-import +pyyaml==6.0 + # via + # mkdocs + # pymdown-extensions + # pyyaml-env-tag +pyyaml-env-tag==0.1 + # via mkdocs +six==1.16.0 + # via python-dateutil +watchdog==3.0.0 + # via mkdocs +zipp==3.15.0 + # via importlib-metadata diff --git a/requirements/linting.in b/requirements/linting.in new file mode 100644 index 0000000..44df076 --- /dev/null +++ b/requirements/linting.in @@ -0,0 +1,5 @@ +black +mypy==0.971 +ruff +# required so mypy can find stubs +sqlalchemy diff --git a/requirements/linting.txt b/requirements/linting.txt new file mode 100644 index 0000000..92279cd --- /dev/null +++ b/requirements/linting.txt @@ -0,0 +1,34 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --output-file=requirements/linting.txt --resolver=backtracking requirements/linting.in +# +black==23.3.0 + # via -r requirements/linting.in +click==8.1.3 + # via black +mypy==0.971 + # via -r requirements/linting.in +mypy-extensions==1.0.0 + # via + # black + # mypy +packaging==23.0 + # via black +pathspec==0.11.1 + # via black +platformdirs==3.2.0 + # via black +ruff==0.0.261 + # via -r requirements/linting.in +sqlalchemy==2.0.8 + # via -r requirements/linting.in +tomli==2.0.1 + # via + # black + # mypy +typing-extensions==4.5.0 + # via + # mypy + # sqlalchemy diff --git a/requirements/pyproject.txt b/requirements/pyproject.txt new file mode 100644 index 0000000..d385c38 --- /dev/null +++ b/requirements/pyproject.txt @@ -0,0 +1,12 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --output-file=requirements/pyproject.txt --resolver=backtracking pyproject.toml +# +asttokens==2.2.1 + # via devtools (pyproject.toml) +executing==1.2.0 + # via devtools (pyproject.toml) +six==1.16.0 + # via asttokens diff --git a/requirements/testing.in b/requirements/testing.in new file mode 100644 index 0000000..7b1ad14 --- /dev/null +++ b/requirements/testing.in @@ -0,0 +1,11 @@ +coverage[toml] +pygments +pytest +pytest-mock +pytest-pretty +# these packages are used in tests so install the latest version +pydantic +asyncpg +numpy; python_version>='3.8' +multidict; python_version>='3.8' +sqlalchemy diff --git a/requirements/testing.txt b/requirements/testing.txt new file mode 100644 index 0000000..b62eaba --- /dev/null +++ b/requirements/testing.txt @@ -0,0 +1,55 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --output-file=requirements/testing.txt --resolver=backtracking requirements/testing.in +# +asyncpg==0.27.0 + # via -r requirements/testing.in +attrs==22.2.0 + # via pytest +coverage[toml]==7.2.2 + # via -r requirements/testing.in +exceptiongroup==1.1.1 + # via pytest +iniconfig==2.0.0 + # via pytest +markdown-it-py==2.2.0 + # via rich +mdurl==0.1.2 + # via markdown-it-py +multidict==6.0.4 ; python_version >= "3.8" + # via -r requirements/testing.in +numpy==1.24.2 ; python_version >= "3.8" + # via -r requirements/testing.in +packaging==23.0 + # via pytest +pluggy==1.0.0 + # via pytest +pydantic==1.10.7 + # via -r requirements/testing.in +pygments==2.14.0 + # via + # -r requirements/testing.in + # rich +pytest==7.2.2 + # via + # -r requirements/testing.in + # pytest-mock + # pytest-pretty +pytest-mock==3.10.0 + # via -r requirements/testing.in +pytest-pretty==1.1.1 + # via -r requirements/testing.in +rich==13.3.3 + # via pytest-pretty +sqlalchemy==2.0.8 + # via -r requirements/testing.in +tomli==2.0.1 + # via + # coverage + # pytest +typing-extensions==4.5.0 + # via + # pydantic + # sqlalchemy diff --git a/tests/requirements-linting.txt b/tests/requirements-linting.txt deleted file mode 100644 index b84d0b2..0000000 --- a/tests/requirements-linting.txt +++ /dev/null @@ -1,6 +0,0 @@ -black==22.6.0 -flake8==4.0.1 -isort[colors]==5.10.1 -mypy==0.971 -pycodestyle==2.8.0 -pyflakes==2.4.0 diff --git a/tests/requirements.txt b/tests/requirements.txt deleted file mode 100644 index cd9c6b2..0000000 --- a/tests/requirements.txt +++ /dev/null @@ -1,11 +0,0 @@ -coverage[toml]==6.5.0 -Pygments==2.13.0 -pytest==7.2.0 -pytest-mock==3.10.0 -pytest-pretty==0.0.1 -# these packages are used in tests so install the latest version -pydantic -asyncpg -numpy -multidict -sqlalchemy diff --git a/tests/test_custom_pretty.py b/tests/test_custom_pretty.py index d63392b..552e5d3 100644 --- a/tests/test_custom_pretty.py +++ b/tests/test_custom_pretty.py @@ -23,12 +23,15 @@ def __pretty__(self, fmt, **kwargs): my_cls = CustomCls() v = pformat(my_cls) - assert v == """\ + assert ( + v + == """\ Thing( [], [0], [0, 1], )""" + ) def test_skip(): diff --git a/tests/test_expr_render.py b/tests/test_expr_render.py index 6b30bad..eed9469 100644 --- a/tests/test_expr_render.py +++ b/tests/test_expr_render.py @@ -48,14 +48,14 @@ def test_exotic_types(): (a for a in aa), ) s = normalise_output(str(v)) - print('\n---\n{}\n---'.format(v)) + print(f'\n---\n{v}\n---') # Generator expression source changed in 3.8 to include parentheses, see: # https://github.com/gristlabs/asttokens/pull/50 # https://bugs.python.org/issue31241 - genexpr_source = "a for a in aa" + genexpr_source = 'a for a in aa' if sys.version_info[:2] > (3, 7): - genexpr_source = f"({genexpr_source})" + genexpr_source = f'({genexpr_source})' assert ( "tests/test_expr_render.py: test_exotic_types\n" diff --git a/tests/test_main.py b/tests/test_main.py index f0cce7d..1057313 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -2,7 +2,7 @@ import sys from collections.abc import Generator from pathlib import Path -from subprocess import PIPE, run +from subprocess import run import pytest @@ -19,9 +19,7 @@ def test_print(capsys): stdout, stderr = capsys.readouterr() print(stdout) assert normalise_output(stdout) == ( - 'tests/test_main.py: test_print\n' - ' a: 1 (int)\n' - ' b: 2 (int)\n' + 'tests/test_main.py: test_print\n' ' a: 1 (int)\n' ' b: 2 (int)\n' ) assert stderr == '' assert result == (1, 2) @@ -64,7 +62,7 @@ def test_print_generator(capsys): def test_format(): a = b'i might bite' - b = "hello this is a test" + b = 'hello this is a test' v = debug.format(a, b) s = normalise_output(str(v)) print(s) @@ -81,7 +79,8 @@ def test_format(): ) def test_print_subprocess(tmpdir): f = tmpdir.join('test.py') - f.write("""\ + f.write( + """\ from devtools import debug def test_func(v): @@ -92,9 +91,10 @@ def test_func(v): debug(foobar) test_func(42) print('debug run.') - """) + """ + ) env = {'PYTHONPATH': str(Path(__file__).parent.parent.resolve())} - p = run([sys.executable, str(f)], stdout=PIPE, stderr=PIPE, universal_newlines=True, env=env) + p = run([sys.executable, str(f)], capture_output=True, text=True, env=env) assert p.stderr == '' assert p.returncode == 0, (p.stderr, p.stdout) assert p.stdout.replace(str(f), '/path/to/test.py') == ( @@ -113,10 +113,10 @@ def test_odd_path(mocker): mocked_relative_to = mocker.patch('pathlib.Path.relative_to') mocked_relative_to.side_effect = ValueError() v = debug.format('test') - if sys.platform == "win32": - pattern = r"\w:\\.*?\\" + if sys.platform == 'win32': + pattern = r'\w:\\.*?\\' else: - pattern = r"/.*?/" + pattern = r'/.*?/' pattern += r"test_main.py:\d{2,} test_odd_path\n 'test' \(str\) len=4" assert re.search(pattern, str(v)), v @@ -129,10 +129,7 @@ def test_small_call_frame(): 3, ) assert normalise_output(str(v)) == ( - 'tests/test_main.py: test_small_call_frame\n' - ' 1 (int)\n' - ' 2 (int)\n' - ' 3 (int)' + 'tests/test_main.py: test_small_call_frame\n' ' 1 (int)\n' ' 2 (int)\n' ' 3 (int)' ) @@ -143,12 +140,9 @@ def test_small_call_frame_warning(): 2, 3, ) - print('\n---\n{}\n---'.format(v)) + print(f'\n---\n{v}\n---') assert normalise_output(str(v)) == ( - 'tests/test_main.py: test_small_call_frame_warning\n' - ' 1 (int)\n' - ' 2 (int)\n' - ' 3 (int)' + 'tests/test_main.py: test_small_call_frame_warning\n' ' 1 (int)\n' ' 2 (int)\n' ' 3 (int)' ) @@ -171,7 +165,7 @@ def test_kwargs_orderless(): v = debug.format(first=a, second='literal') s = normalise_output(str(v)) assert set(s.split('\n')) == { - "tests/test_main.py: test_kwargs_orderless", + 'tests/test_main.py: test_kwargs_orderless', " first: 'variable' (str) len=8 variable=a", " second: 'literal' (str) len=7", } @@ -181,10 +175,7 @@ def test_simple_vars(): v = debug.format('test', 1, 2) s = normalise_output(str(v)) assert s == ( - "tests/test_main.py: test_simple_vars\n" - " 'test' (str) len=4\n" - " 1 (int)\n" - " 2 (int)" + "tests/test_main.py: test_simple_vars\n" " 'test' (str) len=4\n" " 1 (int)\n" " 2 (int)" ) r = normalise_output(repr(v)) assert r == ( @@ -222,18 +213,14 @@ def test_eval_kwargs(): v = eval('debug.format(1, apple="pear")') assert set(str(v).split('\n')) == { - ":1 (no code context for debug call, code inspection impossible)", - " 1 (int)", + ':1 (no code context for debug call, code inspection impossible)', + ' 1 (int)', " apple: 'pear' (str) len=4", } def test_exec(capsys): - exec( - 'a = 1\n' - 'b = 2\n' - 'debug(b, a + b)' - ) + exec('a = 1\n' 'b = 2\n' 'debug(b, a + b)') stdout, stderr = capsys.readouterr() assert stdout == ( @@ -314,8 +301,7 @@ def test_multiple_debugs(): v = debug.format([i * 2 for i in range(2)]) s = normalise_output(str(v)) assert s == ( - 'tests/test_main.py: test_multiple_debugs\n' - ' [i * 2 for i in range(2)]: [0, 2] (list) len=2' + 'tests/test_main.py: test_multiple_debugs\n' ' [i * 2 for i in range(2)]: [0, 2] (list) len=2' ) diff --git a/tests/test_prettier.py b/tests/test_prettier.py index 657e1b7..364447b 100644 --- a/tests/test_prettier.py +++ b/tests/test_prettier.py @@ -28,6 +28,7 @@ try: from sqlalchemy import Column, Integer, String + try: from sqlalchemy.orm import declarative_base except ImportError: @@ -41,21 +42,13 @@ def test_dict(): v = pformat({1: 2, 3: 4}) print(v) - assert v == ( - '{\n' - ' 1: 2,\n' - ' 3: 4,\n' - '}') + assert v == ('{\n' ' 1: 2,\n' ' 3: 4,\n' '}') def test_print(capsys): pprint({1: 2, 3: 4}) stdout, stderr = capsys.readouterr() - assert strip_ansi(stdout) == ( - '{\n' - ' 1: 2,\n' - ' 3: 4,\n' - '}\n') + assert strip_ansi(stdout) == ('{\n' ' 1: 2,\n' ' 3: 4,\n' '}\n') assert stderr == '' @@ -68,64 +61,33 @@ def test_colours(): def test_list(): v = pformat(list(range(6))) - assert v == ( - '[\n' - ' 0,\n' - ' 1,\n' - ' 2,\n' - ' 3,\n' - ' 4,\n' - ' 5,\n' - ']') + assert v == ('[\n' ' 0,\n' ' 1,\n' ' 2,\n' ' 3,\n' ' 4,\n' ' 5,\n' ']') def test_set(): v = pformat(set(range(5))) - assert v == ( - '{\n' - ' 0,\n' - ' 1,\n' - ' 2,\n' - ' 3,\n' - ' 4,\n' - '}') + assert v == ('{\n' ' 0,\n' ' 1,\n' ' 2,\n' ' 3,\n' ' 4,\n' '}') def test_tuple(): v = pformat(tuple(range(5))) - assert v == ( - '(\n' - ' 0,\n' - ' 1,\n' - ' 2,\n' - ' 3,\n' - ' 4,\n' - ')') + assert v == ('(\n' ' 0,\n' ' 1,\n' ' 2,\n' ' 3,\n' ' 4,\n' ')') def test_generator(): - v = pformat((i for i in range(3))) - assert v == ( - '(\n' - ' 0,\n' - ' 1,\n' - ' 2,\n' - ')') + v = pformat(i for i in range(3)) + assert v == ('(\n' ' 0,\n' ' 1,\n' ' 2,\n' ')') def test_named_tuple(): f = namedtuple('Foobar', ['foo', 'bar', 'spam']) v = pformat(f('x', 'y', 1)) - assert v == ("Foobar(\n" - " foo='x',\n" - " bar='y',\n" - " spam=1,\n" - ")") + assert v == ("Foobar(\n" " foo='x',\n" " bar='y',\n" " spam=1,\n" ")") def test_generator_no_yield(): pformat_ = PrettyFormat(yield_from_generators=False) - v = pformat_((i for i in range(3))) + v = pformat_(i for i in range(3)) assert v.startswith('. at ') @@ -157,7 +119,9 @@ def test_str_repr(): def test_bytes(): pformat_ = PrettyFormat(width=12) v = pformat_(string.ascii_lowercase.encode()) - assert v == """( + assert ( + v + == """( b'abcde' b'fghij' b'klmno' @@ -165,6 +129,7 @@ def test_bytes(): b'uvwxy' b'z' )""" + ) def test_short_bytes(): @@ -174,40 +139,52 @@ def test_short_bytes(): def test_bytearray(): pformat_ = PrettyFormat(width=18) v = pformat_(bytearray(string.ascii_lowercase.encode())) - assert v == """\ + assert ( + v + == """\ bytearray( b'abcdefghijk' b'lmnopqrstuv' b'wxyz' )""" + ) def test_bytearray_short(): v = pformat(bytearray(b'boo')) - assert v == """\ + assert ( + v + == """\ bytearray( b'boo' )""" + ) def test_map(): v = pformat(map(str.strip, ['x', 'y ', ' z'])) - assert v == """\ + assert ( + v + == """\ map( 'x', 'y', 'z', )""" + ) def test_filter(): v = pformat(filter(None, [1, 2, False, 3])) - assert v == """\ + assert ( + v + == """\ filter( 1, 2, 3, )""" + ) def test_counter(): @@ -216,11 +193,14 @@ def test_counter(): c['x'] += 1 c['y'] += 1 v = pformat(c) - assert v == """\ + assert ( + v + == """\ """ + ) def test_dataclass(): @@ -232,7 +212,9 @@ class FooDataclass: f = FooDataclass(123, [1, 2, 3, 4]) v = pformat(f) print(v) - assert v == """\ + assert ( + v + == """\ FooDataclass( x=123, y=[ @@ -242,6 +224,7 @@ class FooDataclass: 4, ], )""" + ) def test_nested_dataclasses(): @@ -258,68 +241,78 @@ class BarDataclass: b = BarDataclass(10.0, f) v = pformat(b) print(v) - assert v == """\ + assert ( + v + == """\ BarDataclass( a=10.0, b=FooDataclass( x=123, ), )""" + ) @pytest.mark.skipif(numpy is None, reason='numpy not installed') def test_indent_numpy(): v = pformat({'numpy test': numpy.array(range(20))}) - assert v == """{ + assert ( + v + == """{ 'numpy test': ( array([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]) ), }""" + ) @pytest.mark.skipif(numpy is None, reason='numpy not installed') def test_indent_numpy_short(): v = pformat({'numpy test': numpy.array(range(10))}) - assert v == """{ + assert ( + v + == """{ 'numpy test': array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]), }""" + ) def test_ordered_dict(): v = pformat(OrderedDict([(1, 2), (3, 4), (5, 6)])) print(v) - assert v == """\ + assert ( + v + == """\ OrderedDict([ (1, 2), (3, 4), (5, 6), ])""" + ) def test_frozenset(): v = pformat(frozenset(range(3))) print(v) - assert v == """\ + assert ( + v + == """\ frozenset({ 0, 1, 2, })""" + ) def test_deep_objects(): f = namedtuple('Foobar', ['foo', 'bar', 'spam']) - v = pformat(( - ( - f('x', 'y', OrderedDict([(1, 2), (3, 4), (5, 6)])), - frozenset(range(3)), - [1, 2, {1: 2}] - ), - {1, 2, 3} - )) + v = pformat(((f('x', 'y', OrderedDict([(1, 2), (3, 4), (5, 6)])), frozenset(range(3)), [1, 2, {1: 2}]), {1, 2, 3})) print(v) - assert v == """\ + assert ( + v + == """\ ( ( Foobar( @@ -344,6 +337,7 @@ def test_deep_objects(): ), {1, 2, 3}, )""" + ) def test_call_args(): @@ -351,11 +345,14 @@ def test_call_args(): m(1, 2, 3, a=4) v = pformat(m.call_args) - assert v == """\ + assert ( + v + == """\ _Call( _fields=(1, 2, 3), {'a': 4}, )""" + ) @pytest.mark.skipif(MultiDict is None, reason='MultiDict not installed') @@ -364,11 +361,11 @@ def test_multidict(): d.add('b', 3) v = pformat(d) assert set(v.split('\n')) == { - "", + '})>', } @@ -376,10 +373,10 @@ def test_multidict(): def test_cimultidict(): v = pformat(CIMultiDict({'a': 1, 'b': 2})) assert set(v.split('\n')) == { - "", + '})>', } @@ -399,21 +396,11 @@ def __init__(self): def test_dir(): - assert pformat(vars(Foo())) == ( - "{\n" - " 'b': 2,\n" - " 'c': 3,\n" - "}" - ) + assert pformat(vars(Foo())) == ("{\n" " 'b': 2,\n" " 'c': 3,\n" "}") def test_instance_dict(): - assert pformat(Foo().__dict__) == ( - "{\n" - " 'b': 2,\n" - " 'c': 3,\n" - "}" - ) + assert pformat(Foo().__dict__) == ("{\n" " 'b': 2,\n" " 'c': 3,\n" "}") def test_class_dict(): @@ -434,25 +421,14 @@ def items(self): def __getitem__(self, item): return self._d[item] - assert pformat(Dictlike()) == ( - "" - ) + assert pformat(Dictlike()) == ("") @pytest.mark.skipif(Record is None, reason='asyncpg not installed') def test_asyncpg_record(): r = Record({'a': 0, 'b': 1}, (41, 42)) assert dict(r) == {'a': 41, 'b': 42} - assert pformat(r) == ( - "" - ) + assert pformat(r) == ("") def test_dict_type(): @@ -467,11 +443,12 @@ class User(SQLAlchemyBase): name = Column(String) fullname = Column(String) nickname = Column(String) + user = User() user.id = 1 - user.name = "Test" - user.fullname = "Test For SQLAlchemy" - user.nickname = "test" + user.name = 'Test' + user.fullname = 'Test For SQLAlchemy' + user.nickname = 'test' assert pformat(user) == ( "User(\n" " fullname='Test For SQLAlchemy',\n" From 3d5ff6597fe3fc176dfc83062a9cf4e1203cb83a Mon Sep 17 00:00:00 2001 From: Samuel Colvin Date: Wed, 5 Apr 2023 12:45:32 +0100 Subject: [PATCH 3/7] update licence and history --- HISTORY.md | 9 +++++++++ LICENSE | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 2ebbb56..1aa231e 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,12 @@ +## v0.10.0 (2022-11-28) + +* Use secure builtins standard module, instead of the `__builtins__` by @0xsirsaif in #109 +* upgrade executing to fix 3.10 by @samuelcolvin in #110 +* Fix windows build by @samuelcolvin in #111 +* Allow executing dependency to be >1.0.0 by @staticf0x in #115 +* more precise timer summary by @banteg in #113 +* Python 3.11 by @samuelcolvin in #118 + ## v0.9.0 (2022-07-26) * fix format of nested dataclasses, #99 thanks @aliereno diff --git a/LICENSE b/LICENSE index 3338ce9..bdd9b15 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2017 Samuel Colvin +Copyright (c) 2017 to present Samuel Colvin Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal From f0e0fb2b139e1980c87363a754b6a42bb1a9fbc4 Mon Sep 17 00:00:00 2001 From: Samuel Colvin Date: Wed, 5 Apr 2023 13:48:42 +0100 Subject: [PATCH 4/7] support displaying ast types (#125) * support displaying ast types * support 3.7 & 3.8 --- devtools/prettier.py | 13 +++++++++++++ requirements/testing.in | 4 +++- requirements/testing.txt | 2 +- tests/test_prettier.py | 25 +++++++++++++++++++++++++ 4 files changed, 42 insertions(+), 2 deletions(-) diff --git a/devtools/prettier.py b/devtools/prettier.py index 7489d33..e378030 100644 --- a/devtools/prettier.py +++ b/devtools/prettier.py @@ -1,3 +1,4 @@ +import ast import io import os from collections import OrderedDict @@ -80,6 +81,7 @@ def __init__( (bytearray, self._format_bytearray), (generator_types, self._format_generator), # put these last as the check can be slow + (ast.AST, self._format_ast_expression), (LaxMapping, self._format_dict), (DataClassType, self._format_dataclass), (SQLAlchemyClassType, self._format_sqlalchemy_class), @@ -240,6 +242,17 @@ def _format_bytearray(self, value: 'Any', _: str, indent_current: int, indent_ne lines = self._wrap_lines(bytes(value), indent_new) self._str_lines(lines, indent_current, indent_new) + def _format_ast_expression(self, value: ast.AST, _: str, indent_current: int, indent_new: int) -> None: + try: + s = ast.dump(value, indent=self._indent_step) + except TypeError: + # no indent before 3.9 + s = ast.dump(value) + lines = s.splitlines(True) + self._stream.write(lines[0]) + for line in lines[1:]: + self._stream.write(indent_current * self._c + line) + def _format_dataclass(self, value: 'Any', _: str, indent_current: int, indent_new: int) -> None: self._format_fields(value, value.__dict__.items(), indent_current, indent_new) diff --git a/requirements/testing.in b/requirements/testing.in index 7b1ad14..1976d79 100644 --- a/requirements/testing.in +++ b/requirements/testing.in @@ -5,7 +5,9 @@ pytest-mock pytest-pretty # these packages are used in tests so install the latest version pydantic -asyncpg +# no binaries for 3.7 +asyncpg; python_version>='3.8' +# no version is compatible with 3.7 and 3.11 numpy; python_version>='3.8' multidict; python_version>='3.8' sqlalchemy diff --git a/requirements/testing.txt b/requirements/testing.txt index b62eaba..2ce58bf 100644 --- a/requirements/testing.txt +++ b/requirements/testing.txt @@ -4,7 +4,7 @@ # # pip-compile --output-file=requirements/testing.txt --resolver=backtracking requirements/testing.in # -asyncpg==0.27.0 +asyncpg==0.27.0 ; python_version >= "3.8" # via -r requirements/testing.in attrs==22.2.0 # via pytest diff --git a/tests/test_prettier.py b/tests/test_prettier.py index 364447b..6a0247c 100644 --- a/tests/test_prettier.py +++ b/tests/test_prettier.py @@ -1,5 +1,7 @@ +import ast import os import string +import sys from collections import Counter, OrderedDict, namedtuple from dataclasses import dataclass from typing import List @@ -457,3 +459,26 @@ class User(SQLAlchemyBase): " nickname='test',\n" ")" ) + + +@pytest.mark.skipif(sys.version_info < (3, 9), reason='no indent on older versions') +def test_ast_expr(): + assert pformat(ast.parse('print(1, 2, round(3))', mode='eval')) == ( + "Expression(" + "\n body=Call(" + "\n func=Name(id='print', ctx=Load())," + "\n args=[" + "\n Constant(value=1)," + "\n Constant(value=2)," + "\n Call(" + "\n func=Name(id='round', ctx=Load())," + "\n args=[" + "\n Constant(value=3)]," + "\n keywords=[])]," + "\n keywords=[]))" + ) + + +@pytest.mark.skipif(sys.version_info < (3, 9), reason='no indent on older versions') +def test_ast_module(): + assert pformat(ast.parse('print(1, 2, round(3))')).startswith('Module(\n body=[') From 61c6b67472f7a0e818968ab73dcd56762f91d419 Mon Sep 17 00:00:00 2001 From: Samuel Colvin Date: Wed, 5 Apr 2023 18:59:07 +0100 Subject: [PATCH 5/7] Insert assert (#126) * support displaying ast types * support 3.7 & 3.8 * skip tests on older python * add insert_assert pytest fixture * use newest pytest-pretty * try to fix CI * fix mypy and black * add pytest to for mypy * fix mypy * change code to install debug in fixture * tweak install instructions --- .github/workflows/ci.yml | 5 +- devtools/__main__.py | 27 ++-- devtools/prettier.py | 6 +- devtools/pytest_plugin.py | 301 ++++++++++++++++++++++++++++++++++++ pyproject.toml | 5 +- requirements/linting.in | 1 + requirements/linting.txt | 15 +- requirements/testing.in | 5 +- requirements/testing.txt | 17 +- tests/conftest.py | 2 + tests/test_insert_assert.py | 169 ++++++++++++++++++++ 11 files changed, 531 insertions(+), 22 deletions(-) create mode 100644 devtools/pytest_plugin.py create mode 100644 tests/test_insert_assert.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad2c060..e7b81b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,6 +8,9 @@ on: - '**' pull_request: {} +env: + COLUMNS: 150 + jobs: lint: runs-on: ubuntu-latest @@ -51,7 +54,7 @@ jobs: python-version: ${{ matrix.python-version }} - run: pip install -r requirements/testing.txt -r requirements/pyproject.txt - - run: pip install . + - run: pip freeze - name: test with extras diff --git a/devtools/__main__.py b/devtools/__main__.py index 726f825..bfc6155 100644 --- a/devtools/__main__.py +++ b/devtools/__main__.py @@ -1,5 +1,4 @@ import builtins -import os import sys from pathlib import Path @@ -8,13 +7,17 @@ # language=python install_code = """ # add devtools `debug` function to builtins -import builtins -try: - from devtools import debug -except ImportError: - pass -else: - setattr(builtins, 'debug', debug) +import sys +# we don't install here for pytest as it breaks pytest, it is +# installed later by a pytest fixture +if not sys.argv[0].endswith('pytest'): + import builtins + try: + from devtools import debug + except ImportError: + pass + else: + setattr(builtins, 'debug', debug) """ @@ -47,11 +50,11 @@ def install() -> int: print(f'Found path "{install_path}" to install devtools into __builtins__') print('To install devtools, run the following command:\n') - if os.access(install_path, os.W_OK): - print(f' python -m devtools print-code >> {install_path}\n') - else: + print(f' python -m devtools print-code >> {install_path}\n') + if not install_path.is_relative_to(Path.home()): + print('or maybe\n') print(f' python -m devtools print-code | sudo tee -a {install_path} > /dev/null\n') - print('Note: "sudo" is required because the path is not writable by the current user.') + print('Note: "sudo" might be required because the path is in your home directory.') return 0 diff --git a/devtools/prettier.py b/devtools/prettier.py index e378030..4f274de 100644 --- a/devtools/prettier.py +++ b/devtools/prettier.py @@ -44,9 +44,9 @@ class SkipPretty(Exception): @cache def get_pygments() -> 'Tuple[Any, Any, Any]': try: - import pygments # type: ignore - from pygments.formatters import Terminal256Formatter # type: ignore - from pygments.lexers import PythonLexer # type: ignore + import pygments + from pygments.formatters import Terminal256Formatter + from pygments.lexers import PythonLexer except ImportError: # pragma: no cover return None, None, None else: diff --git a/devtools/pytest_plugin.py b/devtools/pytest_plugin.py new file mode 100644 index 0000000..f80efd3 --- /dev/null +++ b/devtools/pytest_plugin.py @@ -0,0 +1,301 @@ +from __future__ import annotations as _annotations + +import ast +import builtins +import sys +import textwrap +from contextvars import ContextVar +from dataclasses import dataclass +from enum import Enum +from functools import lru_cache +from itertools import groupby +from pathlib import Path +from types import FrameType +from typing import TYPE_CHECKING, Any, Callable, Generator, Sized + +import pytest +from executing import Source + +from . import debug + +if TYPE_CHECKING: + pass + +__all__ = ('insert_assert',) + + +@dataclass +class ToReplace: + file: Path + start_line: int + end_line: int | None + code: str + + +to_replace: list[ToReplace] = [] +insert_assert_calls: ContextVar[int] = ContextVar('insert_assert_calls', default=0) +insert_assert_summary: ContextVar[list[str]] = ContextVar('insert_assert_summary') + + +def insert_assert(value: Any) -> int: + call_frame: FrameType = sys._getframe(1) + if sys.version_info < (3, 8): # pragma: no cover + raise RuntimeError('insert_assert() requires Python 3.8+') + + format_code = load_black() + ex = Source.for_frame(call_frame).executing(call_frame) + if ex.node is None: # pragma: no cover + python_code = format_code(str(custom_repr(value))) + raise RuntimeError( + f'insert_assert() was unable to find the frame from which it was called, called with:\n{python_code}' + ) + ast_arg = ex.node.args[0] # type: ignore[attr-defined] + if isinstance(ast_arg, ast.Name): + arg = ast_arg.id + else: + arg = ' '.join(map(str.strip, ex.source.asttokens().get_text(ast_arg).splitlines())) + + python_code = format_code(f'# insert_assert({arg})\nassert {arg} == {custom_repr(value)}') + + python_code = textwrap.indent(python_code, ex.node.col_offset * ' ') + to_replace.append(ToReplace(Path(call_frame.f_code.co_filename), ex.node.lineno, ex.node.end_lineno, python_code)) + calls = insert_assert_calls.get() + 1 + insert_assert_calls.set(calls) + return calls + + +def pytest_addoption(parser: Any) -> None: + parser.addoption( + '--insert-assert-print', + action='store_true', + default=False, + help='Print statements that would be substituted for insert_assert(), instead of writing to files', + ) + parser.addoption( + '--insert-assert-fail', + action='store_true', + default=False, + help='Fail tests which include one or more insert_assert() calls', + ) + + +@pytest.fixture(scope='session', autouse=True) +def insert_assert_add_to_builtins() -> None: + try: + setattr(builtins, 'insert_assert', insert_assert) + # we also install debug here since the default script doesn't install it + setattr(builtins, 'debug', debug) + except TypeError: + # happens on pypy + pass + + +@pytest.fixture(autouse=True) +def insert_assert_maybe_fail(pytestconfig: pytest.Config) -> Generator[None, None, None]: + insert_assert_calls.set(0) + yield + print_instead = pytestconfig.getoption('insert_assert_print') + if not print_instead: + count = insert_assert_calls.get() + if count: + pytest.fail(f'devtools-insert-assert: {count} assert{plural(count)} will be inserted', pytrace=False) + + +@pytest.fixture(name='insert_assert') +def insert_assert_fixture() -> Callable[[Any], int]: + return insert_assert + + +def pytest_report_teststatus(report: pytest.TestReport, config: pytest.Config) -> Any: + if report.when == 'teardown' and report.failed and 'devtools-insert-assert:' in repr(report.longrepr): + return 'insert assert', 'i', ('INSERT ASSERT', {'cyan': True}) + + +@pytest.fixture(scope='session', autouse=True) +def insert_assert_session(pytestconfig: pytest.Config) -> Generator[None, None, None]: + """ + Actual logic for updating code examples. + """ + try: + __builtins__['insert_assert'] = insert_assert + except TypeError: + # happens on pypy + pass + + yield + + if not to_replace: + return None + + print_instead = pytestconfig.getoption('insert_assert_print') + + highlight = None + if print_instead: + highlight = get_pygments() + + files = 0 + dup_count = 0 + summary = [] + for file, group in groupby(to_replace, key=lambda tr: tr.file): + # we have to substitute lines in reverse order to avoid messing up line numbers + lines = file.read_text().splitlines() + duplicates: set[int] = set() + for tr in sorted(group, key=lambda x: x.start_line, reverse=True): + if print_instead: + hr = '-' * 80 + code = highlight(tr.code) if highlight else tr.code + line_no = f'{tr.start_line}' if tr.start_line == tr.end_line else f'{tr.start_line}-{tr.end_line}' + summary.append(f'{file} - {line_no}:\n{hr}\n{code}{hr}\n') + else: + if tr.start_line in duplicates: + dup_count += 1 + else: + duplicates.add(tr.start_line) + lines[tr.start_line - 1 : tr.end_line] = tr.code.splitlines() + if not print_instead: + file.write_text('\n'.join(lines)) + files += 1 + prefix = 'Printed' if print_instead else 'Replaced' + summary.append( + f'{prefix} {len(to_replace)} insert_assert() call{plural(to_replace)} in {files} file{plural(files)}' + ) + if dup_count: + summary.append( + f'\n{dup_count} insert skipped because an assert statement on that line had already be inserted!' + ) + + insert_assert_summary.set(summary) + to_replace.clear() + + +def pytest_terminal_summary() -> None: + summary = insert_assert_summary.get(None) + if summary: + print('\n'.join(summary)) + + +def custom_repr(value: Any) -> Any: + if isinstance(value, (list, tuple, set, frozenset)): + return value.__class__(map(custom_repr, value)) + elif isinstance(value, dict): + return value.__class__((custom_repr(k), custom_repr(v)) for k, v in value.items()) + if isinstance(value, Enum): + return PlainRepr(f'{value.__class__.__name__}.{value.name}') + else: + return PlainRepr(repr(value)) + + +class PlainRepr(str): + """ + String class where repr doesn't include quotes. + """ + + def __repr__(self) -> str: + return str(self) + + +def plural(v: int | Sized) -> str: + if isinstance(v, (int, float)): + n = v + else: + n = len(v) + return '' if n == 1 else 's' + + +@lru_cache(maxsize=None) +def load_black() -> Callable[[str], str]: + """ + Build black configuration from "pyproject.toml". + + Black doesn't have a nice self-contained API for reading pyproject.toml, hence all this. + """ + try: + from black import format_file_contents + from black.files import find_pyproject_toml, parse_pyproject_toml + from black.mode import Mode, TargetVersion + from black.parsing import InvalidInput + except ImportError: + return lambda x: x + + def convert_target_version(target_version_config: Any) -> set[Any] | None: + if target_version_config is not None: + return None + elif not isinstance(target_version_config, list): + raise ValueError('Config key "target_version" must be a list') + else: + return {TargetVersion[tv.upper()] for tv in target_version_config} + + @dataclass + class ConfigArg: + config_name: str + keyword_name: str + converter: Callable[[Any], Any] + + config_mapping: list[ConfigArg] = [ + ConfigArg('target_version', 'target_versions', convert_target_version), + ConfigArg('line_length', 'line_length', int), + ConfigArg('skip_string_normalization', 'string_normalization', lambda x: not x), + ConfigArg('skip_magic_trailing_commas', 'magic_trailing_comma', lambda x: not x), + ] + + config_str = find_pyproject_toml((str(Path.cwd()),)) + mode_ = None + fast = False + if config_str: + try: + config = parse_pyproject_toml(config_str) + except (OSError, ValueError) as e: + raise ValueError(f'Error reading configuration file: {e}') + + if config: + kwargs = dict() + for config_arg in config_mapping: + try: + value = config[config_arg.config_name] + except KeyError: + pass + else: + value = config_arg.converter(value) + if value is not None: + kwargs[config_arg.keyword_name] = value + + mode_ = Mode(**kwargs) + fast = bool(config.get('fast')) + + mode = mode_ or Mode() + + def format_code(code: str) -> str: + try: + return format_file_contents(code, fast=fast, mode=mode) + except InvalidInput as e: + print('black error, you will need to format the code manually,', e) + return code + + return format_code + + +# isatty() is false inside pytest, hence calling this now +try: + std_out_istty = sys.stdout.isatty() +except Exception: + std_out_istty = False + + +@lru_cache(maxsize=None) +def get_pygments() -> Callable[[str], str] | None: # pragma: no cover + if not std_out_istty: + return None + try: + import pygments + from pygments.formatters import Terminal256Formatter + from pygments.lexers import PythonLexer + except ImportError as e: # pragma: no cover + print(e) + return None + else: + pyg_lexer, terminal_formatter = PythonLexer(), Terminal256Formatter() + + def highlight(code: str) -> str: + return pygments.highlight(code, lexer=pyg_lexer, formatter=terminal_formatter) + + return highlight diff --git a/pyproject.toml b/pyproject.toml index 193aabc..7e7c87f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,9 @@ Funding = 'https://github.com/sponsors/samuelcolvin' Source = 'https://github.com/samuelcolvin/python-devtools' Changelog = 'https://github.com/samuelcolvin/python-devtools/releases' +[project.entry-points.pytest11] +devtools = 'devtools.pytest_plugin' + [tool.pytest.ini_options] testpaths = 'tests' filterwarnings = 'error' @@ -90,5 +93,5 @@ strict = true warn_return_any = false [[tool.mypy.overrides]] -module = ['executing.*'] +module = ['executing.*', 'pygments.*'] ignore_missing_imports = true diff --git a/requirements/linting.in b/requirements/linting.in index 44df076..e284107 100644 --- a/requirements/linting.in +++ b/requirements/linting.in @@ -3,3 +3,4 @@ mypy==0.971 ruff # required so mypy can find stubs sqlalchemy +pytest diff --git a/requirements/linting.txt b/requirements/linting.txt index 92279cd..21c1a03 100644 --- a/requirements/linting.txt +++ b/requirements/linting.txt @@ -4,10 +4,16 @@ # # pip-compile --output-file=requirements/linting.txt --resolver=backtracking requirements/linting.in # +attrs==22.2.0 + # via pytest black==23.3.0 # via -r requirements/linting.in click==8.1.3 # via black +exceptiongroup==1.1.1 + # via pytest +iniconfig==2.0.0 + # via pytest mypy==0.971 # via -r requirements/linting.in mypy-extensions==1.0.0 @@ -15,11 +21,17 @@ mypy-extensions==1.0.0 # black # mypy packaging==23.0 - # via black + # via + # black + # pytest pathspec==0.11.1 # via black platformdirs==3.2.0 # via black +pluggy==1.0.0 + # via pytest +pytest==7.2.2 + # via -r requirements/linting.in ruff==0.0.261 # via -r requirements/linting.in sqlalchemy==2.0.8 @@ -28,6 +40,7 @@ tomli==2.0.1 # via # black # mypy + # pytest typing-extensions==4.5.0 # via # mypy diff --git a/requirements/testing.in b/requirements/testing.in index 1976d79..0543021 100644 --- a/requirements/testing.in +++ b/requirements/testing.in @@ -4,10 +4,11 @@ pytest pytest-mock pytest-pretty # these packages are used in tests so install the latest version -pydantic # no binaries for 3.7 asyncpg; python_version>='3.8' +black +multidict; python_version>='3.8' # no version is compatible with 3.7 and 3.11 numpy; python_version>='3.8' -multidict; python_version>='3.8' +pydantic sqlalchemy diff --git a/requirements/testing.txt b/requirements/testing.txt index 2ce58bf..9fb1f82 100644 --- a/requirements/testing.txt +++ b/requirements/testing.txt @@ -8,6 +8,10 @@ asyncpg==0.27.0 ; python_version >= "3.8" # via -r requirements/testing.in attrs==22.2.0 # via pytest +black==23.3.0 + # via -r requirements/testing.in +click==8.1.3 + # via black coverage[toml]==7.2.2 # via -r requirements/testing.in exceptiongroup==1.1.1 @@ -20,10 +24,18 @@ mdurl==0.1.2 # via markdown-it-py multidict==6.0.4 ; python_version >= "3.8" # via -r requirements/testing.in +mypy-extensions==1.0.0 + # via black numpy==1.24.2 ; python_version >= "3.8" # via -r requirements/testing.in packaging==23.0 - # via pytest + # via + # black + # pytest +pathspec==0.11.1 + # via black +platformdirs==3.2.0 + # via black pluggy==1.0.0 # via pytest pydantic==1.10.7 @@ -39,7 +51,7 @@ pytest==7.2.2 # pytest-pretty pytest-mock==3.10.0 # via -r requirements/testing.in -pytest-pretty==1.1.1 +pytest-pretty==1.2.0 # via -r requirements/testing.in rich==13.3.3 # via pytest-pretty @@ -47,6 +59,7 @@ sqlalchemy==2.0.8 # via -r requirements/testing.in tomli==2.0.1 # via + # black # coverage # pytest typing-extensions==4.5.0 diff --git a/tests/conftest.py b/tests/conftest.py index 2591e75..2a62df2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,7 @@ import os +pytest_plugins = ['pytester'] + def pytest_sessionstart(session): os.environ.pop('PY_DEVTOOLS_HIGHLIGHT', None) diff --git a/tests/test_insert_assert.py b/tests/test_insert_assert.py new file mode 100644 index 0000000..299cee8 --- /dev/null +++ b/tests/test_insert_assert.py @@ -0,0 +1,169 @@ +import os +import sys + +import pytest + +from devtools.pytest_plugin import load_black + +pytestmark = pytest.mark.skipif(sys.version_info < (3, 8), reason='requires Python 3.8+') + + +config = "pytest_plugins = ['devtools.pytest_plugin']" +# language=Python +default_test = """\ +def test_ok(): + assert 1 + 2 == 3 + +def test_string_assert(insert_assert): + thing = 'foobar' + insert_assert(thing)\ +""" + + +def test_insert_assert(pytester_pretty): + os.environ.pop('CI', None) + pytester_pretty.makeconftest(config) + test_file = pytester_pretty.makepyfile(default_test) + result = pytester_pretty.runpytest() + result.assert_outcomes(passed=2) + # print(result.outlines) + assert test_file.read_text() == ( + 'def test_ok():\n' + ' assert 1 + 2 == 3\n' + '\n' + 'def test_string_assert(insert_assert):\n' + " thing = 'foobar'\n" + ' # insert_assert(thing)\n' + ' assert thing == "foobar"' + ) + + +def test_insert_assert_no_pretty(pytester): + os.environ.pop('CI', None) + pytester.makeconftest(config) + test_file = pytester.makepyfile(default_test) + result = pytester.runpytest('-p', 'no:pretty') + result.assert_outcomes(passed=2) + assert test_file.read_text() == ( + 'def test_ok():\n' + ' assert 1 + 2 == 3\n' + '\n' + 'def test_string_assert(insert_assert):\n' + " thing = 'foobar'\n" + ' # insert_assert(thing)\n' + ' assert thing == "foobar"' + ) + + +def test_insert_assert_print(pytester_pretty, capsys): + os.environ.pop('CI', None) + pytester_pretty.makeconftest(config) + test_file = pytester_pretty.makepyfile(default_test) + # assert r == 0 + result = pytester_pretty.runpytest('--insert-assert-print') + result.assert_outcomes(passed=2) + assert test_file.read_text() == default_test + captured = capsys.readouterr() + assert 'test_insert_assert_print.py - 6:' in captured.out + assert 'Printed 1 insert_assert() call in 1 file\n' in captured.out + + +def test_insert_assert_fail(pytester_pretty): + os.environ.pop('CI', None) + pytester_pretty.makeconftest(config) + test_file = pytester_pretty.makepyfile(default_test) + # assert r == 0 + result = pytester_pretty.runpytest() + assert result.parseoutcomes() == {'passed': 2, 'warning': 1, 'insert': 1} + assert test_file.read_text() != default_test + + +def test_deep(pytester_pretty): + os.environ.pop('CI', None) + pytester_pretty.makeconftest(config) + # language=Python + test_file = pytester_pretty.makepyfile( + """ + def test_deep(insert_assert): + insert_assert([{'a': i, 'b': 2 * 2} for i in range(3)]) + """ + ) + result = pytester_pretty.runpytest() + result.assert_outcomes(passed=1) + assert test_file.read_text() == ( + 'def test_deep(insert_assert):\n' + " # insert_assert([{'a': i, 'b': 2 * 2} for i in range(3)])\n" + ' assert [{"a": i, "b": 2 * 2} for i in range(3)] == [\n' + ' {"a": 0, "b": 4},\n' + ' {"a": 1, "b": 4},\n' + ' {"a": 2, "b": 4},\n' + ' ]' + ) + + +def test_enum(pytester_pretty, capsys): + os.environ.pop('CI', None) + pytester_pretty.makeconftest(config) + # language=Python + pytester_pretty.makepyfile( + """ +from enum import Enum + +class Foo(Enum): + A = 1 + B = 2 + +def test_deep(insert_assert): + x = Foo.A + insert_assert(x) + """ + ) + result = pytester_pretty.runpytest('--insert-assert-print') + result.assert_outcomes(passed=1) + captured = capsys.readouterr() + assert ' assert x == Foo.A\n' in captured.out + + +def test_insert_assert_black(tmp_path): + old_wd = os.getcwd() + try: + os.chdir(tmp_path) + (tmp_path / 'pyproject.toml').write_text( + """\ +[tool.black] +target-version = ["py39"] +skip-string-normalization = true""" + ) + load_black.cache_clear() + finally: + os.chdir(old_wd) + + f = load_black() + # no string normalization + assert f("'foobar'") == "'foobar'\n" + + +def test_insert_assert_repeat(pytester_pretty, capsys): + os.environ.pop('CI', None) + pytester_pretty.makeconftest(config) + test_file = pytester_pretty.makepyfile( + """\ +import pytest + +@pytest.mark.parametrize('x', [1, 2, 3]) +def test_string_assert(x, insert_assert): + insert_assert(x)\ +""" + ) + result = pytester_pretty.runpytest() + result.assert_outcomes(passed=3) + assert test_file.read_text() == ( + 'import pytest\n' + '\n' + "@pytest.mark.parametrize('x', [1, 2, 3])\n" + 'def test_string_assert(x, insert_assert):\n' + ' # insert_assert(x)\n' + ' assert x == 1' + ) + captured = capsys.readouterr() + assert '2 insert skipped because an assert statement on that line had already be inserted!\n' in captured.out From f416c9b8df4f087b2302cd0fe3f878ffecd40661 Mon Sep 17 00:00:00 2001 From: Samuel Colvin Date: Wed, 5 Apr 2023 19:00:13 +0100 Subject: [PATCH 6/7] uprev --- devtools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devtools/version.py b/devtools/version.py index ea301cb..d3c01ed 100644 --- a/devtools/version.py +++ b/devtools/version.py @@ -1 +1 @@ -VERSION = '0.10.0' +VERSION = '0.11.0' From 71edb0d6957895615ac258a2080e9424e99b0e99 Mon Sep 17 00:00:00 2001 From: Samuel Colvin Date: Wed, 5 Apr 2023 19:04:44 +0100 Subject: [PATCH 7/7] uprev mypy --- requirements/linting.in | 2 +- requirements/linting.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/linting.in b/requirements/linting.in index e284107..41aa444 100644 --- a/requirements/linting.in +++ b/requirements/linting.in @@ -1,5 +1,5 @@ black -mypy==0.971 +mypy ruff # required so mypy can find stubs sqlalchemy diff --git a/requirements/linting.txt b/requirements/linting.txt index 21c1a03..bd6c1fc 100644 --- a/requirements/linting.txt +++ b/requirements/linting.txt @@ -14,7 +14,7 @@ exceptiongroup==1.1.1 # via pytest iniconfig==2.0.0 # via pytest -mypy==0.971 +mypy==1.1.1 # via -r requirements/linting.in mypy-extensions==1.0.0 # via