diff --git a/.github/workflows/test-osx.yml b/.github/workflows/test-osx.yml deleted file mode 100644 index c9afbe0a0ea..00000000000 --- a/.github/workflows/test-osx.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Run tests on OSX - -on: [push, pull_request] - -jobs: - test: - runs-on: macos-latest - - steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.7 - uses: actions/setup-python@v2 - with: - python-version: 3.7 - - name: Install and update Python dependencies - run: | - python -m pip install --upgrade pip setuptools wheel - python -m pip install --upgrade -e file://$PWD#egg=ipython[test] - python -m pip install --upgrade --upgrade-strategy eager trio curio - python -m pip install --upgrade pytest pytest-trio 'matplotlib!=3.2.0' - python -m pip install --upgrade anyio - - name: pytest - run: pytest diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b9bbae48cf7..f5840135a29 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,13 +1,27 @@ name: Run tests -on: [push, pull_request] +on: + push: + pull_request: + # Run weekly on Monday at 1:23 UTC + schedule: + - cron: '23 1 * * 1' + workflow_dispatch: + jobs: test: - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: matrix: - python-version: [3.7, 3.8, 3.9] + os: [ubuntu-latest] + python-version: ["3.7", "3.8", "3.9", "3.10"] + # Test all on ubuntu, test ends on macos + include: + - os: macos-latest + python-version: "3.7" + - os: macos-latest + python-version: "3.9" steps: - uses: actions/checkout@v2 @@ -25,6 +39,7 @@ jobs: - name: Check manifest run: check-manifest - name: iptest + if: matrix.python-version != '3.10' run: | cd /tmp && iptest --coverage xml && cd - cp /tmp/ipy_coverage.xml ./ @@ -33,4 +48,4 @@ jobs: run: | pytest - name: Upload coverage to Codecov - uses: codecov/codecov-action@v1 + uses: codecov/codecov-action@v2 diff --git a/IPython/__init__.py b/IPython/__init__.py index 4fb77107680..c17ec76a602 100644 --- a/IPython/__init__.py +++ b/IPython/__init__.py @@ -65,6 +65,10 @@ __license__ = release.license __version__ = release.version version_info = release.version_info +# list of CVEs that should have been patched in this release. +# this is informational and should not be relied upon. +__patched_cves__ = {"CVE-2022-21699"} + def embed_kernel(module=None, local_ns=None, **kwargs): """Embed and start an IPython kernel in a given scope. diff --git a/IPython/core/application.py b/IPython/core/application.py index 93639d88e2c..b319888b59b 100644 --- a/IPython/core/application.py +++ b/IPython/core/application.py @@ -133,7 +133,7 @@ def _config_file_name_changed(self, change): config_file_paths = List(Unicode()) @default('config_file_paths') def _config_file_paths_default(self): - return [os.getcwd()] + return [] extra_config_file = Unicode( help="""Path to an extra config file to load. @@ -374,7 +374,7 @@ def init_profile_dir(self): self.log.fatal("Profile %r not found."%self.profile) self.exit(1) else: - self.log.debug("Using existing profile dir: %r"%p.location) + self.log.debug(f"Using existing profile dir: {p.location!r}") else: location = self.config.ProfileDir.location # location is fully specified @@ -394,7 +394,7 @@ def init_profile_dir(self): self.log.fatal("Profile directory %r not found."%location) self.exit(1) else: - self.log.info("Using existing profile dir: %r"%location) + self.log.debug(f"Using existing profile dir: {p.location!r}") # if profile_dir is specified explicitly, set profile name dir_name = os.path.basename(p.location) if dir_name.startswith('profile_'): diff --git a/IPython/core/async_helpers.py b/IPython/core/async_helpers.py index fb4cc193250..fca78def85a 100644 --- a/IPython/core/async_helpers.py +++ b/IPython/core/async_helpers.py @@ -13,19 +13,29 @@ import ast import sys +import asyncio import inspect from textwrap import dedent, indent class _AsyncIORunner: + def __init__(self): + self._loop = None + + @property + def loop(self): + """Always returns a non-closed event loop""" + if self._loop is None or self._loop.is_closed(): + policy = asyncio.get_event_loop_policy() + self._loop = policy.new_event_loop() + policy.set_event_loop(self._loop) + return self._loop def __call__(self, coro): """ Handler for asyncio autoawait """ - import asyncio - - return asyncio.get_event_loop().run_until_complete(coro) + return self.loop.run_until_complete(coro) def __str__(self): return 'asyncio' diff --git a/IPython/core/completer.py b/IPython/core/completer.py index 238929cc6ff..e4d39958126 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -166,7 +166,13 @@ # may have trouble processing. MATCHES_LIMIT = 500 -_deprecation_readline_sentinel = object() + +class Sentinel: + def __repr__(self): + return "" + + +_deprecation_readline_sentinel = Sentinel() class ProvisionalCompleterWarning(FutureWarning): @@ -1143,18 +1149,18 @@ def matchers(self): if self.use_jedi: return [ *self.custom_matchers, + self.dict_key_matches, self.file_matches, self.magic_matches, - self.dict_key_matches, ] else: return [ *self.custom_matchers, + self.dict_key_matches, self.python_matches, self.file_matches, self.magic_matches, self.python_func_kw_matches, - self.dict_key_matches, ] def all_completions(self, text) -> List[str]: @@ -1496,7 +1502,7 @@ def _default_arguments(self, obj): inspect.Parameter.POSITIONAL_OR_KEYWORD) try: - sig = inspect.signature(call_obj) + sig = inspect.signature(obj) ret.extend(k for k, v in sig.parameters.items() if v.kind in _keeps) except ValueError: diff --git a/IPython/core/debugger.py b/IPython/core/debugger.py index 02d01add393..1744bdb8a8e 100644 --- a/IPython/core/debugger.py +++ b/IPython/core/debugger.py @@ -2,6 +2,76 @@ """ Pdb debugger class. + +This is an extension to PDB which adds a number of new features. +Note that there is also the `IPython.terminal.debugger` class which provides UI +improvements. + +We also strongly recommend to use this via the `ipdb` package, which provides +extra configuration options. + +Among other things, this subclass of PDB: + - supports many IPython magics like pdef/psource + - hide frames in tracebacks based on `__tracebackhide__` + - allows to skip frames based on `__debuggerskip__` + +The skipping and hiding frames are configurable via the `skip_predicates` +command. + +By default, frames from readonly files will be hidden, frames containing +``__tracebackhide__=True`` will be hidden. + +Frames containing ``__debuggerskip__`` will be stepped over, frames who's parent +frames value of ``__debuggerskip__`` is ``True`` will be skipped. + + >>> def helpers_helper(): + ... pass + ... + ... def helper_1(): + ... print("don't step in me") + ... helpers_helpers() # will be stepped over unless breakpoint set. + ... + ... + ... def helper_2(): + ... print("in me neither") + ... + +One can define a decorator that wraps a function between the two helpers: + + >>> def pdb_skipped_decorator(function): + ... + ... + ... def wrapped_fn(*args, **kwargs): + ... __debuggerskip__ = True + ... helper_1() + ... __debuggerskip__ = False + ... result = function(*args, **kwargs) + ... __debuggerskip__ = True + ... helper_2() + ... # setting __debuggerskip__ to False again is not necessary + ... return result + ... + ... return wrapped_fn + +When decorating a function, ipdb will directly step into ``bar()`` by +default: + + >>> @foo_decorator + ... def bar(x, y): + ... return x * y + + +You can toggle the behavior with + + ipdb> skip_predicates debuggerskip false + +or configure it in your ``.pdbrc`` + + + +Licencse +-------- + Modified from the standard pdb.Pdb class to avoid including readline, so that the command line completion of other programs which include this isn't damaged. @@ -9,11 +79,16 @@ In the future, this class will be expanded with improvements over the standard pdb. -The code in this file is mainly lifted out of cmd.py in Python 2.2, with minor -changes. Licensing should therefore be under the standard Python terms. For -details on the PSF (Python Software Foundation) standard license, see: +The original code in this file is mainly lifted out of cmd.py in Python 2.2, +with minor changes. Licensing should therefore be under the standard Python +terms. For details on the PSF (Python Software Foundation) standard license, +see: https://docs.python.org/2/license.html + + +All the changes since then are under the same license as IPython. + """ #***************************************************************************** @@ -51,6 +126,9 @@ # it does so with some limitations. The rest of this support is implemented in # the Tracer constructor. +DEBUGGERSKIP = "__debuggerskip__" + + def make_arrow(pad): """generate the leading arrow in front of traceback or debugger""" if pad >= 2: @@ -206,7 +284,12 @@ class Pdb(OldPdb): """ - default_predicates = {"tbhide": True, "readonly": False, "ipython_internal": True} + default_predicates = { + "tbhide": True, + "readonly": False, + "ipython_internal": True, + "debuggerskip": True, + } def __init__(self, color_scheme=None, completekey=None, stdin=None, stdout=None, context=5, **kwargs): @@ -305,6 +388,7 @@ def __init__(self, color_scheme=None, completekey=None, # list of predicates we use to skip frames self._predicates = self.default_predicates + # def set_colors(self, scheme): """Shorthand access to the color table scheme selector method.""" self.color_scheme_table.set_active_scheme(scheme) @@ -331,9 +415,10 @@ def _hidden_predicate(self, frame): if self._predicates["tbhide"]: if frame in (self.curframe, getattr(self, "initial_frame", None)): return False - else: - return self._get_frame_locals(frame).get("__tracebackhide__", False) - + frame_locals = self._get_frame_locals(frame) + if "__tracebackhide__" not in frame_locals: + return False + return frame_locals["__tracebackhide__"] return False def hidden_frames(self, stack): @@ -804,10 +889,61 @@ def do_where(self, arg): do_w = do_where + def break_anywhere(self, frame): + """ + + _stop_in_decorator_internals is overly restrictive, as we may still want + to trace function calls, so we need to also update break_anywhere so + that is we don't `stop_here`, because of debugger skip, we may still + stop at any point inside the function + + """ + + sup = super().break_anywhere(frame) + if sup: + return sup + if self._predicates["debuggerskip"]: + if DEBUGGERSKIP in frame.f_code.co_varnames: + return True + if frame.f_back and self._get_frame_locals(frame.f_back).get(DEBUGGERSKIP): + return True + return False + + @skip_doctest + def _is_in_decorator_internal_and_should_skip(self, frame): + """ + Utility to tell us whether we are in a decorator internal and should stop. + + + + """ + + # if we are disabled don't skip + if not self._predicates["debuggerskip"]: + return False + + # if frame is tagged, skip by default. + if DEBUGGERSKIP in frame.f_code.co_varnames: + return True + + # if one of the parent frame value set to True skip as well. + + cframe = frame + while getattr(cframe, "f_back", None): + cframe = cframe.f_back + if self._get_frame_locals(cframe).get(DEBUGGERSKIP): + return True + + return False + def stop_here(self, frame): """Check if pdb should stop here""" if not super().stop_here(frame): return False + + if self._is_in_decorator_internal_and_should_skip(frame) is True: + return False + hidden = False if self.skip_hidden: hidden = self._hidden_predicate(frame) @@ -929,10 +1065,10 @@ def do_context(self, context): class InterruptiblePdb(Pdb): """Version of debugger where KeyboardInterrupt exits the debugger altogether.""" - def cmdloop(self): + def cmdloop(self, intro=None): """Wrap cmdloop() such that KeyboardInterrupt stops the debugger.""" try: - return OldPdb.cmdloop(self) + return OldPdb.cmdloop(self, intro=intro) except KeyboardInterrupt: self.stop_here = lambda frame: False self.do_quit("") diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 4ae21e4b3ca..835dc8d0a8c 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -43,7 +43,7 @@ from IPython.core.builtin_trap import BuiltinTrap from IPython.core.events import EventManager, available_events from IPython.core.compilerop import CachingCompiler, check_linecache_ipython -from IPython.core.debugger import Pdb +from IPython.core.debugger import InterruptiblePdb from IPython.core.display_trap import DisplayTrap from IPython.core.displayhook import DisplayHook from IPython.core.displaypub import DisplayPublisher @@ -898,9 +898,7 @@ def init_virtualenv(self): virtualenv was built, and it ignores the --no-site-packages option. A warning will appear suggesting the user installs IPython in the virtualenv, but for many cases, it probably works well enough. - Adapted from code snippets online. - http://blog.ufsoft.org/2009/1/29/ipython-and-virtualenv """ if 'VIRTUAL_ENV' not in os.environ: @@ -931,17 +929,27 @@ def init_virtualenv(self): # Our exe is inside or has access to the virtualenv, don't need to do anything. return - warn("Attempting to work in a virtualenv. If you encounter problems, please " - "install IPython inside the virtualenv.") if sys.platform == "win32": - virtual_env = Path(os.environ["VIRTUAL_ENV"]).joinpath( - "Lib", "site-packages" - ) + virtual_env = str(Path(os.environ["VIRTUAL_ENV"], "Lib", "site-packages")) else: - virtual_env = Path(os.environ["VIRTUAL_ENV"]).joinpath( - "lib", "python{}.{}".format(*sys.version_info[:2]), "site-packages" + virtual_env_path = Path( + os.environ["VIRTUAL_ENV"], "lib", "python{}.{}", "site-packages" ) - + p_ver = sys.version_info[:2] + + # Predict version from py[thon]-x.x in the $VIRTUAL_ENV + re_m = re.search(r"\bpy(?:thon)?([23])\.(\d+)\b", os.environ["VIRTUAL_ENV"]) + if re_m: + predicted_path = Path(str(virtual_env_path).format(*re_m.groups())) + if predicted_path.exists(): + p_ver = re_m.groups() + + virtual_env = str(virtual_env_path).format(*p_ver) + + warn( + "Attempting to work in a virtualenv. If you encounter problems, " + "please install IPython inside the virtualenv." + ) import site sys.path.insert(0, virtual_env) site.addsitedir(virtual_env) @@ -1804,8 +1812,13 @@ def object_inspect_mime(self, oname, detail_level=0): with self.builtin_trap: info = self._object_find(oname) if info.found: - return self.inspector._get_info(info.obj, oname, info=info, - detail_level=detail_level + docformat = sphinxify if self.sphinxify_docstring else None + return self.inspector._get_info( + info.obj, + oname, + info=info, + detail_level=detail_level, + formatter=docformat, ) else: raise KeyError(oname) @@ -1823,7 +1836,7 @@ def init_history(self): # Things related to exception handling and tracebacks (not debugging) #------------------------------------------------------------------------- - debugger_cls = Pdb + debugger_cls = InterruptiblePdb def init_traceback_handlers(self, custom_exceptions): # Syntax error handler. @@ -2968,7 +2981,7 @@ def should_run_async( result: bool Whether the code needs to be run with a coroutine runner or not - .. versionadded: 7.0 + .. versionadded:: 7.0 """ if not self.autoawait: return False @@ -3033,7 +3046,7 @@ async def run_cell_async( ------- result : :class:`ExecutionResult` - .. versionadded: 7.0 + .. versionadded:: 7.0 """ info = ExecutionInfo( raw_cell, store_history, silent, shell_futures) diff --git a/IPython/core/magics/execution.py b/IPython/core/magics/execution.py index 940b69f0313..6b651939f8f 100644 --- a/IPython/core/magics/execution.py +++ b/IPython/core/magics/execution.py @@ -532,7 +532,18 @@ def run(self, parameter_s='', runner=None, %run [-n -i -e -G] [( -t [-N] | -d [-b] | -p [profile options] )] - ( -m mod | file ) [args] + ( -m mod | filename ) [args] + + The filename argument should be either a pure Python script (with + extension ``.py``), or a file with custom IPython syntax (such as + magics). If the latter, the file can be either a script with ``.ipy`` + extension, or a Jupyter notebook with ``.ipynb`` extension. When running + a Jupyter notebook, the output from print statements and other + displayed objects will appear in the terminal (even matplotlib figures + will open, if a terminal-compliant backend is being used). Note that, + at the system command line, the ``jupyter run`` command offers similar + functionality for executing notebooks (albeit currently with some + differences in supported options). Parameters after the filename are passed as command-line arguments to the program (put in sys.argv). Then, control returns to IPython's diff --git a/IPython/core/oinspect.py b/IPython/core/oinspect.py index ab25eeeffca..272916c9663 100644 --- a/IPython/core/oinspect.py +++ b/IPython/core/oinspect.py @@ -182,11 +182,12 @@ def getsource(obj, oname='') -> Union[str,None]: except TypeError: # The object itself provided no meaningful source, try looking for # its class definition instead. - if hasattr(obj, '__class__'): - try: - src = inspect.getsource(obj.__class__) - except TypeError: - return None + try: + src = inspect.getsource(obj.__class__) + except (OSError, TypeError): + return None + except OSError: + return None return src @@ -308,14 +309,14 @@ def find_file(obj) -> str: except TypeError: # For an instance, the file that matters is where its class was # declared. - if hasattr(obj, '__class__'): - try: - fname = inspect.getabsfile(obj.__class__) - except TypeError: - # Can happen for builtins - pass - except: + try: + fname = inspect.getabsfile(obj.__class__) + except (OSError, TypeError): + # Can happen for builtins + pass + except OSError: pass + return cast_unicode(fname) @@ -338,15 +339,14 @@ def find_source_lines(obj): obj = _get_wrapped(obj) try: + lineno = inspect.getsourcelines(obj)[1] + except TypeError: + # For instances, try the class object like getsource() does try: - lineno = inspect.getsourcelines(obj)[1] - except TypeError: - # For instances, try the class object like getsource() does - if hasattr(obj, '__class__'): - lineno = inspect.getsourcelines(obj.__class__)[1] - else: - lineno = None - except: + lineno = inspect.getsourcelines(obj.__class__)[1] + except (OSError, TypeError): + return None + except OSError: return None return lineno diff --git a/IPython/core/profileapp.py b/IPython/core/profileapp.py index 97434e3d0b5..9a1bae55ac5 100644 --- a/IPython/core/profileapp.py +++ b/IPython/core/profileapp.py @@ -181,9 +181,10 @@ def list_profile_dirs(self): profiles = list_profiles_in(os.getcwd()) if profiles: print() - print("Available profiles in current directory (%s):" % os.getcwd()) - self._print_profiles(profiles) - + print( + "Profiles from CWD have been removed for security reason, see CVE-2022-21699:" + ) + print() print("To use any of the above profiles, start IPython with:") print(" ipython --profile=") diff --git a/IPython/core/profiledir.py b/IPython/core/profiledir.py index 3199dfd5d64..2c48e4c2f1c 100644 --- a/IPython/core/profiledir.py +++ b/IPython/core/profiledir.py @@ -186,7 +186,7 @@ def find_profile_dir_by_name(cls, ipython_dir, name=u'default', config=None): is not found, a :class:`ProfileDirError` exception will be raised. The search path algorithm is: - 1. ``os.getcwd()`` + 1. ``os.getcwd()`` # removed for security reason. 2. ``ipython_dir`` Parameters @@ -198,7 +198,7 @@ def find_profile_dir_by_name(cls, ipython_dir, name=u'default', config=None): will be "profile_". """ dirname = u'profile_' + name - paths = [os.getcwd(), ipython_dir] + paths = [ipython_dir] for p in paths: profile_dir = os.path.join(p, dirname) if os.path.isdir(profile_dir): diff --git a/IPython/core/pylabtools.py b/IPython/core/pylabtools.py index 8e3aade62d0..c9c8e14aa28 100644 --- a/IPython/core/pylabtools.py +++ b/IPython/core/pylabtools.py @@ -5,6 +5,8 @@ # Distributed under the terms of the Modified BSD License. from io import BytesIO +from binascii import b2a_base64 +from functools import partial import warnings from IPython.core.display import _pngxy @@ -39,8 +41,6 @@ # most part it's just a reverse of the above dict, but we also need to add a # few others that map to the same GUI manually: backend2gui = dict(zip(backends.values(), backends.keys())) -# Our tests expect backend2gui to just return 'qt' -backend2gui['Qt4Agg'] = 'qt' # In the reverse mapping, there are a few extra valid matplotlib backends that # map to the same GUI support backend2gui["GTK"] = backend2gui["GTKCairo"] = "gtk" @@ -48,6 +48,13 @@ backend2gui["GTK4Cairo"] = "gtk4" backend2gui["WX"] = "wx" backend2gui["CocoaAgg"] = "osx" +# There needs to be a hysteresis here as the new QtAgg Matplotlib backend +# supports either Qt5 or Qt6 and the IPython qt event loop support Qt4, Qt5, +# and Qt6. +backend2gui["QtAgg"] = "qt" +backend2gui["Qt4Agg"] = "qt" +backend2gui["Qt5Agg"] = "qt" + # And some backends that don't need GUI integration del backend2gui["nbAgg"] del backend2gui["agg"] @@ -55,6 +62,7 @@ del backend2gui["pdf"] del backend2gui["ps"] del backend2gui["module://matplotlib_inline.backend_inline"] +del backend2gui["module://ipympl.backend_nbagg"] #----------------------------------------------------------------------------- # Matplotlib utilities @@ -99,7 +107,7 @@ def figsize(sizex, sizey): matplotlib.rcParams['figure.figsize'] = [sizex, sizey] -def print_figure(fig, fmt='png', bbox_inches='tight', **kwargs): +def print_figure(fig, fmt="png", bbox_inches="tight", base64=False, **kwargs): """Print a figure to an image, and return the resulting file data Returned data will be bytes unless ``fmt='svg'``, @@ -107,6 +115,12 @@ def print_figure(fig, fmt='png', bbox_inches='tight', **kwargs): Any keyword args are passed to fig.canvas.print_figure, such as ``quality`` or ``bbox_inches``. + + If `base64` is True, return base64-encoded str instead of raw bytes + for binary-encoded image formats + + .. versionadded:: 7.29 + base64 argument """ # When there's an empty figure, we shouldn't return anything, otherwise we # get big blank areas in the qt console. @@ -138,19 +152,31 @@ def print_figure(fig, fmt='png', bbox_inches='tight', **kwargs): data = bytes_io.getvalue() if fmt == 'svg': data = data.decode('utf-8') + elif base64: + data = b2a_base64(data).decode("ascii") return data -def retina_figure(fig, **kwargs): - """format a figure as a pixel-doubled (retina) PNG""" - pngdata = print_figure(fig, fmt='retina', **kwargs) +def retina_figure(fig, base64=False, **kwargs): + """format a figure as a pixel-doubled (retina) PNG + + If `base64` is True, return base64-encoded str instead of raw bytes + for binary-encoded image formats + + .. versionadded:: 7.29 + base64 argument + """ + pngdata = print_figure(fig, fmt="retina", base64=False, **kwargs) # Make sure that retina_figure acts just like print_figure and returns # None when the figure is empty. if pngdata is None: return w, h = _pngxy(pngdata) metadata = {"width": w//2, "height":h//2} + if base64: + pngdata = b2a_base64(pngdata).decode("ascii") return pngdata, metadata + # We need a little factory function here to create the closure where # safe_execfile can live. def mpl_runner(safe_execfile): @@ -249,16 +275,22 @@ def select_figure_formats(shell, formats, **kwargs): gs = "%s" % ','.join([repr(f) for f in supported]) raise ValueError("supported formats are: %s not %s" % (gs, bs)) - if 'png' in formats: - png_formatter.for_type(Figure, lambda fig: print_figure(fig, 'png', **kwargs)) - if 'retina' in formats or 'png2x' in formats: - png_formatter.for_type(Figure, lambda fig: retina_figure(fig, **kwargs)) - if 'jpg' in formats or 'jpeg' in formats: - jpg_formatter.for_type(Figure, lambda fig: print_figure(fig, 'jpg', **kwargs)) - if 'svg' in formats: - svg_formatter.for_type(Figure, lambda fig: print_figure(fig, 'svg', **kwargs)) - if 'pdf' in formats: - pdf_formatter.for_type(Figure, lambda fig: print_figure(fig, 'pdf', **kwargs)) + if "png" in formats: + png_formatter.for_type( + Figure, partial(print_figure, fmt="png", base64=True, **kwargs) + ) + if "retina" in formats or "png2x" in formats: + png_formatter.for_type(Figure, partial(retina_figure, base64=True, **kwargs)) + if "jpg" in formats or "jpeg" in formats: + jpg_formatter.for_type( + Figure, partial(print_figure, fmt="jpg", base64=True, **kwargs) + ) + if "svg" in formats: + svg_formatter.for_type(Figure, partial(print_figure, fmt="svg", **kwargs)) + if "pdf" in formats: + pdf_formatter.for_type( + Figure, partial(print_figure, fmt="pdf", base64=True, **kwargs) + ) #----------------------------------------------------------------------------- # Code for initializing matplotlib and importing pylab @@ -367,7 +399,7 @@ def import_pylab(user_ns, import_all=True): def configure_inline_support(shell, backend): """ - .. deprecated: 7.23 + .. deprecated:: 7.23 use `matplotlib_inline.backend_inline.configure_inline_support()` diff --git a/IPython/core/release.py b/IPython/core/release.py index 234b4871465..90d9e6caf59 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -20,8 +20,8 @@ # release. 'dev' as a _version_extra string means this is a development # version _version_major = 7 -_version_minor = 28 -_version_patch = 0 +_version_minor = 31 +_version_patch = 1 _version_extra = '.dev' # _version_extra = 'b1' _version_extra = "" # Uncomment this for full releases diff --git a/IPython/core/tests/test_completer.py b/IPython/core/tests/test_completer.py index 2c19e2e0187..5f39b5f6ae7 100644 --- a/IPython/core/tests/test_completer.py +++ b/IPython/core/tests/test_completer.py @@ -12,6 +12,7 @@ from contextlib import contextmanager import nose.tools as nt +import pytest from traitlets.config.loader import Config from IPython import get_ipython @@ -29,6 +30,15 @@ ) from nose.tools import assert_in, assert_not_in +if sys.version_info >= (3, 10): + import jedi + from pkg_resources import parse_version + + # Requires https://github.com/davidhalter/jedi/pull/1795 + jedi_issue = parse_version(jedi.__version__) <= parse_version("0.18.0") +else: + jedi_issue = False + # ----------------------------------------------------------------------------- # Test functions # ----------------------------------------------------------------------------- @@ -381,6 +391,8 @@ def test_all_completions_dups(self): matches = c.all_completions("TestCl") assert matches == ['TestClass'], jedi_status matches = c.all_completions("TestClass.") + if jedi_status and jedi_issue: + continue assert len(matches) > 2, jedi_status matches = c.all_completions("TestClass.a") assert matches == ['TestClass.a', 'TestClass.a1'], jedi_status @@ -435,6 +447,7 @@ def test_completion_have_signature(self): "encoding" in c.signature ), "Signature of function was not found by completer" + @pytest.mark.xfail(jedi_issue, reason="Known failure on jedi<=0.18.0") def test_deduplicate_completions(self): """ Test that completions are correctly deduplicated (even if ranges are not the same) diff --git a/IPython/core/tests/test_debugger.py b/IPython/core/tests/test_debugger.py index 7c94592df6d..eb3b5f353b5 100644 --- a/IPython/core/tests/test_debugger.py +++ b/IPython/core/tests/test_debugger.py @@ -12,6 +12,7 @@ import sys import time import warnings + from subprocess import PIPE, CalledProcessError, check_output from tempfile import NamedTemporaryFile from textwrap import dedent @@ -326,6 +327,191 @@ def g(): child.close() +skip_decorators_blocks = ( + """ + def helpers_helper(): + pass # should not stop here except breakpoint + """, + """ + def helper_1(): + helpers_helper() # should not stop here + """, + """ + def helper_2(): + pass # should not stop here + """, + """ + def pdb_skipped_decorator2(function): + def wrapped_fn(*args, **kwargs): + __debuggerskip__ = True + helper_2() + __debuggerskip__ = False + result = function(*args, **kwargs) + __debuggerskip__ = True + helper_2() + return result + return wrapped_fn + """, + """ + def pdb_skipped_decorator(function): + def wrapped_fn(*args, **kwargs): + __debuggerskip__ = True + helper_1() + __debuggerskip__ = False + result = function(*args, **kwargs) + __debuggerskip__ = True + helper_2() + return result + return wrapped_fn + """, + """ + @pdb_skipped_decorator + @pdb_skipped_decorator2 + def bar(x, y): + return x * y + """, + """import IPython.terminal.debugger as ipdb""", + """ + def f(): + ipdb.set_trace() + bar(3, 4) + """, + """ + f() + """, +) + + +def _decorator_skip_setup(): + import pexpect + + env = os.environ.copy() + env["IPY_TEST_SIMPLE_PROMPT"] = "1" + + child = pexpect.spawn( + sys.executable, ["-m", "IPython", "--colors=nocolor"], env=env + ) + child.timeout = 5 * IPYTHON_TESTING_TIMEOUT_SCALE + + child.expect("IPython") + child.expect("\n") + + dedented_blocks = [dedent(b).strip() for b in skip_decorators_blocks] + in_prompt_number = 1 + for cblock in dedented_blocks: + child.expect_exact(f"In [{in_prompt_number}]:") + in_prompt_number += 1 + for line in cblock.splitlines(): + child.sendline(line) + child.expect_exact(line) + child.sendline("") + return child + + +@skip_win32 +def test_decorator_skip(): + """test that decorator frames can be skipped.""" + + child = _decorator_skip_setup() + + child.expect_exact("3 bar(3, 4)") + child.expect("ipdb>") + + child.expect("ipdb>") + child.sendline("step") + child.expect_exact("step") + + child.expect_exact("1 @pdb_skipped_decorator") + + child.sendline("s") + child.expect_exact("return x * y") + + child.close() + + +@skip_win32 +def test_decorator_skip_disabled(): + """test that decorator frame skipping can be disabled""" + + child = _decorator_skip_setup() + + child.expect_exact("3 bar(3, 4)") + + for input_, expected in [ + ("skip_predicates debuggerskip False", ""), + ("skip_predicates", "debuggerskip : False"), + ("step", "---> 2 def wrapped_fn"), + ("step", "----> 3 __debuggerskip__"), + ("step", "----> 4 helper_1()"), + ("step", "---> 1 def helper_1():"), + ("next", "----> 2 helpers_helper()"), + ("next", "--Return--"), + ("next", "----> 5 __debuggerskip__ = False"), + ]: + child.expect("ipdb>") + child.sendline(input_) + child.expect_exact(input_) + child.expect_exact(expected) + + child.close() + + +@skip_win32 +def test_decorator_skip_with_breakpoint(): + """test that decorator frame skipping can be disabled""" + + import pexpect + + env = os.environ.copy() + env["IPY_TEST_SIMPLE_PROMPT"] = "1" + + child = pexpect.spawn( + sys.executable, ["-m", "IPython", "--colors=nocolor"], env=env + ) + child.timeout = 5 * IPYTHON_TESTING_TIMEOUT_SCALE + + child.expect("IPython") + child.expect("\n") + + ### we need a filename, so we need to exec the full block with a filename + with NamedTemporaryFile(suffix=".py", dir=".", delete=True) as tf: + + name = tf.name[:-3].split("/")[-1] + tf.write("\n".join([dedent(x) for x in skip_decorators_blocks[:-1]]).encode()) + tf.flush() + codeblock = f"from {name} import f" + + dedented_blocks = [ + codeblock, + "f()", + ] + + in_prompt_number = 1 + for cblock in dedented_blocks: + child.expect_exact(f"In [{in_prompt_number}]:") + in_prompt_number += 1 + for line in cblock.splitlines(): + child.sendline(line) + child.expect_exact(line) + child.sendline("") + + # as the filename does not exists, we'll rely on the filename prompt + child.expect_exact("47 bar(3, 4)") + + for input_, expected in [ + (f"b {name}.py:3", ""), + ("step", "1---> 3 pass # should not stop here except"), + ("step", "---> 38 @pdb_skipped_decorator"), + ("continue", ""), + ]: + child.expect("ipdb>") + child.sendline(input_) + child.expect_exact(input_) + child.expect_exact(expected) + + child.close() + + @skip_win32 def test_where_erase_value(): """Test that `where` does not access f_locals and erase values.""" diff --git a/IPython/core/tests/test_display.py b/IPython/core/tests/test_display.py index 9f5fc7ecbf5..7ba11324c1b 100644 --- a/IPython/core/tests/test_display.py +++ b/IPython/core/tests/test_display.py @@ -187,10 +187,12 @@ def test_set_matplotlib_formats_kwargs(): display.set_matplotlib_formats('png', **kwargs) formatter = ip.display_formatter.formatters['image/png'] f = formatter.lookup_by_type(Figure) - cell = f.__closure__[0].cell_contents + formatter_kwargs = f.keywords expected = kwargs + expected["base64"] = True + expected["fmt"] = "png" expected.update(cfg.print_figure_kwargs) - nt.assert_equal(cell, expected) + nt.assert_equal(formatter_kwargs, expected) def test_display_available(): """ diff --git a/IPython/core/tests/test_inputsplitter.py b/IPython/core/tests/test_inputsplitter.py index a39943aed80..807335276a8 100644 --- a/IPython/core/tests/test_inputsplitter.py +++ b/IPython/core/tests/test_inputsplitter.py @@ -14,6 +14,7 @@ from IPython.core.inputtransformer import InputTransformer from IPython.core.tests.test_inputtransformer import syntax, syntax_ml from IPython.testing import tools as tt +from IPython.testing.decorators import skipif #----------------------------------------------------------------------------- # Semi-complete examples (also used as tests) @@ -323,6 +324,7 @@ def test_unicode(self): self.isp.push(u'\xc3\xa9') self.isp.push(u"u'\xc3\xa9'") + @skipif(sys.version_info[:3] == (3, 9, 8)) def test_line_continuation(self): """ Test issue #2108.""" isp = self.isp diff --git a/IPython/core/tests/test_inputtransformer2.py b/IPython/core/tests/test_inputtransformer2.py index 61f183cb571..f6d4ac1ecb0 100644 --- a/IPython/core/tests/test_inputtransformer2.py +++ b/IPython/core/tests/test_inputtransformer2.py @@ -6,11 +6,14 @@ """ import nose.tools as nt import string +import sys +from textwrap import dedent -from IPython.core import inputtransformer2 as ipt2 -from IPython.core.inputtransformer2 import make_tokens_by_line, _find_assign_op +import pytest -from textwrap import dedent +from IPython.core import inputtransformer2 as ipt2 +from IPython.core.inputtransformer2 import _find_assign_op, make_tokens_by_line +from IPython.testing.decorators import skip MULTILINE_MAGIC = ("""\ a = f() @@ -253,20 +256,38 @@ def __init__(self, s): nt.assert_equal(_find_assign_op([Tk(s) for s in ('','a','=','b')]), 2) nt.assert_equal(_find_assign_op([Tk(s) for s in ('','(', 'a','=','b', ')', '=' ,'5')]), 6) +examples = [ + pytest.param("a = 1", "complete", None), + pytest.param("for a in range(5):", "incomplete", 4), + pytest.param("for a in range(5):\n if a > 0:", "incomplete", 8), + pytest.param("raise = 2", "invalid", None), + pytest.param("a = [1,\n2,", "incomplete", 0), + pytest.param("(\n))", "incomplete", 0), + pytest.param("\\\r\n", "incomplete", 0), + pytest.param("a = '''\n hi", "incomplete", 3), + pytest.param("def a():\n x=1\n global x", "invalid", None), + pytest.param( + "a \\ ", + "invalid", + None, + marks=pytest.mark.xfail( + reason="Bug in python 3.9.8 – bpo 45738", + condition=sys.version_info[:3] == (3, 9, 8), + raises=SystemError, + strict=True, + ), + ), # Nothing allowed after backslash, + pytest.param("1\\\n+2", "complete", None), +] + + +@skip('Tested on master, skip only on iptest not available on 7.x') +@pytest.mark.xfail( + reason="Bug in python 3.9.8 – bpo 45738", + condition=sys.version_info[:3] == (3, 9, 8), +) def test_check_complete(): cc = ipt2.TransformerManager().check_complete - nt.assert_equal(cc("a = 1"), ('complete', None)) - nt.assert_equal(cc("for a in range(5):"), ('incomplete', 4)) - nt.assert_equal(cc("for a in range(5):\n if a > 0:"), ('incomplete', 8)) - nt.assert_equal(cc("raise = 2"), ('invalid', None)) - nt.assert_equal(cc("a = [1,\n2,"), ('incomplete', 0)) - nt.assert_equal(cc(")"), ('incomplete', 0)) - nt.assert_equal(cc("\\\r\n"), ('incomplete', 0)) - nt.assert_equal(cc("a = '''\n hi"), ('incomplete', 3)) - nt.assert_equal(cc("def a():\n x=1\n global x"), ('invalid', None)) - nt.assert_equal(cc("a \\ "), ('invalid', None)) # Nothing allowed after backslash - nt.assert_equal(cc("1\\\n+2"), ('complete', None)) - nt.assert_equal(cc("exit"), ('complete', None)) example = dedent(""" if True: diff --git a/IPython/core/tests/test_interactiveshell.py b/IPython/core/tests/test_interactiveshell.py index 84bea9cd457..598b6dcd158 100644 --- a/IPython/core/tests/test_interactiveshell.py +++ b/IPython/core/tests/test_interactiveshell.py @@ -1037,6 +1037,22 @@ def test_run_cell_async(): assert result.result == 5 +def test_run_cell_await(): + ip.run_cell("import asyncio") + result = ip.run_cell("await asyncio.sleep(0.01); 10") + assert ip.user_ns["_"] == 10 + + +def test_run_cell_asyncio_run(): + ip.run_cell("import asyncio") + result = ip.run_cell("await asyncio.sleep(0.01); 1") + assert ip.user_ns["_"] == 1 + result = ip.run_cell("asyncio.run(asyncio.sleep(0.01)); 2") + assert ip.user_ns["_"] == 2 + result = ip.run_cell("await asyncio.sleep(0.01); 3") + assert ip.user_ns["_"] == 3 + + def test_should_run_async(): assert not ip.should_run_async("a = 5") assert ip.should_run_async("await x") diff --git a/IPython/core/tests/test_magic_arguments.py b/IPython/core/tests/test_magic_arguments.py index 5dea32dd8ec..a14c47886a1 100644 --- a/IPython/core/tests/test_magic_arguments.py +++ b/IPython/core/tests/test_magic_arguments.py @@ -7,6 +7,7 @@ #----------------------------------------------------------------------------- import argparse +import sys from nose.tools import assert_equal from IPython.core.magic_arguments import (argument, argument_group, kwds, @@ -74,7 +75,12 @@ def foo(self, args): def test_magic_arguments(): - assert_equal(magic_foo1.__doc__, '::\n\n %foo1 [-f FOO]\n\n A docstring.\n\noptional arguments:\n -f FOO, --foo FOO an argument\n') + # “optional arguments” was replaced with “options” in argparse help + # https://docs.python.org/3/whatsnew/3.10.html#argparse + # https://bugs.python.org/issue9694 + options = "optional arguments" if sys.version_info < (3, 10) else "options" + + assert_equal(magic_foo1.__doc__, f"::\n\n %foo1 [-f FOO]\n\n A docstring.\n\n{options}:\n -f FOO, --foo FOO an argument\n") assert_equal(getattr(magic_foo1, 'argcmd_name', None), None) assert_equal(real_name(magic_foo1), 'foo1') assert_equal(magic_foo1(None, ''), argparse.Namespace(foo=None)) @@ -86,32 +92,32 @@ def test_magic_arguments(): assert_equal(magic_foo2(None, ''), argparse.Namespace()) assert hasattr(magic_foo2, 'has_arguments') - assert_equal(magic_foo3.__doc__, '::\n\n %foo3 [-f FOO] [-b BAR] [-z BAZ]\n\n A docstring.\n\noptional arguments:\n -f FOO, --foo FOO an argument\n\nGroup:\n -b BAR, --bar BAR a grouped argument\n\nSecond Group:\n -z BAZ, --baz BAZ another grouped argument\n') + assert_equal(magic_foo3.__doc__, f"::\n\n %foo3 [-f FOO] [-b BAR] [-z BAZ]\n\n A docstring.\n\n{options}:\n -f FOO, --foo FOO an argument\n\nGroup:\n -b BAR, --bar BAR a grouped argument\n\nSecond Group:\n -z BAZ, --baz BAZ another grouped argument\n") assert_equal(getattr(magic_foo3, 'argcmd_name', None), None) assert_equal(real_name(magic_foo3), 'foo3') assert_equal(magic_foo3(None, ''), argparse.Namespace(bar=None, baz=None, foo=None)) assert hasattr(magic_foo3, 'has_arguments') - assert_equal(magic_foo4.__doc__, '::\n\n %foo4 [-f FOO]\n\n A docstring.\n\noptional arguments:\n -f FOO, --foo FOO an argument\n') + assert_equal(magic_foo4.__doc__, f"::\n\n %foo4 [-f FOO]\n\n A docstring.\n\n{options}:\n -f FOO, --foo FOO an argument\n") assert_equal(getattr(magic_foo4, 'argcmd_name', None), None) assert_equal(real_name(magic_foo4), 'foo4') assert_equal(magic_foo4(None, ''), argparse.Namespace()) assert hasattr(magic_foo4, 'has_arguments') - assert_equal(magic_foo5.__doc__, '::\n\n %frobnicate [-f FOO]\n\n A docstring.\n\noptional arguments:\n -f FOO, --foo FOO an argument\n') + assert_equal(magic_foo5.__doc__, f"::\n\n %frobnicate [-f FOO]\n\n A docstring.\n\n{options}:\n -f FOO, --foo FOO an argument\n") assert_equal(getattr(magic_foo5, 'argcmd_name', None), 'frobnicate') assert_equal(real_name(magic_foo5), 'frobnicate') assert_equal(magic_foo5(None, ''), argparse.Namespace(foo=None)) assert hasattr(magic_foo5, 'has_arguments') - assert_equal(magic_magic_foo.__doc__, '::\n\n %magic_foo [-f FOO]\n\n A docstring.\n\noptional arguments:\n -f FOO, --foo FOO an argument\n') + assert_equal(magic_magic_foo.__doc__, f"::\n\n %magic_foo [-f FOO]\n\n A docstring.\n\n{options}:\n -f FOO, --foo FOO an argument\n") assert_equal(getattr(magic_magic_foo, 'argcmd_name', None), None) assert_equal(real_name(magic_magic_foo), 'magic_foo') assert_equal(magic_magic_foo(None, ''), argparse.Namespace(foo=None)) assert hasattr(magic_magic_foo, 'has_arguments') - assert_equal(foo.__doc__, '::\n\n %foo [-f FOO]\n\n A docstring.\n\noptional arguments:\n -f FOO, --foo FOO an argument\n') + assert_equal(foo.__doc__, f"::\n\n %foo [-f FOO]\n\n A docstring.\n\n{options}:\n -f FOO, --foo FOO an argument\n") assert_equal(getattr(foo, 'argcmd_name', None), None) assert_equal(real_name(foo), 'foo') assert_equal(foo(None, ''), argparse.Namespace(foo=None)) diff --git a/IPython/core/tests/test_oinspect.py b/IPython/core/tests/test_oinspect.py index 19c6db7c4f8..df06a810926 100644 --- a/IPython/core/tests/test_oinspect.py +++ b/IPython/core/tests/test_oinspect.py @@ -6,10 +6,11 @@ from inspect import signature, Signature, Parameter +import inspect import os +import pytest import re - -import nose.tools as nt +import sys from .. import oinspect @@ -30,6 +31,10 @@ def setup_module(): inspector = oinspect.Inspector() +class SourceModuleMainTest: + __module__ = "__main__" + + #----------------------------------------------------------------------------- # Local utilities #----------------------------------------------------------------------------- @@ -38,15 +43,28 @@ def setup_module(): # defined, if any code is inserted above, the following line will need to be # updated. Do NOT insert any whitespace between the next line and the function # definition below. -THIS_LINE_NUMBER = 41 # Put here the actual number of this line +THIS_LINE_NUMBER = 46 # Put here the actual number of this line + + +def test_find_source_lines(): + assert oinspect.find_source_lines(test_find_source_lines) == THIS_LINE_NUMBER + 3 + assert oinspect.find_source_lines(type) is None + assert oinspect.find_source_lines(SourceModuleMainTest) is None + assert oinspect.find_source_lines(SourceModuleMainTest()) is None + -from unittest import TestCase +def test_getsource(): + assert oinspect.getsource(type) is None + assert oinspect.getsource(SourceModuleMainTest) is None + assert oinspect.getsource(SourceModuleMainTest()) is None -class Test(TestCase): - def test_find_source_lines(self): - self.assertEqual(oinspect.find_source_lines(Test.test_find_source_lines), - THIS_LINE_NUMBER+6) +def test_inspect_getfile_raises_exception(): + """Check oinspect.find_file/getsource/find_source_lines expectations""" + with pytest.raises(TypeError): + inspect.getfile(type) + with pytest.raises(OSError if sys.version_info >= (3, 10) else TypeError): + inspect.getfile(SourceModuleMainTest) # A couple of utilities to ensure these tests work the same from a source or a @@ -56,11 +74,14 @@ def pyfile(fname): def match_pyfiles(f1, f2): - nt.assert_equal(pyfile(f1), pyfile(f2)) + assert pyfile(f1) == pyfile(f2) def test_find_file(): match_pyfiles(oinspect.find_file(test_find_file), os.path.abspath(__file__)) + assert oinspect.find_file(type) is None + assert oinspect.find_file(SourceModuleMainTest) is None + assert oinspect.find_file(SourceModuleMainTest()) is None def test_find_file_decorated1(): @@ -74,9 +95,9 @@ def wrapper(*a, **kw): @noop1 def f(x): "My docstring" - + match_pyfiles(oinspect.find_file(f), os.path.abspath(__file__)) - nt.assert_equal(f.__doc__, "My docstring") + assert f.__doc__ == "My docstring" def test_find_file_decorated2(): @@ -90,14 +111,14 @@ def noop2(f, *a, **kw): @noop2 def f(x): "My docstring 2" - + match_pyfiles(oinspect.find_file(f), os.path.abspath(__file__)) - nt.assert_equal(f.__doc__, "My docstring 2") - + assert f.__doc__ == "My docstring 2" + def test_find_file_magic(): run = ip.find_line_magic('run') - nt.assert_not_equal(oinspect.find_file(run), None) + assert oinspect.find_file(run) is not None # A few generic objects we can then inspect in the tests below @@ -167,41 +188,46 @@ def __getattr__(self, item): def test_info(): "Check that Inspector.info fills out various fields as expected." - i = inspector.info(Call, oname='Call') - nt.assert_equal(i['type_name'], 'type') - expted_class = str(type(type)) # (Python 3) or - nt.assert_equal(i['base_class'], expted_class) - nt.assert_regex(i['string_form'], "") + i = inspector.info(Call, oname="Call") + assert i["type_name"] == "type" + expected_class = str(type(type)) # (Python 3) or + assert i["base_class"] == expected_class + assert re.search( + "", + i["string_form"], + ) fname = __file__ if fname.endswith(".pyc"): fname = fname[:-1] # case-insensitive comparison needed on some filesystems # e.g. Windows: - nt.assert_equal(i['file'].lower(), compress_user(fname).lower()) - nt.assert_equal(i['definition'], None) - nt.assert_equal(i['docstring'], Call.__doc__) - nt.assert_equal(i['source'], None) - nt.assert_true(i['isclass']) - nt.assert_equal(i['init_definition'], "Call(x, y=1)") - nt.assert_equal(i['init_docstring'], Call.__init__.__doc__) + assert i["file"].lower() == compress_user(fname).lower() + assert i["definition"] == None + assert i["docstring"] == Call.__doc__ + assert i["source"] == None + assert i["isclass"] is True + assert i["init_definition"] == "Call(x, y=1)" + assert i["init_docstring"] == Call.__init__.__doc__ i = inspector.info(Call, detail_level=1) - nt.assert_not_equal(i['source'], None) - nt.assert_equal(i['docstring'], None) + assert i["source"] is not None + assert i["docstring"] == None c = Call(1) c.__doc__ = "Modified instance docstring" i = inspector.info(c) - nt.assert_equal(i['type_name'], 'Call') - nt.assert_equal(i['docstring'], "Modified instance docstring") - nt.assert_equal(i['class_docstring'], Call.__doc__) - nt.assert_equal(i['init_docstring'], Call.__init__.__doc__) - nt.assert_equal(i['call_docstring'], Call.__call__.__doc__) + assert i["type_name"] == "Call" + assert i["docstring"] == "Modified instance docstring" + assert i["class_docstring"] == Call.__doc__ + assert i["init_docstring"] == Call.__init__.__doc__ + assert i["call_docstring"] == Call.__call__.__doc__ + def test_class_signature(): - info = inspector.info(HasSignature, 'HasSignature') - nt.assert_equal(info['init_definition'], "HasSignature(test)") - nt.assert_equal(info['init_docstring'], HasSignature.__init__.__doc__) + info = inspector.info(HasSignature, "HasSignature") + assert info["init_definition"] == "HasSignature(test)" + assert info["init_docstring"] == HasSignature.__init__.__doc__ + def test_info_awkward(): # Just test that this doesn't throw an error. @@ -216,7 +242,7 @@ def test_info_serialliar(): # Nested attribute access should be cut off at 100 levels deep to avoid # infinite loops: https://github.com/ipython/ipython/issues/9122 - nt.assert_less(fib_tracker[0], 9000) + assert fib_tracker[0] < 9000 def support_function_one(x, y=2, *a, **kw): """A simple function.""" @@ -225,14 +251,16 @@ def test_calldef_none(): # We should ignore __call__ for all of these. for obj in [support_function_one, SimpleClass().method, any, str.upper]: i = inspector.info(obj) - nt.assert_is(i['call_def'], None) + assert i["call_def"] is None + def f_kwarg(pos, *, kwonly): pass def test_definition_kwonlyargs(): - i = inspector.info(f_kwarg, oname='f_kwarg') # analysis:ignore - nt.assert_equal(i['definition'], "f_kwarg(pos, *, kwonly)") + i = inspector.info(f_kwarg, oname="f_kwarg") # analysis:ignore + assert i["definition"] == "f_kwarg(pos, *, kwonly)" + def test_getdoc(): class A(object): @@ -243,34 +271,33 @@ class B(object): """standard docstring""" def getdoc(self): return "custom docstring" - + class C(object): """standard docstring""" def getdoc(self): return None - + a = A() b = B() c = C() - - nt.assert_equal(oinspect.getdoc(a), "standard docstring") - nt.assert_equal(oinspect.getdoc(b), "custom docstring") - nt.assert_equal(oinspect.getdoc(c), "standard docstring") + + assert oinspect.getdoc(a) == "standard docstring" + assert oinspect.getdoc(b) == "custom docstring" + assert oinspect.getdoc(c) == "standard docstring" def test_empty_property_has_no_source(): i = inspector.info(property(), detail_level=1) - nt.assert_is(i['source'], None) + assert i["source"] is None def test_property_sources(): - import posixpath # A simple adder whose source and signature stays # the same across Python distributions def simple_add(a, b): "Adds two numbers" return a + b - + class A(object): @property def foo(self): @@ -278,18 +305,18 @@ def foo(self): foo = foo.setter(lambda self, v: setattr(self, 'bar', v)) - dname = property(posixpath.dirname) - adder = property(simple_add) + dname = property(oinspect.getdoc) + adder = property(simple_add) i = inspector.info(A.foo, detail_level=1) - nt.assert_in('def foo(self):', i['source']) - nt.assert_in('lambda self, v:', i['source']) + assert "def foo(self):" in i["source"] + assert "lambda self, v:" in i["source"] i = inspector.info(A.dname, detail_level=1) - nt.assert_in('def dirname(p)', i['source']) - + assert "def getdoc(obj)" in i["source"] + i = inspector.info(A.adder, detail_level=1) - nt.assert_in('def simple_add(a, b)', i['source']) + assert "def simple_add(a, b)" in i["source"] def test_property_docstring_is_in_info_for_detail_level_0(): @@ -299,15 +326,17 @@ def foobar(self): """This is `foobar` property.""" pass - ip.user_ns['a_obj'] = A() - nt.assert_equal( - 'This is `foobar` property.', - ip.object_inspect('a_obj.foobar', detail_level=0)['docstring']) + ip.user_ns["a_obj"] = A() + assert ( + "This is `foobar` property." + == ip.object_inspect("a_obj.foobar", detail_level=0)["docstring"] + ) - ip.user_ns['a_cls'] = A - nt.assert_equal( - 'This is `foobar` property.', - ip.object_inspect('a_cls.foobar', detail_level=0)['docstring']) + ip.user_ns["a_cls"] = A + assert ( + "This is `foobar` property." + == ip.object_inspect("a_cls.foobar", detail_level=0)["docstring"] + ) def test_pdef(): @@ -359,11 +388,11 @@ def test_pinfo_docstring_if_detail_and_no_source(): def bar(self): """ This is a docstring for Foo.bar """ pass - ''' - + ''' + ip.run_cell(obj_def) ip.run_cell('foo = Foo()') - + with AssertNotPrints("Source:"): with AssertPrints('Docstring:'): ip._inspect('pinfo', 'foo', detail_level=0) @@ -388,14 +417,14 @@ def test_pinfo_magic(): def test_init_colors(): # ensure colors are not present in signature info info = inspector.info(HasSignature) - init_def = info['init_definition'] - nt.assert_not_in('[0m', init_def) + init_def = info["init_definition"] + assert "[0m" not in init_def def test_builtin_init(): info = inspector.info(list) init_def = info['init_definition'] - nt.assert_is_not_none(init_def) + assert init_def is not None def test_render_signature_short(): @@ -404,7 +433,7 @@ def short_fun(a=1): pass signature(short_fun), short_fun.__name__, ) - nt.assert_equal(sig, 'short_fun(a=1)') + assert sig == "short_fun(a=1)" def test_render_signature_long(): @@ -420,7 +449,7 @@ def long_function( signature(long_function), long_function.__name__, ) - nt.assert_in(sig, [ + assert sig in [ # Python >=3.9 '''\ long_function( @@ -444,4 +473,4 @@ def long_function( let_us_make_sure_this_is_looong:Union[str, NoneType]=None, ) -> bool\ ''', - ]) + ] \ No newline at end of file diff --git a/IPython/core/tests/test_paths.py b/IPython/core/tests/test_paths.py index ab1c4132a8e..2182cb7cb0c 100644 --- a/IPython/core/tests/test_paths.py +++ b/IPython/core/tests/test_paths.py @@ -160,6 +160,10 @@ def test_get_ipython_dir_7(): @skip_win32 def test_get_ipython_dir_8(): """test_get_ipython_dir_8, test / home directory""" + if not os.access("/", os.W_OK): + # test only when HOME directory actually writable + return + with patch.object(paths, '_writable_dir', lambda path: bool(path)), \ patch.object(paths, 'get_xdg_dir', return_value=None), \ modified_env({ diff --git a/IPython/core/tests/test_pylabtools.py b/IPython/core/tests/test_pylabtools.py index f2adb3b1793..027d6342e1c 100644 --- a/IPython/core/tests/test_pylabtools.py +++ b/IPython/core/tests/test_pylabtools.py @@ -5,7 +5,8 @@ # Distributed under the terms of the Modified BSD License. -from io import UnsupportedOperation, BytesIO +from binascii import a2b_base64 +from io import BytesIO import matplotlib matplotlib.use('Agg') @@ -104,8 +105,11 @@ def test_select_figure_formats_kwargs(): pt.select_figure_formats(ip, 'png', **kwargs) formatter = ip.display_formatter.formatters['image/png'] f = formatter.lookup_by_type(Figure) - cell = f.__closure__[0].cell_contents - nt.assert_equal(cell, kwargs) + cell = f.keywords + expected = kwargs + expected["base64"] = True + expected["fmt"] = "png" + assert cell == expected # check that the formatter doesn't raise fig = plt.figure() @@ -114,7 +118,9 @@ def test_select_figure_formats_kwargs(): plt.draw() formatter.enabled = True png = formatter(fig) - assert png.startswith(_PNG) + assert isinstance(png, str) + png_bytes = a2b_base64(png) + assert png_bytes.startswith(_PNG) def test_select_figure_formats_set(): ip = get_ipython() diff --git a/IPython/core/usage.py b/IPython/core/usage.py index 37024c44567..53219bceb25 100644 --- a/IPython/core/usage.py +++ b/IPython/core/usage.py @@ -305,7 +305,7 @@ _i, _ii, _iii : Previous, next previous, next next previous input _i4, _ih[2:5] : Input history line 4, lines 2-4 -exec _i81 : Execute input history line #81 again +exec(_i81) : Execute input history line #81 again %rep 81 : Edit input history line #81 _, __, ___ : previous, next previous, next next previous output _dh : Directory history diff --git a/IPython/lib/latextools.py b/IPython/lib/latextools.py index cbcc7d9639e..f976f2edb13 100644 --- a/IPython/lib/latextools.py +++ b/IPython/lib/latextools.py @@ -151,9 +151,26 @@ def latex_to_png_dvipng(s, wrap, color='Black', scale=1.0): resolution = round(150*scale) subprocess.check_call( - ["dvipng", "-T", "tight", "-D", str(resolution), "-z", "9", - "-bg", "transparent", "-o", outfile, dvifile, "-fg", color], - cwd=workdir, stdout=devnull, stderr=devnull) + [ + "dvipng", + "-T", + "tight", + "-D", + str(resolution), + "-z", + "9", + "-bg", + "Transparent", + "-o", + outfile, + dvifile, + "-fg", + color, + ], + cwd=workdir, + stdout=devnull, + stderr=devnull, + ) with open(outfile, "rb") as f: return f.read() diff --git a/IPython/lib/tests/test_pretty.py b/IPython/lib/tests/test_pretty.py index ba4c3296694..77941f18daa 100644 --- a/IPython/lib/tests/test_pretty.py +++ b/IPython/lib/tests/test_pretty.py @@ -9,6 +9,7 @@ import os import types import string +import sys import unittest import nose.tools as nt @@ -118,12 +119,15 @@ def test_sets(): yield nt.assert_equal, got_output, expected_output -@skip_without('xxlimited') +@skip_without("xxlimited" if sys.version_info < (3, 10) else "xxlimited_35") def test_pprint_heap_allocated_type(): """ Test that pprint works for heap allocated types. """ - import xxlimited + if sys.version_info < (3, 10): + import xxlimited + else: + import xxlimited_35 as xxlimited output = pretty.pretty(xxlimited.Null) nt.assert_equal(output, 'xxlimited.Null') diff --git a/IPython/paths.py b/IPython/paths.py index bbe3d5c7cad..e19269058af 100644 --- a/IPython/paths.py +++ b/IPython/paths.py @@ -66,6 +66,8 @@ def get_ipython_dir() -> str: warn("IPython parent '{0}' is not a writable location," " using a temp directory.".format(parent)) ipdir = tempfile.mkdtemp() + else: + os.makedirs(ipdir, exist_ok=True) assert isinstance(ipdir, str), "all path manipulation should be str(unicode), but are not." return ipdir diff --git a/IPython/terminal/debugger.py b/IPython/terminal/debugger.py index 9ec246697d4..db8ecac0d26 100644 --- a/IPython/terminal/debugger.py +++ b/IPython/terminal/debugger.py @@ -1,10 +1,8 @@ import asyncio import signal import sys -import threading from IPython.core.debugger import Pdb - from IPython.core.completer import IPCompleter from .ptutils import IPythonPTCompleter from .shortcuts import create_ipython_shortcuts, suspend_to_bg, cursor_in_leading_ws @@ -18,6 +16,7 @@ from prompt_toolkit.shortcuts.prompt import PromptSession from prompt_toolkit.enums import EditingMode from prompt_toolkit.formatted_text import PygmentsTokens +from concurrent.futures import ThreadPoolExecutor from prompt_toolkit import __version__ as ptk_version PTK3 = ptk_version.startswith('3.') @@ -30,6 +29,7 @@ def __init__(self, *args, pt_session_options=None, **kwargs): Pdb.__init__(self, *args, **kwargs) self._ptcomp = None self.pt_init(pt_session_options) + self.thread_executor = ThreadPoolExecutor(1) def pt_init(self, pt_session_options=None): """Initialize the prompt session and the prompt loop @@ -71,7 +71,7 @@ def gen_comp(self, text): enable_history_search=True, mouse_support=self.shell.mouse_support, complete_style=self.shell.pt_complete_style, - style=self.shell.style, + style=getattr(self.shell, "style", None), color_depth=self.shell.color_depth, ) @@ -96,14 +96,13 @@ def cmdloop(self, intro=None): # prompt itself in a different thread (we can't start an event loop # within an event loop). This new thread won't have any event loop # running, and here we run our prompt-loop. - self.preloop() try: if intro is not None: self.intro = intro if self.intro: - self.stdout.write(str(self.intro)+"\n") + print(self.intro, file=self.stdout) stop = None while not stop: if self.cmdqueue: @@ -113,24 +112,10 @@ def cmdloop(self, intro=None): self._ptcomp.ipy_completer.global_namespace = self.curframe.f_globals # Run the prompt in a different thread. - line = '' - keyboard_interrupt = False - - def in_thread(): - nonlocal line, keyboard_interrupt - try: - line = self.pt_app.prompt() - except EOFError: - line = 'EOF' - except KeyboardInterrupt: - keyboard_interrupt = True - - th = threading.Thread(target=in_thread) - th.start() - th.join() - - if keyboard_interrupt: - raise KeyboardInterrupt + try: + line = self.thread_executor.submit(self.pt_app.prompt).result() + except EOFError: + line = "EOF" line = self.precmd(line) stop = self.onecmd(line) diff --git a/IPython/terminal/interactiveshell.py b/IPython/terminal/interactiveshell.py index c7d41679598..4e35aadd616 100644 --- a/IPython/terminal/interactiveshell.py +++ b/IPython/terminal/interactiveshell.py @@ -460,13 +460,15 @@ def prompt_for_code(self): # If we don't do this, people could spawn coroutine with a # while/true inside which will freeze the prompt. + policy = asyncio.get_event_loop_policy() try: - old_loop = asyncio.get_event_loop() + old_loop = policy.get_event_loop() except RuntimeError: - # This happens when the user used `asyncio.run()`. + # This happens when the the event loop is closed, + # e.g. by calling `asyncio.run()`. old_loop = None - asyncio.set_event_loop(self.pt_loop) + policy.set_event_loop(self.pt_loop) try: with patch_stdout(raw=True): text = self.pt_app.prompt( @@ -474,7 +476,8 @@ def prompt_for_code(self): **self._extra_prompt_options()) finally: # Restore the original event loop. - asyncio.set_event_loop(old_loop) + if old_loop is not None: + policy.set_event_loop(old_loop) return text diff --git a/IPython/terminal/magics.py b/IPython/terminal/magics.py index 3c7e82b45e5..42231c3f803 100644 --- a/IPython/terminal/magics.py +++ b/IPython/terminal/magics.py @@ -109,7 +109,7 @@ def cpaste(self, parameter_s=''): Just press enter and type -- (and press enter again) and the block will be what was just pasted. - IPython statements (magics, shell escapes) are not supported (yet). + Shell escapes are not supported (yet). See also -------- @@ -122,9 +122,19 @@ def cpaste(self, parameter_s=''): In [8]: %cpaste Pasting code; enter '--' alone on the line to stop. :>>> a = ["world!", "Hello"] - :>>> print " ".join(sorted(a)) + :>>> print(" ".join(sorted(a))) :-- Hello world! + + :: + In [8]: %cpaste + Pasting code; enter '--' alone on the line to stop. + :>>> %alias_magic t timeit + :>>> %t -n1 pass + :-- + Created `%t` as an alias for `%timeit`. + Created `%%t` as an alias for `%%timeit`. + 354 ns ± 224 ns per loop (mean ± std. dev. of 7 runs, 1 loop each) """ opts, name = self.parse_options(parameter_s, 'rqs:', mode='string') if 'r' in opts: diff --git a/IPython/terminal/pt_inputhooks/qt.py b/IPython/terminal/pt_inputhooks/qt.py index ef2a1f55bbf..b999f5aa173 100644 --- a/IPython/terminal/pt_inputhooks/qt.py +++ b/IPython/terminal/pt_inputhooks/qt.py @@ -64,7 +64,7 @@ def inputhook(context): timer.timeout.connect(event_loop.quit) while not context.input_is_ready(): timer.start(50) # 50 ms - event_loop.exec_() + _exec(event_loop) timer.stop() else: # On POSIX platforms, we can use a file descriptor to quit the event diff --git a/IPython/tests/cve.py b/IPython/tests/cve.py new file mode 100644 index 00000000000..026415a57a4 --- /dev/null +++ b/IPython/tests/cve.py @@ -0,0 +1,56 @@ +""" +Test that CVEs stay fixed. +""" + +from IPython.utils.tempdir import TemporaryDirectory, TemporaryWorkingDirectory +from pathlib import Path +import random +import sys +import os +import string +import subprocess +import time + +def test_cve_2022_21699(): + """ + Here we test CVE-2022-21699. + + We create a temporary directory, cd into it. + Make a profile file that should not be executed and start IPython in a subprocess, + checking for the value. + + + + """ + + dangerous_profile_dir = Path('profile_default') + + dangerous_startup_dir = dangerous_profile_dir / 'startup' + dangerous_expected = 'CVE-2022-21699-'+''.join([random.choice(string.ascii_letters) for i in range(10)]) + + with TemporaryWorkingDirectory() as t: + dangerous_startup_dir.mkdir(parents=True) + (dangerous_startup_dir/ 'foo.py').write_text(f'print("{dangerous_expected}")') + # 1 sec to make sure FS is flushed. + #time.sleep(1) + cmd = [sys.executable,'-m', 'IPython'] + env = os.environ.copy() + env['IPY_TEST_SIMPLE_PROMPT'] = '1' + + + # First we fake old behavior, making sure the profile is/was actually dangerous + p_dangerous = subprocess.Popen(cmd + [f'--profile-dir={dangerous_profile_dir}'], env=env, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out_dangerous, err_dangerouns = p_dangerous.communicate(b"exit\r") + assert dangerous_expected in out_dangerous.decode() + + # Now that we know it _would_ have been dangerous, we test it's not loaded + p = subprocess.Popen(cmd, env=env, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate(b"exit\r") + assert b'IPython' in out + assert dangerous_expected not in out.decode() + assert err == b'' + + + diff --git a/IPython/utils/text.py b/IPython/utils/text.py index 72a87c4145b..256fdab5843 100644 --- a/IPython/utils/text.py +++ b/IPython/utils/text.py @@ -597,6 +597,9 @@ def parse(self, fmt_string): # Re-yield the {foo} style pattern yield (txt + literal_txt[continue_from:], field_name, format_spec, conversion) + def __repr__(self): + return "" + #----------------------------------------------------------------------------- # Utils to columnize a list of string #----------------------------------------------------------------------------- diff --git a/IPython/utils/timing.py b/IPython/utils/timing.py index 32741acdd9c..92f6883c4af 100644 --- a/IPython/utils/timing.py +++ b/IPython/utils/timing.py @@ -62,7 +62,6 @@ def clock2(): Similar to clock(), but return a tuple of user/system times.""" return resource.getrusage(resource.RUSAGE_SELF)[:2] - else: # There is no distinction of user/system time under windows, so we just use # time.perff_counter() for everything... diff --git a/appveyor.yml b/appveyor.yml index d20effdf815..fb5e59500a4 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -23,7 +23,7 @@ init: install: - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" - "%CMD_IN_ENV% python -m pip install --upgrade setuptools pip" - - "%CMD_IN_ENV% pip install nose coverage" + - "%CMD_IN_ENV% pip install nose coverage pytest" - "%CMD_IN_ENV% pip install .[test]" - "%CMD_IN_ENV% mkdir results" - "%CMD_IN_ENV% cd results" diff --git a/codecov.yml b/codecov.yml index eb9b9dff30c..2d3b8bb058c 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,9 +1,26 @@ coverage: status: + patch: off project: - default: + default: false + library: target: auto - threshold: 10 - patch: - default: - target: 0% + paths: ['!.*/tests/.*'] + threshold: 0.1% + tests: + target: auto + paths: ['.*/tests/.*'] +codecov: + require_ci_to_pass: false + +ignore: + - IPython/kernel/* + - IPython/consoleapp.py + - IPython/core/inputsplitter.py + - IPython/lib/inputhook*.py + - IPython/lib/kernel.py + - IPython/utils/jsonutil.py + - IPython/utils/localinterfaces.py + - IPython/utils/log.py + - IPython/utils/signatures.py + - IPython/utils/traitlets.py diff --git a/docs/source/whatsnew/pr/enable-to-add-extra-attrs-to-iframe.rst b/docs/source/whatsnew/pr/enable-to-add-extra-attrs-to-iframe.rst deleted file mode 100644 index 1954bc439b8..00000000000 --- a/docs/source/whatsnew/pr/enable-to-add-extra-attrs-to-iframe.rst +++ /dev/null @@ -1,38 +0,0 @@ -``YouTubeVideo`` autoplay and the ability to add extra attributes to ``IFrame`` -=============================================================================== - -You can add any extra attributes to the ``