From 98b1e108aab70a7574ba2a861024db75821f5834 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 28 Feb 2020 16:48:25 -0800 Subject: [PATCH 01/57] back to dev --- IPython/core/release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index d818b071dc5..feb09c0e335 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -24,7 +24,7 @@ _version_patch = 0 _version_extra = '.dev' # _version_extra = 'b1' -_version_extra = '' # Uncomment this for full releases +# _version_extra = '' # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] From c0dfe4cf6f01ae83662fdc2e80c4832c41e13c49 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 3 Mar 2020 08:00:18 -0800 Subject: [PATCH 02/57] Backport PR #12167: Set .py tempfile suffix when using vi/emacs editor shortcuts. --- IPython/terminal/interactiveshell.py | 1 + IPython/terminal/shortcuts.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/terminal/interactiveshell.py b/IPython/terminal/interactiveshell.py index 9f7d335ead8..7f18ac2d6bd 100644 --- a/IPython/terminal/interactiveshell.py +++ b/IPython/terminal/interactiveshell.py @@ -322,6 +322,7 @@ def prompt(): mouse_support=self.mouse_support, enable_open_in_editor=self.extra_open_editor_shortcuts, color_depth=self.color_depth, + tempfile_suffix=".py", **self._extra_prompt_options()) def _make_style_from_name_or_cls(self, name_or_cls): diff --git a/IPython/terminal/shortcuts.py b/IPython/terminal/shortcuts.py index e44e34277e2..24224d4f804 100644 --- a/IPython/terminal/shortcuts.py +++ b/IPython/terminal/shortcuts.py @@ -250,7 +250,6 @@ def newline_autoindent(event): def open_input_in_editor(event): - event.app.current_buffer.tempfile_suffix = ".py" event.app.current_buffer.open_in_editor() From 3ed12106b61e8d4c894434e4f460c8750e7379b7 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sat, 21 Mar 2020 10:21:28 -0700 Subject: [PATCH 03/57] Backport PR #12198: Fix typos in 7.x series release notes --- docs/source/whatsnew/version7.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/whatsnew/version7.rst b/docs/source/whatsnew/version7.rst index f54f0e5ef0c..62990dfc1e7 100644 --- a/docs/source/whatsnew/version7.rst +++ b/docs/source/whatsnew/version7.rst @@ -7,9 +7,9 @@ IPython 7.13 ============ -IPython 7.13 is the first release of the 7.x branch since master is diverging +IPython 7.13 is the final release of the 7.x branch since master is diverging toward an 8.0. Exiting new features have already been merged in 8.0 and will -not be available on the 7.x branch. All the changes bellow have been backported +not be available on the 7.x branch. All the changes below have been backported from the master branch. From ae6e88f9deefeda6671c8ad80503828a252f04f9 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sat, 28 Mar 2020 11:11:01 -0700 Subject: [PATCH 04/57] Backport PR #12208: support for unicode identifiers --- IPython/core/completerlib.py | 2 +- IPython/core/inputtransformer.py | 4 ++-- IPython/core/inputtransformer2.py | 4 ++-- IPython/core/magics/namespace.py | 6 ------ IPython/core/prefilter.py | 2 +- IPython/core/tests/test_inputtransformer.py | 1 + IPython/core/tests/test_inputtransformer2.py | 8 ++++++++ IPython/core/tests/test_magic.py | 2 ++ IPython/core/tests/test_prefilter.py | 10 ++++++++++ 9 files changed, 27 insertions(+), 12 deletions(-) diff --git a/IPython/core/completerlib.py b/IPython/core/completerlib.py index 9e592b0817e..7860cb67dcb 100644 --- a/IPython/core/completerlib.py +++ b/IPython/core/completerlib.py @@ -52,7 +52,7 @@ TIMEOUT_GIVEUP = 20 # Regular expression for the python import statement -import_re = re.compile(r'(?P[a-zA-Z_][a-zA-Z0-9_]*?)' +import_re = re.compile(r'(?P[^\W\d]\w*?)' r'(?P[/\\]__init__)?' r'(?P%s)$' % r'|'.join(re.escape(s) for s in _suffixes)) diff --git a/IPython/core/inputtransformer.py b/IPython/core/inputtransformer.py index 1c35eb64f32..afeca93cc0e 100644 --- a/IPython/core/inputtransformer.py +++ b/IPython/core/inputtransformer.py @@ -278,8 +278,8 @@ def escaped_commands(line): _initial_space_re = re.compile(r'\s*') _help_end_re = re.compile(r"""(%{0,2} - [a-zA-Z_*][\w*]* # Variable name - (\.[a-zA-Z_*][\w*]*)* # .etc.etc + (?!\d)[\w*]+ # Variable name + (\.(?!\d)[\w*]+)* # .etc.etc ) (\?\??)$ # ? or ?? """, diff --git a/IPython/core/inputtransformer2.py b/IPython/core/inputtransformer2.py index 4562fe01d2c..0443e6829b4 100644 --- a/IPython/core/inputtransformer2.py +++ b/IPython/core/inputtransformer2.py @@ -405,8 +405,8 @@ def transform(self, lines): return lines_before + [new_line] + lines_after _help_end_re = re.compile(r"""(%{0,2} - [a-zA-Z_*][\w*]* # Variable name - (\.[a-zA-Z_*][\w*]*)* # .etc.etc + (?!\d)[\w*]+ # Variable name + (\.(?!\d)[\w*]+)* # .etc.etc ) (\?\??)$ # ? or ?? """, diff --git a/IPython/core/magics/namespace.py b/IPython/core/magics/namespace.py index cef6ddba8d7..acc4620549b 100644 --- a/IPython/core/magics/namespace.py +++ b/IPython/core/magics/namespace.py @@ -208,12 +208,6 @@ def psearch(self, parameter_s=''): %psearch -l list all available object types """ - try: - parameter_s.encode('ascii') - except UnicodeEncodeError: - print('Python identifiers can only contain ascii characters.') - return - # default namespaces to be searched def_search = ['user_local', 'user_global', 'builtin'] diff --git a/IPython/core/prefilter.py b/IPython/core/prefilter.py index dbf185e6a42..bf801f999c4 100644 --- a/IPython/core/prefilter.py +++ b/IPython/core/prefilter.py @@ -37,7 +37,7 @@ class PrefilterError(Exception): # RegExp to identify potential function names -re_fun_name = re.compile(r'[a-zA-Z_]([a-zA-Z0-9_.]*) *$') +re_fun_name = re.compile(r'[^\W\d]([\w.]*) *$') # RegExp to exclude strings with this start from autocalling. In # particular, all binary operators should be excluded, so that if foo is diff --git a/IPython/core/tests/test_inputtransformer.py b/IPython/core/tests/test_inputtransformer.py index 90a1d5afd1e..0d97fd4d6b1 100644 --- a/IPython/core/tests/test_inputtransformer.py +++ b/IPython/core/tests/test_inputtransformer.py @@ -113,6 +113,7 @@ def transform_checker(tests, transformer, **kwargs): (u'%hist2??', "get_ipython().run_line_magic('pinfo2', '%hist2')"), (u'%%hist3?', "get_ipython().run_line_magic('pinfo', '%%hist3')"), (u'%%hist4??', "get_ipython().run_line_magic('pinfo2', '%%hist4')"), + (u'π.foo?', "get_ipython().run_line_magic('pinfo', 'π.foo')"), (u'f*?', "get_ipython().run_line_magic('psearch', 'f*')"), (u'ax.*aspe*?', "get_ipython().run_line_magic('psearch', 'ax.*aspe*')"), (u'a = abc?', "get_ipython().set_next_input('a = abc');" diff --git a/IPython/core/tests/test_inputtransformer2.py b/IPython/core/tests/test_inputtransformer2.py index cde9eca0d06..b29a0196d3f 100644 --- a/IPython/core/tests/test_inputtransformer2.py +++ b/IPython/core/tests/test_inputtransformer2.py @@ -119,6 +119,11 @@ def test(): [r"get_ipython().set_next_input('(a,\nb) = zip');get_ipython().run_line_magic('pinfo', 'zip')" + "\n"] ) +HELP_UNICODE = ( + ["π.foo?\n"], (1, 0), + ["get_ipython().run_line_magic('pinfo', 'π.foo')\n"] +) + def null_cleanup_transformer(lines): """ @@ -223,6 +228,9 @@ def test_transform_help(): tf = ipt2.HelpEnd((1, 0), (2, 8)) nt.assert_equal(tf.transform(HELP_MULTILINE[0]), HELP_MULTILINE[2]) + tf = ipt2.HelpEnd((1, 0), (1, 0)) + nt.assert_equal(tf.transform(HELP_UNICODE[0]), HELP_UNICODE[2]) + def test_find_assign_op_dedent(): """ be careful that empty token like dedent are not counted as parens diff --git a/IPython/core/tests/test_magic.py b/IPython/core/tests/test_magic.py index 9882d052db1..877326ccfc3 100644 --- a/IPython/core/tests/test_magic.py +++ b/IPython/core/tests/test_magic.py @@ -625,6 +625,8 @@ def doctest_precision(): def test_psearch(): with tt.AssertPrints("dict.fromkeys"): _ip.run_cell("dict.fr*?") + with tt.AssertPrints("π.is_integer"): + _ip.run_cell("π = 3.14;\nπ.is_integ*?") def test_timeit_shlex(): """test shlex issues with timeit (#1109)""" diff --git a/IPython/core/tests/test_prefilter.py b/IPython/core/tests/test_prefilter.py index 0e61b4693f7..ca447b3d0b7 100644 --- a/IPython/core/tests/test_prefilter.py +++ b/IPython/core/tests/test_prefilter.py @@ -115,3 +115,13 @@ def __call__(self, x): finally: del ip.user_ns['x'] ip.magic('autocall 0') + + +def test_autocall_should_support_unicode(): + ip.magic('autocall 2') + ip.user_ns['π'] = lambda x: x + try: + nt.assert_equal(ip.prefilter('π 3'),'π(3)') + finally: + ip.magic('autocall 0') + del ip.user_ns['π'] From cfc7ad90136045f885e397bd97a41f37533b644a Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sat, 28 Mar 2020 12:07:49 -0700 Subject: [PATCH 05/57] Backport PR #12210: update link to rpy2 object inventory --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index afdff72b5c4..3dde0f3ecfb 100755 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -223,7 +223,7 @@ def is_stable(extra): htmlhelp_basename = 'ipythondoc' intersphinx_mapping = {'python': ('https://docs.python.org/3/', None), - 'rpy2': ('https://rpy2.readthedocs.io/en/latest/', None), + 'rpy2': ('https://rpy2.github.io/doc/latest/html/', None), 'jupyterclient': ('https://jupyter-client.readthedocs.io/en/latest/', None), 'jupyter': ('https://jupyter.readthedocs.io/en/latest/', None), 'jedi': ('https://jedi.readthedocs.io/en/latest/', None), From 036da90aa4bcaf583985300aebb4b40f94665487 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 17 Apr 2020 15:40:56 -0700 Subject: [PATCH 06/57] Backport PR #12235: Fix compatibility with the latest release of Sphinx --- .travis.yml | 5 ----- docs/source/conf.py | 23 ++++++++++++++++++++++ docs/source/development/wrapperkernels.rst | 2 +- docs/sphinxext/configtraits.py | 3 +-- 4 files changed, 25 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9bb3a9bb0a1..0a748ebdcde 100644 --- a/.travis.yml +++ b/.travis.yml @@ -69,11 +69,6 @@ matrix: python: "3.7" dist: xenial sudo: true - - arch: arm64 - python: "3.7" - dist: xenial - env: ARM64=True IPYTHON_TESTING_TIMEOUT_SCALE=2 - sudo: true - arch: amd64 python: "3.8-dev" dist: xenial diff --git a/docs/source/conf.py b/docs/source/conf.py index 3dde0f3ecfb..5012da2694e 100755 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -121,6 +121,29 @@ def is_stable(extra): numpydoc_class_members_toctree = False warning_is_error = True +import logging + +class ConfigtraitFilter(logging.Filter): + """ + This is a filter to remove in sphinx 3+ the error about config traits being duplicated. + + As we autogenerate configuration traits from, subclasses have lots of + duplication and we want to silence them. Indeed we build on travis with + warnings-as-error set to True, so those duplicate items make the build fail. + """ + + def filter(self, record): + if record.args and record.args[0] == 'configtrait' and 'duplicate' in record.msg: + return False + return True + +ct_filter = ConfigtraitFilter() + +import sphinx.util +logger = sphinx.util.logging.getLogger('sphinx.domains.std').logger + +logger.addFilter(ct_filter) + # The default replacements for |version| and |release|, also used in various # other places throughout the built documents. # diff --git a/docs/source/development/wrapperkernels.rst b/docs/source/development/wrapperkernels.rst index eb0a0488807..d734c30ee99 100644 --- a/docs/source/development/wrapperkernels.rst +++ b/docs/source/development/wrapperkernels.rst @@ -116,7 +116,7 @@ You can override a number of other methods to improve the functionality of your kernel. All of these methods should return a dictionary as described in the relevant section of the :doc:`messaging spec `. -.. class:: MyKernel +.. class:: MyBetterKernel .. method:: do_complete(code, cusor_pos) diff --git a/docs/sphinxext/configtraits.py b/docs/sphinxext/configtraits.py index 4e767694097..2b05d2bf6c8 100644 --- a/docs/sphinxext/configtraits.py +++ b/docs/sphinxext/configtraits.py @@ -8,8 +8,7 @@ Cross reference like this: :configtrait:`Application.log_datefmt`. """ -from sphinx.locale import l_ -from sphinx.util.docfields import Field + def setup(app): app.add_object_type('configtrait', 'configtrait', objname='Config option') From ce107da41dbddbb2ff2a17eb2261855b993f902f Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 10 Mar 2020 12:58:00 -0700 Subject: [PATCH 07/57] Backport PR #12184: exclude broken matplotlib in travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9bb3a9bb0a1..ed4fdad6141 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,7 +38,7 @@ install: - pip install setuptools --upgrade - pip install -e file://$PWD#egg=ipython[test] --upgrade - pip install trio curio --upgrade --upgrade-strategy eager - - pip install pytest matplotlib + - pip install pytest 'matplotlib !=3.2.0' - pip install codecov check-manifest --upgrade script: From 2f2473bd178a6f5cca6bfca8b9d24022a51cf1a0 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sat, 18 Apr 2020 14:39:52 -0700 Subject: [PATCH 08/57] Backport PR #12232: ensure SVG(url=...) is set correctly --- IPython/core/display.py | 49 +++++++++++++++++++++--------- IPython/core/tests/test_display.py | 39 ++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 14 deletions(-) diff --git a/IPython/core/display.py b/IPython/core/display.py index 465c000c55a..d8bdbfa3eeb 100644 --- a/IPython/core/display.py +++ b/IPython/core/display.py @@ -615,9 +615,12 @@ def __init__(self, data=None, url=None, filename=None, metadata=None): filename = data data = None - self.data = data self.url = url self.filename = filename + # because of @data.setter methods in + # subclasses ensure url and filename are set + # before assigning to self.data + self.data = data if metadata is not None: self.metadata = metadata @@ -652,23 +655,36 @@ def reload(self): with open(self.filename, self._read_flags) as f: self.data = f.read() elif self.url is not None: - try: - # Deferred import - from urllib.request import urlopen - response = urlopen(self.url) - self.data = response.read() - # extract encoding from header, if there is one: - encoding = None + # Deferred import + from urllib.request import urlopen + response = urlopen(self.url) + data = response.read() + # extract encoding from header, if there is one: + encoding = None + if 'content-type' in response.headers: for sub in response.headers['content-type'].split(';'): sub = sub.strip() if sub.startswith('charset'): encoding = sub.split('=')[-1].strip() break - # decode data, if an encoding was specified - if encoding: - self.data = self.data.decode(encoding, 'replace') - except: - self.data = None + if 'content-encoding' in response.headers: + # TODO: do deflate? + if 'gzip' in response.headers['content-encoding']: + import gzip + from io import BytesIO + with gzip.open(BytesIO(data), 'rt', encoding=encoding) as fp: + encoding = None + data = fp.read() + + # decode data, if an encoding was specified + # We only touch self.data once since + # subclasses such as SVG have @data.setter methods + # that transform self.data into ... well svg. + if encoding: + self.data = data.decode(encoding, 'replace') + else: + self.data = data + class TextDisplayObject(DisplayObject): """Validate that display data is text""" @@ -736,6 +752,11 @@ def _repr_latex_(self): class SVG(DisplayObject): + """Embed an SVG into the display. + + Note if you just want to view a svg image via a URL use `:class:Image` with + a url=URL keyword argument. + """ _read_flags = 'rb' # wrap data in a property, which extracts the tag, discarding @@ -879,7 +900,7 @@ def data(self, data): data = str(data) if isinstance(data, str): - if getattr(self, 'filename', None) is None: + if self.filename is None and self.url is None: warnings.warn("JSON expects JSONable dict or list, not JSON strings") data = json.loads(data) self._data = data diff --git a/IPython/core/tests/test_display.py b/IPython/core/tests/test_display.py index 0ce4ad2b429..04c93258448 100644 --- a/IPython/core/tests/test_display.py +++ b/IPython/core/tests/test_display.py @@ -72,6 +72,45 @@ def test_retina_png(): nt.assert_equal(md['width'], 1) nt.assert_equal(md['height'], 1) +def test_embed_svg_url(): + import gzip + from io import BytesIO + svg_data = b'' + url = 'http://test.com/circle.svg' + + gzip_svg = BytesIO() + with gzip.open(gzip_svg, 'wb') as fp: + fp.write(svg_data) + gzip_svg = gzip_svg.getvalue() + + def mocked_urlopen(*args, **kwargs): + class MockResponse: + def __init__(self, svg): + self._svg_data = svg + self.headers = {'content-type': 'image/svg+xml'} + + def read(self): + return self._svg_data + + if args[0] == url: + return MockResponse(svg_data) + elif args[0] == url + 'z': + ret= MockResponse(gzip_svg) + ret.headers['content-encoding']= 'gzip' + return ret + return MockResponse(None) + + with mock.patch('urllib.request.urlopen', side_effect=mocked_urlopen): + svg = display.SVG(url=url) + nt.assert_true(svg._repr_svg_().startswith(' Date: Sat, 18 Apr 2020 14:40:51 -0700 Subject: [PATCH 09/57] Backport PR #12219: Catch ValueError --- IPython/core/debugger.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/IPython/core/debugger.py b/IPython/core/debugger.py index ebb8dcac0d8..5c2887ff7cf 100644 --- a/IPython/core/debugger.py +++ b/IPython/core/debugger.py @@ -622,7 +622,11 @@ def do_where(self, arg): Take a number as argument as an (optional) number of context line to print""" if arg: - context = int(arg) + try: + context = int(arg) + except ValueError as err: + self.error(err) + return self.print_stack_trace(context) else: self.print_stack_trace() From bef36f795cc639891ae2f78cddf3ea1bbbb6ba7f Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sat, 18 Apr 2020 14:43:41 -0700 Subject: [PATCH 10/57] Backport PR #12230: Add pretty-printing for types.SimpleNamespace --- IPython/lib/pretty.py | 17 +++++++++++++++++ IPython/lib/tests/test_pretty.py | 20 ++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/IPython/lib/pretty.py b/IPython/lib/pretty.py index 3115d3f6dbf..1cb46b1413d 100644 --- a/IPython/lib/pretty.py +++ b/IPython/lib/pretty.py @@ -648,6 +648,22 @@ def _re_pattern_pprint(obj, p, cycle): p.text(')') +def _types_simplenamespace_pprint(obj, p, cycle): + """The pprint function for types.SimpleNamespace.""" + name = 'namespace' + with p.group(len(name) + 1, name + '(', ')'): + if cycle: + p.text('...') + else: + for idx, (attr, value) in enumerate(obj.__dict__.items()): + if idx: + p.text(',') + p.breakable() + attr_kwarg = '{}='.format(attr) + with p.group(len(attr_kwarg), attr_kwarg): + p.pretty(value) + + def _type_pprint(obj, p, cycle): """The pprint for classes and types.""" # Heap allocated types might not have the module attribute, @@ -741,6 +757,7 @@ def _exception_pprint(obj, p, cycle): types.FunctionType: _function_pprint, types.BuiltinFunctionType: _function_pprint, types.MethodType: _repr_pprint, + types.SimpleNamespace: _types_simplenamespace_pprint, datetime.datetime: _repr_pprint, datetime.timedelta: _repr_pprint, _exception_base: _exception_pprint diff --git a/IPython/lib/tests/test_pretty.py b/IPython/lib/tests/test_pretty.py index 695012d6fc9..ba4c3296694 100644 --- a/IPython/lib/tests/test_pretty.py +++ b/IPython/lib/tests/test_pretty.py @@ -407,6 +407,26 @@ def test_mappingproxy(): nt.assert_equal(pretty.pretty(obj), expected) +def test_simplenamespace(): + SN = types.SimpleNamespace + + sn_recursive = SN() + sn_recursive.first = sn_recursive + sn_recursive.second = sn_recursive + cases = [ + (SN(), "namespace()"), + (SN(x=SN()), "namespace(x=namespace())"), + (SN(a_long_name=[SN(s=string.ascii_lowercase)]*3, a_short_name=None), + "namespace(a_long_name=[namespace(s='abcdefghijklmnopqrstuvwxyz'),\n" + " namespace(s='abcdefghijklmnopqrstuvwxyz'),\n" + " namespace(s='abcdefghijklmnopqrstuvwxyz')],\n" + " a_short_name=None)"), + (sn_recursive, "namespace(first=namespace(...), second=namespace(...))"), + ] + for obj, expected in cases: + nt.assert_equal(pretty.pretty(obj), expected) + + def test_pretty_environ(): dict_repr = pretty.pretty(dict(os.environ)) # reindent to align with 'environ' prefix From 4f4b8aacf8b577b27a5f3a8a29e8ac573b76c416 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sat, 18 Apr 2020 14:45:18 -0700 Subject: [PATCH 11/57] Backport PR #12212: Add option to change HTML attributes in display.Video --- IPython/core/display.py | 29 ++++++++++++------- .../whatsnew/pr/video-display-attributes.rst | 4 +++ 2 files changed, 23 insertions(+), 10 deletions(-) create mode 100644 docs/source/whatsnew/pr/video-display-attributes.rst diff --git a/IPython/core/display.py b/IPython/core/display.py index 465c000c55a..14a3b648d09 100644 --- a/IPython/core/display.py +++ b/IPython/core/display.py @@ -1308,7 +1308,7 @@ def _find_ext(self, s): class Video(DisplayObject): def __init__(self, data=None, url=None, filename=None, embed=False, - mimetype=None, width=None, height=None): + mimetype=None, width=None, height=None, html_attributes="controls"): """Create a video object given raw data or an URL. When this object is returned by an input cell or passed to the @@ -1346,14 +1346,22 @@ def __init__(self, data=None, url=None, filename=None, embed=False, height : int Height in pixels to which to constrain the video in html. If not supplied, defaults to the height of the video. + html_attributes : str + Attributes for the HTML `video element. - """.format(url, width, height) + """.format(url, self.html_attributes, width, height) return output # Embedded videos are base64-encoded. @@ -1411,10 +1420,10 @@ def _repr_html_(self): else: b64_video = b2a_base64(video).decode('ascii').rstrip() - output = """""".format(self.html_attributes, width, height, mimetype, b64_video) return output def reload(self): diff --git a/docs/source/whatsnew/pr/video-display-attributes.rst b/docs/source/whatsnew/pr/video-display-attributes.rst new file mode 100644 index 00000000000..1e7434e895d --- /dev/null +++ b/docs/source/whatsnew/pr/video-display-attributes.rst @@ -0,0 +1,4 @@ +Video HTML attributes +===================== + +Add an option to `IPython.display.Video` to change the attributes of the HTML display of the video. From 51cc220c314cbb79f97152d0ee0706c6861df398 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sat, 18 Apr 2020 20:57:34 -0700 Subject: [PATCH 12/57] Backport PR #12250: remove deprecated usage of matplotlib quality parameter --- IPython/core/tests/test_pylabtools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/core/tests/test_pylabtools.py b/IPython/core/tests/test_pylabtools.py index 7b724000f58..7b64aab111a 100644 --- a/IPython/core/tests/test_pylabtools.py +++ b/IPython/core/tests/test_pylabtools.py @@ -61,7 +61,7 @@ def test_figure_to_jpeg(): ax = fig.add_subplot(1,1,1) ax.plot([1,2,3]) plt.draw() - jpeg = pt.print_figure(fig, 'jpeg', quality=50)[:100].lower() + jpeg = pt.print_figure(fig, 'jpeg', pil_kwargs={'optimize': 50})[:100].lower() assert jpeg.startswith(_JPEG) def test_retina_figure(): From 937ada355f2d2123179a207e2f71fec0ca63be12 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 20 Apr 2020 13:04:54 -0700 Subject: [PATCH 13/57] Backport PR #12253: prepare release notes for 7.14 --- docs/source/whatsnew/development.rst | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/source/whatsnew/development.rst b/docs/source/whatsnew/development.rst index 9d4493a3a53..9fab2b878ec 100644 --- a/docs/source/whatsnew/development.rst +++ b/docs/source/whatsnew/development.rst @@ -24,6 +24,28 @@ Need to be updated: .. DO NOT EDIT THIS LINE BEFORE RELEASE. FEATURE INSERTION POINT. +As a reminder, IPython master has diverged from the 7.x branch, thus master may +have more feature and API changes. + +Important changes: + + - Fix compatibility with Sphinx 3+ :ghpull:`12235` + - Remove deprecated matplotlib parameter usage, compatibility with matplotlib + 3.3+ :`122250` + +Misc Changes + + - set ``.py`` extension when editing current buffer in vi/emacs. :ghpull:`12167` + - support for unicode identifiers in ``?``/``??`` :ghpull:`12208` + - add extra options to the ``Video`` Rich objects :ghpull:`12212` + - add pretty-printing to SimpleNamespace :ghpull:`12230` + +New Deprecation Warnings. + + - Many object present in ``IPython.core.display`` should be imported from + ``IPython.display`` by user and external libraries. Trying to import those + from ``IPython.core.display`` will trigger a deprecation warning in IPython + 7.14+ and _may_ raise an error in IPython 8+ Backwards incompatible changes ------------------------------ From e2f24a7a02ca3e374a31e43f8ed6751704315996 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Wed, 22 Apr 2020 18:38:57 -0700 Subject: [PATCH 14/57] Backport PR #12168: Make IPython.core.debugger interruptible by KeyboardInterrupt --- IPython/core/debugger.py | 28 +++++++++++++++++ IPython/core/tests/test_debugger.py | 31 +++++++++++++++++++ .../whatsnew/pr/interruptible-debugger.rst | 4 +++ 3 files changed, 63 insertions(+) create mode 100644 docs/source/whatsnew/pr/interruptible-debugger.rst diff --git a/IPython/core/debugger.py b/IPython/core/debugger.py index 5c2887ff7cf..89eae383dd2 100644 --- a/IPython/core/debugger.py +++ b/IPython/core/debugger.py @@ -634,6 +634,34 @@ def do_where(self, arg): do_w = do_where +class InterruptiblePdb(Pdb): + """Version of debugger where KeyboardInterrupt exits the debugger altogether.""" + + def cmdloop(self): + """Wrap cmdloop() such that KeyboardInterrupt stops the debugger.""" + try: + return OldPdb.cmdloop(self) + except KeyboardInterrupt: + self.stop_here = lambda frame: False + self.do_quit("") + sys.settrace(None) + self.quitting = False + raise + + def _cmdloop(self): + while True: + try: + # keyboard interrupts allow for an easy way to cancel + # the current command, so allow them during interactive input + self.allow_kbdint = True + self.cmdloop() + self.allow_kbdint = False + break + except KeyboardInterrupt: + self.message('--KeyboardInterrupt--') + raise + + def set_trace(frame=None): """ Start debugging from `frame`. diff --git a/IPython/core/tests/test_debugger.py b/IPython/core/tests/test_debugger.py index dcfd9a42438..7f3720a5ab9 100644 --- a/IPython/core/tests/test_debugger.py +++ b/IPython/core/tests/test_debugger.py @@ -4,8 +4,16 @@ # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. +import signal import sys +import time import warnings +from tempfile import NamedTemporaryFile +from subprocess import check_output, CalledProcessError, PIPE +import subprocess +from unittest.mock import patch +import builtins +import bdb import nose.tools as nt @@ -223,3 +231,26 @@ def can_exit(): >>> sys.settrace(old_trace) ''' + + +def test_interruptible_core_debugger(): + """The debugger can be interrupted. + + The presumption is there is some mechanism that causes a KeyboardInterrupt + (this is implemented in ipykernel). We want to ensure the + KeyboardInterrupt cause debugging to cease. + """ + def raising_input(msg="", called=[0]): + called[0] += 1 + if called[0] == 1: + raise KeyboardInterrupt() + else: + raise AssertionError("input() should only be called once!") + + with patch.object(builtins, "input", raising_input): + debugger.InterruptiblePdb().set_trace() + # The way this test will fail is by set_trace() never exiting, + # resulting in a timeout by the test runner. The alternative + # implementation would involve a subprocess, but that adds issues with + # interrupting subprocesses that are rather complex, so it's simpler + # just to do it this way. diff --git a/docs/source/whatsnew/pr/interruptible-debugger.rst b/docs/source/whatsnew/pr/interruptible-debugger.rst new file mode 100644 index 00000000000..f501b3476b8 --- /dev/null +++ b/docs/source/whatsnew/pr/interruptible-debugger.rst @@ -0,0 +1,4 @@ +IPython.core.debugger.Pdb is now interruptible +============================================== + +A ``KeyboardInterrupt`` will now interrupt IPython's extended debugger, in order to make Jupyter able to interrupt it. From 68a63a052ca74e0817496b67d34d92b32527351a Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Thu, 23 Apr 2020 09:09:31 -0700 Subject: [PATCH 15/57] Backport PR #12261: Do not exit if self.interact is True --- IPython/core/shellapp.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/IPython/core/shellapp.py b/IPython/core/shellapp.py index 33052c44d15..9e8bfbfbb81 100644 --- a/IPython/core/shellapp.py +++ b/IPython/core/shellapp.py @@ -418,7 +418,8 @@ def _run_cmd_line_code(self): fname = os.path.join(fname, "__main__.py") if not os.path.exists(fname): self.log.warning("File '%s' doesn't exist", fname) - self.exit(2) + if not self.interact: + self.exit(2) try: self._exec_file(fname, shell_futures=True) except: From 01aae92a85270fa59218e1e57eb94a112b87713e Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 1 May 2020 11:52:47 -0700 Subject: [PATCH 16/57] Backport PR #12272: Update what's new for 7.14 release --- docs/source/whatsnew/development.rst | 20 -------------- docs/source/whatsnew/version7.rst | 39 ++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 20 deletions(-) diff --git a/docs/source/whatsnew/development.rst b/docs/source/whatsnew/development.rst index 9fab2b878ec..99821d1105c 100644 --- a/docs/source/whatsnew/development.rst +++ b/docs/source/whatsnew/development.rst @@ -27,26 +27,6 @@ Need to be updated: As a reminder, IPython master has diverged from the 7.x branch, thus master may have more feature and API changes. -Important changes: - - - Fix compatibility with Sphinx 3+ :ghpull:`12235` - - Remove deprecated matplotlib parameter usage, compatibility with matplotlib - 3.3+ :`122250` - -Misc Changes - - - set ``.py`` extension when editing current buffer in vi/emacs. :ghpull:`12167` - - support for unicode identifiers in ``?``/``??`` :ghpull:`12208` - - add extra options to the ``Video`` Rich objects :ghpull:`12212` - - add pretty-printing to SimpleNamespace :ghpull:`12230` - -New Deprecation Warnings. - - - Many object present in ``IPython.core.display`` should be imported from - ``IPython.display`` by user and external libraries. Trying to import those - from ``IPython.core.display`` will trigger a deprecation warning in IPython - 7.14+ and _may_ raise an error in IPython 8+ - Backwards incompatible changes ------------------------------ diff --git a/docs/source/whatsnew/version7.rst b/docs/source/whatsnew/version7.rst index 62990dfc1e7..f68c8101404 100644 --- a/docs/source/whatsnew/version7.rst +++ b/docs/source/whatsnew/version7.rst @@ -2,6 +2,45 @@ 7.x Series ============ +.. _version 714: + +IPython 7.14 +============ + +IPython 7.14 is a minor release that fix a couple of bugs and prepare +compatibility with new or future versions of some libraries. + +Important changes: +------------------ + + - Fix compatibility with Sphinx 3+ :ghpull:`12235` + - Remove deprecated matplotlib parameter usage, compatibility with matplotlib + 3.3+ :`122250` + +Misc Changes +------------ + + - set ``.py`` extension when editing current buffer in vi/emacs. :ghpull:`12167` + - support for unicode identifiers in ``?``/``??`` :ghpull:`12208` + - add extra options to the ``Video`` Rich objects :ghpull:`12212` + - add pretty-printing to ``SimpleNamespace`` :ghpull:`12230` + +Pending deprecated imports +-------------------------- + +Many object present in ``IPython.core.display`` are there for internal use only, +and should already been imported from ``IPython.display`` by users and external +libraries. Trying to import those from ``IPython.core.display`` is still possible +but will trigger a +deprecation warning in later versions of IPython and will become errors in the +future. + +This will simplify compatibility with other Python kernels (like Xeus-Python), +and simplify code base. + + + + .. _version 713: IPython 7.13 From e78905a3cb7222a00ea17a4b16a2cbc048e93e4f Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 1 May 2020 12:05:31 -0700 Subject: [PATCH 17/57] release 7.14.0 --- IPython/core/release.py | 4 ++-- docs/source/whatsnew/github-stats-7.rst | 25 ++++++++++++++++++++++++- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index feb09c0e335..c01d886b892 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -20,11 +20,11 @@ # release. 'dev' as a _version_extra string means this is a development # version _version_major = 7 -_version_minor = 13 +_version_minor = 14 _version_patch = 0 _version_extra = '.dev' # _version_extra = 'b1' -# _version_extra = '' # Uncomment this for full releases +_version_extra = '' # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] diff --git a/docs/source/whatsnew/github-stats-7.rst b/docs/source/whatsnew/github-stats-7.rst index 91955bb2afa..2c0c5d81cf3 100644 --- a/docs/source/whatsnew/github-stats-7.rst +++ b/docs/source/whatsnew/github-stats-7.rst @@ -1,9 +1,31 @@ Issues closed in the 7.x development cycle ========================================== -Issues closed in 8.12 +Issues closed in 7.14 --------------------- +GitHub stats for 2020/02/29 - 2020/05/01 (tag: 7.13.0) + +These lists are automatically generated, and may be incomplete or contain duplicates. + +We closed 0 issues and merged 30 pull requests. +The full list can be seen `on GitHub `__ + +The following 10 authors contributed 47 commits. + +* Eric Wieser +* foobarbyte +* Ian Castleden +* Itamar Turner-Trauring +* Lumir Balhar +* Markus Wageringel +* Matthias Bussonnier +* Matthieu Ancellin +* Quentin Peter +* Theo Ouzhinski + +Issues closed in 7.13 +--------------------- GitHub stats for 2020/02/01 - 2020/02/28 (tag: 7.12.0) @@ -27,6 +49,7 @@ The following 12 authors contributed 108 commits. * Nathan Goldbaum * Terry Davis + Issues closed in 7.12 --------------------- From 56db1609ad109d1d9dfd617dbaa8f3416a453676 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 1 May 2020 12:06:39 -0700 Subject: [PATCH 18/57] back to dev --- IPython/core/release.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index c01d886b892..1bc5595cac9 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -20,11 +20,11 @@ # release. 'dev' as a _version_extra string means this is a development # version _version_major = 7 -_version_minor = 14 +_version_minor = 15 _version_patch = 0 _version_extra = '.dev' # _version_extra = 'b1' -_version_extra = '' # Uncomment this for full releases +# _version_extra = '' # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] From b8beeb3a94f0cfb113132763912b3d5cb306eaa1 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 4 May 2020 15:22:45 -0700 Subject: [PATCH 19/57] Backport PR #12280: Unify API between completers. --- IPython/core/completer.py | 5 ++--- IPython/core/tests/test_completer.py | 15 +++++++++++---- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/IPython/core/completer.py b/IPython/core/completer.py index 985ac5ab9eb..32144e1e9b6 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -1699,8 +1699,6 @@ def latex_matches(self, text): u"""Match Latex syntax for unicode characters. This does both ``\\alp`` -> ``\\alpha`` and ``\\alpha`` -> ``α`` - - Used on Python 3 only. """ slashpos = text.rfind('\\') if slashpos > -1: @@ -1713,7 +1711,8 @@ def latex_matches(self, text): # If a user has partially typed a latex symbol, give them # a full list of options \al -> [\aleph, \alpha] matches = [k for k in latex_symbols if k.startswith(s)] - return s, matches + if matches: + return s, matches return u'', [] def dispatch_custom_completer(self, text): diff --git a/IPython/core/tests/test_completer.py b/IPython/core/tests/test_completer.py index 2920d453936..2c19e2e0187 100644 --- a/IPython/core/tests/test_completer.py +++ b/IPython/core/tests/test_completer.py @@ -224,20 +224,27 @@ def test_latex_completions(self): nt.assert_in("\\alpha", matches) nt.assert_in("\\aleph", matches) + def test_latex_no_results(self): + """ + forward latex should really return nothing in either field if nothing is found. + """ + ip = get_ipython() + text, matches = ip.Completer.latex_matches("\\really_i_should_match_nothing") + nt.assert_equal(text, "") + nt.assert_equal(matches, []) + def test_back_latex_completion(self): ip = get_ipython() # do not return more than 1 matches fro \beta, only the latex one. name, matches = ip.complete("\\β") - nt.assert_equal(len(matches), 1) - nt.assert_equal(matches[0], "\\beta") + nt.assert_equal(matches, ['\\beta']) def test_back_unicode_completion(self): ip = get_ipython() name, matches = ip.complete("\\Ⅴ") - nt.assert_equal(len(matches), 1) - nt.assert_equal(matches[0], "\\ROMAN NUMERAL FIVE") + nt.assert_equal(matches, ["\\ROMAN NUMERAL FIVE"]) def test_forward_unicode_completion(self): ip = get_ipython() From db177d900add8dbdea1e7973f4300c19fa12b851 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 8 May 2020 09:49:50 -0700 Subject: [PATCH 20/57] Backport PR #12296: Delete 109-112 for removing external test --- IPython/core/tests/test_display.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/IPython/core/tests/test_display.py b/IPython/core/tests/test_display.py index 04c93258448..95f1eb622e4 100644 --- a/IPython/core/tests/test_display.py +++ b/IPython/core/tests/test_display.py @@ -105,11 +105,6 @@ def read(self): nt.assert_true(svg._repr_svg_().startswith(' Date: Sat, 9 May 2020 09:58:12 -0700 Subject: [PATCH 21/57] Backport PR #12284: Try to elide long completion based on user input. --- .travis.yml | 3 +- IPython/core/interactiveshell.py | 2 +- IPython/terminal/ptutils.py | 32 ++++++++++++++++--- .../terminal/tests/test_interactivshell.py | 30 +++++++++++++---- 4 files changed, 55 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index ad1aab7f253..c4a309e1ef0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,7 +38,7 @@ install: - pip install setuptools --upgrade - pip install -e file://$PWD#egg=ipython[test] --upgrade - pip install trio curio --upgrade --upgrade-strategy eager - - pip install pytest 'matplotlib !=3.2.0' + - pip install pytest 'matplotlib !=3.2.0' mypy - pip install codecov check-manifest --upgrade script: @@ -50,6 +50,7 @@ script: fi - cd /tmp && iptest --coverage xml && cd - - pytest IPython + - mypy --ignore-missing-imports -m IPython.terminal.ptutils # On the latest Python (on Linux) only, make sure that the docs build. - | if [[ "$TRAVIS_PYTHON_VERSION" == "3.7" ]] && [[ "$TRAVIS_OS_NAME" == "linux" ]]; then diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 848a14a481c..f5fdba6692b 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -2217,7 +2217,7 @@ def set_custom_completer(self, completer, pos=0): The position argument (defaults to 0) is the index in the completers list where you want the completer to be inserted.""" - newcomp = types.MethodType(completer,self.Completer) + newcomp = types.MethodType(completer, self.Completer) self.Completer.custom_matchers.insert(pos,newcomp) def set_completer_frame(self, frame=None): diff --git a/IPython/terminal/ptutils.py b/IPython/terminal/ptutils.py index 47fd6f40612..ed7ad45eb45 100644 --- a/IPython/terminal/ptutils.py +++ b/IPython/terminal/ptutils.py @@ -23,7 +23,7 @@ _completion_sentinel = object() -def _elide(string, *, min_elide=30): +def _elide_point(string:str, *, min_elide=30)->str: """ If a string is long enough, and has at least 3 dots, replace the middle part with ellipses. @@ -53,6 +53,26 @@ def _elide(string, *, min_elide=30): return string +def _elide_typed(string:str, typed:str, *, min_elide:int=30)->str: + """ + Elide the middle of a long string if the beginning has already been typed. + """ + + if len(string) < min_elide: + return string + cut_how_much = len(typed)-3 + if cut_how_much < 7: + return string + if string.startswith(typed) and len(string)> len(typed): + return f"{string[:3]}\N{HORIZONTAL ELLIPSIS}{string[cut_how_much:]}" + return string + +def _elide(string:str, typed:str, min_elide=30)->str: + return _elide_typed( + _elide_point(string, min_elide=min_elide), + typed, min_elide=min_elide) + + def _adjust_completion_text_based_on_context(text, body, offset): if text.endswith('=') and len(body) > offset and body[offset] == '=': @@ -89,7 +109,11 @@ def get_completions(self, document, complete_event): cursor_col = document.cursor_position_col cursor_position = document.cursor_position offset = cursor_to_position(body, cursor_row, cursor_col) - yield from self._get_completions(body, offset, cursor_position, self.ipy_completer) + try: + yield from self._get_completions(body, offset, cursor_position, self.ipy_completer) + except Exception as e: + from traceback import print_tb + print_tb(e) @staticmethod def _get_completions(body, offset, cursor_position, ipyc): @@ -128,9 +152,9 @@ def _get_completions(body, offset, cursor_position, ipyc): adjusted_text = _adjust_completion_text_based_on_context(c.text, body, offset) if c.type == 'function': - yield Completion(adjusted_text, start_position=c.start - offset, display=_elide(display_text+'()'), display_meta=c.type+c.signature) + yield Completion(adjusted_text, start_position=c.start - offset, display=_elide(display_text+'()', body[c.start:c.end]), display_meta=c.type+c.signature) else: - yield Completion(adjusted_text, start_position=c.start - offset, display=_elide(display_text), display_meta=c.type) + yield Completion(adjusted_text, start_position=c.start - offset, display=_elide(display_text, body[c.start:c.end]), display_meta=c.type) class IPythonPTLexer(Lexer): """ diff --git a/IPython/terminal/tests/test_interactivshell.py b/IPython/terminal/tests/test_interactivshell.py index 6bacc8ea48a..640e5d482ae 100644 --- a/IPython/terminal/tests/test_interactivshell.py +++ b/IPython/terminal/tests/test_interactivshell.py @@ -17,14 +17,32 @@ class TestElide(unittest.TestCase): def test_elide(self): - _elide('concatenate((a1, a2, ...), axis') # do not raise - _elide('concatenate((a1, a2, ..), . axis') # do not raise - nt.assert_equal(_elide('aaaa.bbbb.ccccc.dddddd.eeeee.fffff.gggggg.hhhhhh'), 'aaaa.b…g.hhhhhh') - + _elide('concatenate((a1, a2, ...), axis', '') # do not raise + _elide('concatenate((a1, a2, ..), . axis', '') # do not raise + nt.assert_equal(_elide('aaaa.bbbb.ccccc.dddddd.eeeee.fffff.gggggg.hhhhhh',''), 'aaaa.b…g.hhhhhh') + test_string = os.sep.join(['', 10*'a', 10*'b', 10*'c', '']) expect_stirng = os.sep + 'a' + '\N{HORIZONTAL ELLIPSIS}' + 'b' + os.sep + 10*'c' - nt.assert_equal(_elide(test_string), expect_stirng) - + nt.assert_equal(_elide(test_string, ''), expect_stirng) + + def test_elide_typed_normal(self): + nt.assert_equal(_elide('the quick brown fox jumped over the lazy dog', 'the quick brown fox', min_elide=10), 'the…fox jumped over the lazy dog') + + + def test_elide_typed_short_match(self): + """ + if the match is too short we don't elide. + avoid the "the...the" + """ + nt.assert_equal(_elide('the quick brown fox jumped over the lazy dog', 'the', min_elide=10), 'the quick brown fox jumped over the lazy dog') + + def test_elide_typed_no_match(self): + """ + if the match is too short we don't elide. + avoid the "the...the" + """ + # here we typed red instead of brown + nt.assert_equal(_elide('the quick brown fox jumped over the lazy dog', 'the quick red fox', min_elide=10), 'the quick brown fox jumped over the lazy dog') class TestContextAwareCompletion(unittest.TestCase): From 4d9c6ba67ff2d9b1161f9bad97868de964f14bc4 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sun, 10 May 2020 16:23:46 -0700 Subject: [PATCH 22/57] Backport PR #12303: Fix nightly parso error --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c4a309e1ef0..00c5e3f6bbc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,7 +46,7 @@ script: - | if [[ "$TRAVIS_PYTHON_VERSION" == "nightly" ]]; then # on nightly fake parso known the grammar - cp /home/travis/virtualenv/python3.8-dev/lib/python3.8/site-packages/parso/python/grammar37.txt /home/travis/virtualenv/python3.8-dev/lib/python3.8/site-packages/parso/python/grammar38.txt + cp /home/travis/virtualenv/python3.9-dev/lib/python3.9/site-packages/parso/python/grammar38.txt /home/travis/virtualenv/python3.9-dev/lib/python3.9/site-packages/parso/python/grammar39.txt fi - cd /tmp && iptest --coverage xml && cd - - pytest IPython From bd722769ea84ca8b1798cda7d3cc7c0862764d30 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 11 May 2020 17:04:40 -0700 Subject: [PATCH 23/57] Backport PR #12307: Fix some test on python 3.9 (nightly). --- IPython/core/tests/test_async_helpers.py | 13 ++++++++----- IPython/core/tests/test_oinspect.py | 8 ++++++++ IPython/core/tests/test_ultratb.py | 16 ++++++++++------ 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/IPython/core/tests/test_async_helpers.py b/IPython/core/tests/test_async_helpers.py index 86a516ccc1f..11c475874d7 100644 --- a/IPython/core/tests/test_async_helpers.py +++ b/IPython/core/tests/test_async_helpers.py @@ -8,7 +8,7 @@ from textwrap import dedent, indent from unittest import TestCase from IPython.testing.decorators import skip_without - +import sys iprc = lambda x: ip.run_cell(dedent(x)).raise_error() iprc_nr = lambda x: ip.run_cell(dedent(x)) @@ -275,10 +275,13 @@ def test_autoawait(self): await sleep(0.1) """ ) - - def test_memory_error(self): - with self.assertRaises(MemoryError): - iprc("(" * 200 + ")" * 200) + + if sys.version_info < (3,9): + # new pgen parser in 3.9 does not raise MemoryError on too many nested + # parens anymore + def test_memory_error(self): + with self.assertRaises(MemoryError): + iprc("(" * 200 + ")" * 200) @skip_without('curio') def test_autoawait_curio(self): diff --git a/IPython/core/tests/test_oinspect.py b/IPython/core/tests/test_oinspect.py index 8bdcfad98ce..19c6db7c4f8 100644 --- a/IPython/core/tests/test_oinspect.py +++ b/IPython/core/tests/test_oinspect.py @@ -421,6 +421,14 @@ def long_function( long_function.__name__, ) nt.assert_in(sig, [ + # Python >=3.9 + '''\ +long_function( + a_really_long_parameter: int, + and_another_long_one: bool = False, + let_us_make_sure_this_is_looong: Optional[str] = None, +) -> bool\ +''', # Python >=3.7 '''\ long_function( diff --git a/IPython/core/tests/test_ultratb.py b/IPython/core/tests/test_ultratb.py index 3ab0ce3cf00..3751117b692 100644 --- a/IPython/core/tests/test_ultratb.py +++ b/IPython/core/tests/test_ultratb.py @@ -252,12 +252,16 @@ def test_non_syntaxerror(self): with tt.AssertPrints('QWERTY'): ip.showsyntaxerror() - -class MemoryErrorTest(unittest.TestCase): - def test_memoryerror(self): - memoryerror_code = "(" * 200 + ")" * 200 - with tt.AssertPrints("MemoryError"): - ip.run_cell(memoryerror_code) +import sys +if sys.version_info < (3,9): + """ + New 3.9 Pgen Parser does not raise Memory error, except on failed malloc. + """ + class MemoryErrorTest(unittest.TestCase): + def test_memoryerror(self): + memoryerror_code = "(" * 200 + ")" * 200 + with tt.AssertPrints("MemoryError"): + ip.run_cell(memoryerror_code) class Python3ChainedExceptionsTest(unittest.TestCase): From e3cac25bd6134de06149c3c4446ba8ed25af6e43 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 18 May 2020 16:11:23 -0700 Subject: [PATCH 24/57] Backport PR #12314: Enable high DPI scaling in qt.py --- IPython/terminal/pt_inputhooks/qt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/IPython/terminal/pt_inputhooks/qt.py b/IPython/terminal/pt_inputhooks/qt.py index ce0a9da86bd..1504c3bde72 100644 --- a/IPython/terminal/pt_inputhooks/qt.py +++ b/IPython/terminal/pt_inputhooks/qt.py @@ -25,6 +25,7 @@ def inputhook(context): 'variable. Deactivate Qt5 code.' ) return + QtCore.QCoreApplication.setAttribute(QtCore.Qt.AA_EnableHighDpiScaling) _appref = app = QtGui.QApplication([" "]) event_loop = QtCore.QEventLoop(app) From ddc7e997d11f9f5fad220ecb5888f6c23e8b430f Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 19 May 2020 08:25:52 -0700 Subject: [PATCH 25/57] Backport PR #12321: Fix broken link in docs --- docs/source/config/details.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/config/details.rst b/docs/source/config/details.rst index 6685b14d8f6..9e63232d81d 100644 --- a/docs/source/config/details.rst +++ b/docs/source/config/details.rst @@ -247,7 +247,7 @@ VI input mode to ``Normal`` when in insert mode:: For more information on filters and what you can do with the ``event`` object, `see the prompt_toolkit docs -`__. +`__. Enter to execute From 33036bd61631c789b2a397f2b401d8759323475a Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sat, 23 May 2020 09:43:28 -0700 Subject: [PATCH 26/57] Backport PR #12328: document ipython and ipykernel global configuration --- docs/source/config/intro.rst | 98 ++++++++++++++++++++++++++++++------ 1 file changed, 83 insertions(+), 15 deletions(-) diff --git a/docs/source/config/intro.rst b/docs/source/config/intro.rst index 115315c9ddb..97dc954881b 100644 --- a/docs/source/config/intro.rst +++ b/docs/source/config/intro.rst @@ -19,27 +19,29 @@ To create the blank config files, run:: ipython profile create [profilename] If you leave out the profile name, the files will be created for the -``default`` profile (see :ref:`profiles`). These will typically be -located in :file:`~/.ipython/profile_default/`, and will be named -:file:`ipython_config.py`, :file:`ipython_notebook_config.py`, etc. -The settings in :file:`ipython_config.py` apply to all IPython commands. +``default`` profile (see :ref:`profiles`). These will typically be located in +:file:`~/.ipython/profile_default/`, and will be named +:file:`ipython_config.py`, for hitorical reasons you may also find fiels +named with IPytho nprefix instead of Jupyter: +:file:`ipython_notebook_config.py`, etc. The settings in +:file:`ipython_config.py` apply to all IPython commands. -The files typically start by getting the root config object:: - - c = get_config() +by default, configuration files are fully featured Python scripts that can +execute arbitrary code, the main usage is to set value on the config object +``c`` which exist in your configuration file. You can then configure class attributes like this:: c.InteractiveShell.automagic = False Be careful with spelling--incorrect names will simply be ignored, with -no error. +no error. -To add to a collection which may have already been defined elsewhere, -you can use methods like those found on lists, dicts and sets: append, -extend, :meth:`~traitlets.config.LazyConfigValue.prepend` (like -extend, but at the front), add and update (which works both for dicts -and sets):: +To add to a collection which may have already been defined elsewhere or have +default values, you can use methods like those found on lists, dicts and +sets: append, extend, :meth:`~traitlets.config.LazyConfigValue.prepend` (like +extend, but at the front), add and update (which works both for dicts and +sets):: c.InteractiveShellApp.extensions.append('Cython') @@ -52,7 +54,6 @@ Example config file :: # sample ipython_config.py - c = get_config() c.TerminalIPythonApp.display_banner = True c.InteractiveShellApp.log_level = 20 @@ -78,6 +79,38 @@ Example config file ('la', 'ls -al') ] +Json Configuration files +------------------------ + +in case where executability of configuration can be problematic, or +configurations need to be modified programatically, IPython also support a +limited set of functionalities via ``.json`` config files. + +You can defined most of the configuration options via a json object which +hierarchy represent the value you woudl normally set on the ``c`` object of +``.py`` configuration files. The following ``ipython_config.json`` file:: + + { + "InteractiveShell": { + "colors": "LightBG", + "editor": "nano" + }, + "InteractiveShellApp": { + "extensions": [ + "myextension" + ] + } + } + +Is equivalent to the following ``ipython_config.py``:: + + c.InteractiveShellApp.extensions = [ + 'myextension' + ] + + c.InteractiveShell.colors = 'LightBG' + c.InteractiveShell.editor = 'nano' + Command line arguments ---------------------- @@ -94,7 +127,7 @@ Many frequently used options have short aliases and flags, such as To see all of these abbreviated options, run:: ipython --help - ipython notebook --help + jupyter notebook --help # etc. Options specified at the command line, in either format, override @@ -163,3 +196,38 @@ the directory :file:`~/.ipython/` by default. To see where IPython is looking for the IPython directory, use the command ``ipython locate``, or the Python function :func:`IPython.paths.get_ipython_dir`. + + +Systemwide configuration +======================== + +It can be usefull to deploy systemwide ipython or ipykernel configuration +when managing environment for many users. At startup time IPython and +IPykernel will search for configuration file in multiple systemwide +locations, mainly: + + - ``/etc/ipython/`` + - ``/usr/local/etc/ipython/`` + +When the global install is a standalone python distribution it may also +search in distribution specific location, for example: + + - ``$ANACONDA_LOCATION/etc/ipython/`` + +In those locations, Terminal IPython will look for a file called +``ipython_config.py`` and ``ipython_config.json``, ipykernel will look for +``ipython_kernel_config.py`` and ``ipython_kernel.json``. + +Configuration files are loaded in order and merged with configuration on +later locatoin taking precedence on earlier locations (that is to say a user +can overwrite a systemwide configuration option). + +You can see all locations in which IPython is looking for configuration files +by starting ipython in debug mode:: + + $ ipython --debug -c 'exit()' + +Identically with ipykernel though the command is currently blocking until +this process is killed with ``Ctrl-\``:: + + $ python -m ipykernel --debug \ No newline at end of file From a816d9513623f6096b738ee11c460c9e6398e851 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 25 May 2020 19:54:04 -0700 Subject: [PATCH 27/57] Backport PR #12335: Better documentation for `space_for_menu`. --- IPython/terminal/interactiveshell.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/IPython/terminal/interactiveshell.py b/IPython/terminal/interactiveshell.py index 7f18ac2d6bd..b3bb5777450 100644 --- a/IPython/terminal/interactiveshell.py +++ b/IPython/terminal/interactiveshell.py @@ -102,7 +102,11 @@ class TerminalInteractiveShell(InteractiveShell): mime_renderers = Dict().tag(config=True) space_for_menu = Integer(6, help='Number of line at the bottom of the screen ' - 'to reserve for the completion menu' + 'to reserve for the tab completion menu, ' + 'search history, ...etc, the height of ' + 'these menus will at most this value. ' + 'Increase it is you prefer long and skinny ' + 'menus, decrease for short and wide.' ).tag(config=True) pt_app = None From 7f7f6b8a6bc1dba26b30faab35c8cf8053aebb51 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 25 May 2020 19:54:17 -0700 Subject: [PATCH 28/57] Backport PR #12336: Fix auto formatting to be called only once. --- IPython/terminal/shortcuts.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/IPython/terminal/shortcuts.py b/IPython/terminal/shortcuts.py index 24224d4f804..a23fa091a0e 100644 --- a/IPython/terminal/shortcuts.py +++ b/IPython/terminal/shortcuts.py @@ -130,8 +130,10 @@ def newline_or_execute(event): # if all we have after the cursor is whitespace: reformat current text # before cursor after_cursor = d.text[d.cursor_position:] + reformatted = False if not after_cursor.strip(): reformat_text_before_cursor(b, d, shell) + reformatted = True if not (d.on_last_line or d.cursor_position_row >= d.line_count - d.empty_line_count_at_the_end() ): @@ -142,7 +144,8 @@ def newline_or_execute(event): return if (status != 'incomplete') and b.accept_handler: - reformat_text_before_cursor(b, d, shell) + if not reformatted: + reformat_text_before_cursor(b, d, shell) b.validate_and_handle() else: if shell.autoindent: From c89c9b18168b7babbac66bf6d6c2ba08731c92ee Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 26 May 2020 09:13:32 -0700 Subject: [PATCH 29/57] Backport PR #12340: docs typoes --- docs/source/config/intro.rst | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/source/config/intro.rst b/docs/source/config/intro.rst index 97dc954881b..6dccde1348c 100644 --- a/docs/source/config/intro.rst +++ b/docs/source/config/intro.rst @@ -11,24 +11,24 @@ Many of IPython's classes have configurable attributes (see :doc:`options/index` for the list). These can be configured in several ways. -Python config files -------------------- +Python configuration files +-------------------------- -To create the blank config files, run:: +To create the blank configuration files, run:: ipython profile create [profilename] If you leave out the profile name, the files will be created for the ``default`` profile (see :ref:`profiles`). These will typically be located in :file:`~/.ipython/profile_default/`, and will be named -:file:`ipython_config.py`, for hitorical reasons you may also find fiels -named with IPytho nprefix instead of Jupyter: +:file:`ipython_config.py`, for historical reasons you may also find files +named with IPython prefix instead of Jupyter: :file:`ipython_notebook_config.py`, etc. The settings in :file:`ipython_config.py` apply to all IPython commands. -by default, configuration files are fully featured Python scripts that can -execute arbitrary code, the main usage is to set value on the config object -``c`` which exist in your configuration file. +By default, configuration files are fully featured Python scripts that can +execute arbitrary code, the main usage is to set value on the configuration +object ``c`` which exist in your configuration file. You can then configure class attributes like this:: @@ -48,8 +48,8 @@ sets):: .. versionadded:: 2.0 list, dict and set methods for config values -Example config file -``````````````````` +Example configuration file +`````````````````````````` :: @@ -79,15 +79,15 @@ Example config file ('la', 'ls -al') ] -Json Configuration files +JSON Configuration files ------------------------ -in case where executability of configuration can be problematic, or -configurations need to be modified programatically, IPython also support a -limited set of functionalities via ``.json`` config files. +In case where executability of configuration can be problematic, or +configurations need to be modified programmatically, IPython also support a +limited set of functionalities via ``.json`` configuration files. You can defined most of the configuration options via a json object which -hierarchy represent the value you woudl normally set on the ``c`` object of +hierarchy represent the value you would normally set on the ``c`` object of ``.py`` configuration files. The following ``ipython_config.json`` file:: { @@ -201,7 +201,7 @@ To see where IPython is looking for the IPython directory, use the command Systemwide configuration ======================== -It can be usefull to deploy systemwide ipython or ipykernel configuration +It can be useful to deploy systemwide ipython or ipykernel configuration when managing environment for many users. At startup time IPython and IPykernel will search for configuration file in multiple systemwide locations, mainly: @@ -219,7 +219,7 @@ In those locations, Terminal IPython will look for a file called ``ipython_kernel_config.py`` and ``ipython_kernel.json``. Configuration files are loaded in order and merged with configuration on -later locatoin taking precedence on earlier locations (that is to say a user +later location taking precedence on earlier locations (that is to say a user can overwrite a systemwide configuration option). You can see all locations in which IPython is looking for configuration files @@ -230,4 +230,4 @@ by starting ipython in debug mode:: Identically with ipykernel though the command is currently blocking until this process is killed with ``Ctrl-\``:: - $ python -m ipykernel --debug \ No newline at end of file + $ python -m ipykernel --debug From cd5bdd4968bf8fe0236aad1750d852cf41882c85 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Wed, 27 May 2020 16:29:29 -0700 Subject: [PATCH 30/57] Backport PR #12305: Fix ipython#11508: check if line_buffer is None --- IPython/core/completer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/IPython/core/completer.py b/IPython/core/completer.py index 32144e1e9b6..bc114f0f66b 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -1982,8 +1982,8 @@ def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None, # if text is either None or an empty string, rely on the line buffer if (not line_buffer) and full_text: line_buffer = full_text.split('\n')[cursor_line] - if not text: - text = self.splitter.split_line(line_buffer, cursor_pos) + if not text: # issue #11508: check line_buffer before calling split_line + text = self.splitter.split_line(line_buffer, cursor_pos) if line_buffer else '' if self.backslash_combining_completions: # allow deactivation of these on windows. From d7d455601d9579e0d7addd655e6c5382fc4e5ec2 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 29 May 2020 08:00:40 -0700 Subject: [PATCH 31/57] Backport PR #12342: Fixed bug on windows where relative image path has wrong slashes --- IPython/sphinxext/ipython_directive.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/IPython/sphinxext/ipython_directive.py b/IPython/sphinxext/ipython_directive.py index 0481624c42c..3d5b3768230 100644 --- a/IPython/sphinxext/ipython_directive.py +++ b/IPython/sphinxext/ipython_directive.py @@ -419,8 +419,8 @@ def process_image(self, decorator): # insert relative path to image file in source # as absolute path for Sphinx # sphinx expects a posix path, even on Windows - posix_path = pathlib.Path(savefig_dir,filename).as_posix() - outfile = '/' + os.path.relpath(posix_path, source_dir) + path = pathlib.Path(savefig_dir, filename) + outfile = '/' + path.relative_to(source_dir).as_posix() imagerows = ['.. image:: %s' % outfile] From 6cf72b2a0397204b9b93d24de1724e32c3ccbafe Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 29 May 2020 12:51:25 -0700 Subject: [PATCH 32/57] Backport PR #12347: What's new in 7.15+ Reproducible build working --- docs/source/coredev/index.rst | 17 +++++++ docs/source/whatsnew/version7.rst | 84 +++++++++++++++++++++++++++++++ setup.py | 2 +- tools/build_release | 8 +-- tools/make_tarball.py | 1 - tools/release | 5 +- tools/release_helper.sh | 37 +++++++++++--- tools/retar.py | 60 ++++++++++++++++++++++ 8 files changed, 194 insertions(+), 20 deletions(-) create mode 100644 tools/retar.py diff --git a/docs/source/coredev/index.rst b/docs/source/coredev/index.rst index 60f1cb0fdfd..ee1eadb9b1e 100644 --- a/docs/source/coredev/index.rst +++ b/docs/source/coredev/index.rst @@ -80,6 +80,13 @@ for the release you are actually making:: VERSION=5.0.0 BRANCH=master +For `reproducibility of builds `_, +we recommend setting ``SOURCE_DATE_EPOCH`` prior to running the build; record the used value +of ``SOURCE_DATE_EPOCH`` as it may not be available from build artifact. You +should be able to use ``date +%s`` to get a formatted timestamp:: + + SOURCE_DATE_EPOCH=$(date +%s) + 2. Create GitHub stats and finish release note ---------------------------------------------- @@ -229,6 +236,16 @@ uploading them to PyPI. We do not use an universal wheel as each wheel installs an ``ipython2`` or ``ipython3`` script, depending on the version of Python it is built for. Using an universal wheel would prevent this. +Check the shasum of files with:: + + shasum -a 256 dist/* + +and takes notes of them you might need them to update the conda-forge recipes. +Rerun the command and check the hash have not changed:: + + ./tools/release + shasum -a 256 dist/* + Use the following to actually upload the result of the build:: ./tools/release upload diff --git a/docs/source/whatsnew/version7.rst b/docs/source/whatsnew/version7.rst index f68c8101404..c35a4651904 100644 --- a/docs/source/whatsnew/version7.rst +++ b/docs/source/whatsnew/version7.rst @@ -2,6 +2,90 @@ 7.x Series ============ +.. _version 715: + +IPython 7.15 +============ + +IPython 7.15 brings a number of bug fixes and user facing improvements. + +Misc Noticeable changes: +------------------------ + + - Long completion name have better elision in terminal :ghpull:`12284` + - I've started to test on Python 3.9 :ghpull:`12307` and fix some errors. + - Hi DPI scaling of figures when using qt eventloop :ghpull:`12314` + - Document the ability to have systemwide configuration for IPython. + :ghpull:`12328` + - Fix issues with input autoformatting :ghpull:`12336` + +Reproducible Build +------------------ + +Starting with IPython 7.15, I am attempting to provide reproducible builds, +that is to say you should be able from the source tree to generate an sdist +and wheel that are identical byte for byte with the publish version on PyPI. + +I've only tested on a couple of machines so far and the process is relatively +straightforward, so this mean that IPython not only have a deterministic build +process, but also I have either removed, or put under control all effects of +the build environments on the final artifact. I encourage you to attempt the +build process on your machine as documented in :ref:`core_developer_guide` +and let me know if you do not obtain an identical artifact. + +While reproducible builds is critical to check that the supply chain of (open +source) software has not been compromised, it can also help to speedup many +of the build processes in large environment (conda, apt...) by allowing +better caching of intermediate build steps. + +Learn more on ``_. `Reflections on trusting +trust `_ is also one of the +cornerstone and recommended reads on this subject. + +.. note:: + + The build commit from which the sdist is generated is also `signed + `_, so you should be able to + check it has not been compromised, and the git repository is a `merkle-tree + `_, you can check the consistency + with `git-fsck `_ which you likely `want + to enable by default + `_. + +NEP29: Last version to support Python 3.6 +----------------------------------------- + +IPython 7.15 will be the Last IPython version to officially support Python +3.6, as stated by `NumPy Enhancement Proposal 29 +`_. Starting with +next minor version of IPython I may stop testing on Python 3.6 and may stop +publishing release artifacts that install on Python 3.6 + +Highlighted features +-------------------- + +Highlighted features are not new, but seem to not be widely known, this +section will help you discover in more narrative form what you can do with +IPython. + +Increase Tab Completion Menu Height +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In terminal IPython it is possible to increase the hight of the tab-completion +menu. To do so set the value of +:configtrait:`TerminalInteractiveShell.space_for_menu`, this will reserve more +space at the bottom of the screen for various kind of menus in IPython including +tab completion and searching in history. + +Autoformat Code in the terminal +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you have a preferred code formatter, you can configure IPython to +reformat your code. Set the value of +:configtrait:`TerminalInteractiveShell.autoformatter` to for example ``'black'`` +and IPython will auto format your code when possible. + + .. _version 714: IPython 7.14 diff --git a/setup.py b/setup.py index 593e3a6a0af..daf5f55f000 100755 --- a/setup.py +++ b/setup.py @@ -225,7 +225,7 @@ for key, deps in extras_require.items(): if ':' not in key: everything.update(deps) -extras_require['all'] = everything +extras_require['all'] = list(sorted(everything)) if 'setuptools' in sys.modules: setuptools_extra_args['python_requires'] = '>=3.6' diff --git a/tools/build_release b/tools/build_release index 26dc9ec874a..51fd87d54d9 100755 --- a/tools/build_release +++ b/tools/build_release @@ -2,6 +2,7 @@ """IPython release build script. """ import os +import sys from shutil import rmtree from toollib import sh, pjoin, get_ipdir, cd, sdists, buildwheels @@ -12,15 +13,10 @@ def build_release(): ipdir = get_ipdir() cd(ipdir) - # Cleanup - for d in ['build', 'dist', pjoin('docs', 'build'), pjoin('docs', 'dist'), - pjoin('docs', 'source', 'api', 'generated')]: - if os.path.isdir(d): - rmtree(d) - # Build source and binary distros sh(sdists) buildwheels() + sh(' '.join([sys.executable, 'tools/retar.py', 'dist/*.gz'])) if __name__ == '__main__': build_release() diff --git a/tools/make_tarball.py b/tools/make_tarball.py index bdce25ba804..fb639f61f61 100755 --- a/tools/make_tarball.py +++ b/tools/make_tarball.py @@ -3,7 +3,6 @@ """ import subprocess -import os from toollib import cd, sh diff --git a/tools/release b/tools/release index 5c8686b3145..2de8e120070 100755 --- a/tools/release +++ b/tools/release @@ -81,13 +81,10 @@ else: sh('mv ipython-*.tgz %s' % ipbackupdir) # Build release files - sh('./build_release %s' % ipdir) + sh('./build_release') cd(ipdir) - # Upload all files - sh(sdists) - buildwheels() print("`./release upload` to upload source distribution on PyPI and ipython archive") sys.exit(0) diff --git a/tools/release_helper.sh b/tools/release_helper.sh index 7489c16d756..6e2e8bf201a 100644 --- a/tools/release_helper.sh +++ b/tools/release_helper.sh @@ -98,12 +98,6 @@ then fi -echo -echo $BLUE"Attempting to build package..."$NOR - -tools/build_release -rm dist/* - if ask_section "Should we commit, tag, push... etc ? " then echo @@ -160,14 +154,41 @@ fi if ask_section "Should we build and release ?" then + + echo $BLUE"going to set SOURCE_DATE_EPOCH"$NOR + echo $BLUE'export SOURCE_DATE_EPOCH=$(git show -s --format=%ct HEAD)'$NOR + echo $GREEN"Press enter to continue"$NOR + read + + export SOURCE_DATE_EPOCH=$(git show -s --format=%ct HEAD) + + echo $BLUE"SOURCE_DATE_EPOCH set to $SOURCE_DATE_EPOCH"$NOR + echo $GREEN"Press enter to continue"$NOR + read + + + + echo + echo $BLUE"Attempting to build package..."$NOR + + tools/release + + + echo $RED'$ shasum -a 256 dist/*' + shasum -a 256 dist/* + echo $NOR + + echo $BLUE"We are going to rebuild, node the hash above, and compare them to the rebuild"$NOR + echo $GREEN"Press enter to continue"$NOR + read echo echo $BLUE"Attempting to build package..."$NOR tools/release - echo $RED - echo '$ shasum -a 256 dist/*' + echo $RED"Check the shasum for SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH" + echo $RED'$ shasum -a 256 dist/*' shasum -a 256 dist/* echo $NOR diff --git a/tools/retar.py b/tools/retar.py new file mode 100644 index 00000000000..dbdca0ed7f5 --- /dev/null +++ b/tools/retar.py @@ -0,0 +1,60 @@ +""" +Un-targz and retargz a targz file to ensure reproducible build. + +usage: + + $ export SOURCE_DATE_EPOCH=$(date +%s) + ... + $ python retar.py + +The process of creating an sdist can be non-reproducible: + - directory created during the process get a mtime of the creation date; + - gziping files embed the timestamp of fo zip creation. + +This will untar-retar; ensuring that all mtime > SOURCE_DATE_EPOCH will be set +equal to SOURCE_DATE_EPOCH. + +""" + +import tarfile +import sys +import os +import gzip +import io + +if len(sys.argv) > 2: + raise ValueError("Too many arguments") + + +timestamp = int(os.environ["SOURCE_DATE_EPOCH"]) + +old_buf = io.BytesIO() +with open(sys.argv[1], "rb") as f: + old_buf.write(f.read()) +old_buf.seek(0) +old = tarfile.open(fileobj=old_buf, mode="r:gz") + +buf = io.BytesIO() +new = tarfile.open(fileobj=buf, mode="w", format=tarfile.GNU_FORMAT) +for i, m in enumerate(old): + data = None + # mutation does not work, copy + if m.name.endswith('.DS_Store'): + continue + m2 = tarfile.TarInfo(m.name) + m2.mtime = min(timestamp, m.mtime) + m2.size = m.size + m2.type = m.type + m2.linkname = m.linkname + if m.isdir(): + data = old.extractfile(m) + new.addfile(m2, data) + else: + new.addfile(m2) +new.close() +old.close() + +buf.seek(0) +with open(sys.argv[1], "wb") as f: + with gzip.GzipFile('', "wb", fileobj=f, mtime=timestamp) as gzf: + gzf.write(buf.read()) From c19480ca30e1360d09572c1b595f66c5a13ce7de Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 29 May 2020 15:26:05 -0700 Subject: [PATCH 33/57] Backport PR #12352: Update with forgotten 7.14 release notes. --- docs/source/whatsnew/development.rst | 1 + .../source/whatsnew/pr/interruptible-debugger.rst | 4 ---- .../whatsnew/pr/video-display-attributes.rst | 4 ---- docs/source/whatsnew/version7.rst | 15 +++++++++++++++ 4 files changed, 16 insertions(+), 8 deletions(-) delete mode 100644 docs/source/whatsnew/pr/interruptible-debugger.rst delete mode 100644 docs/source/whatsnew/pr/video-display-attributes.rst diff --git a/docs/source/whatsnew/development.rst b/docs/source/whatsnew/development.rst index 99821d1105c..f2bde112f66 100644 --- a/docs/source/whatsnew/development.rst +++ b/docs/source/whatsnew/development.rst @@ -22,6 +22,7 @@ Need to be updated: pr/* + .. DO NOT EDIT THIS LINE BEFORE RELEASE. FEATURE INSERTION POINT. As a reminder, IPython master has diverged from the 7.x branch, thus master may diff --git a/docs/source/whatsnew/pr/interruptible-debugger.rst b/docs/source/whatsnew/pr/interruptible-debugger.rst deleted file mode 100644 index f501b3476b8..00000000000 --- a/docs/source/whatsnew/pr/interruptible-debugger.rst +++ /dev/null @@ -1,4 +0,0 @@ -IPython.core.debugger.Pdb is now interruptible -============================================== - -A ``KeyboardInterrupt`` will now interrupt IPython's extended debugger, in order to make Jupyter able to interrupt it. diff --git a/docs/source/whatsnew/pr/video-display-attributes.rst b/docs/source/whatsnew/pr/video-display-attributes.rst deleted file mode 100644 index 1e7434e895d..00000000000 --- a/docs/source/whatsnew/pr/video-display-attributes.rst +++ /dev/null @@ -1,4 +0,0 @@ -Video HTML attributes -===================== - -Add an option to `IPython.display.Video` to change the attributes of the HTML display of the video. diff --git a/docs/source/whatsnew/version7.rst b/docs/source/whatsnew/version7.rst index c35a4651904..3d9abc96f6f 100644 --- a/docs/source/whatsnew/version7.rst +++ b/docs/source/whatsnew/version7.rst @@ -18,6 +18,10 @@ Misc Noticeable changes: - Document the ability to have systemwide configuration for IPython. :ghpull:`12328` - Fix issues with input autoformatting :ghpull:`12336` + - ``IPython.core.debugger.Pdb`` is now interruptible (:ghpull:`12168`, in 7.14 + but forgotten in release notes) + - Video HTML attributes (:ghpull:`12212`, in 7.14 but forgotten in release + notes) Reproducible Build ------------------ @@ -109,6 +113,17 @@ Misc Changes - add extra options to the ``Video`` Rich objects :ghpull:`12212` - add pretty-printing to ``SimpleNamespace`` :ghpull:`12230` +IPython.core.debugger.Pdb is now interruptible +---------------------------------------------- + +A ``KeyboardInterrupt`` will now interrupt IPython's extended debugger, in order to make Jupyter able to interrupt it. (:ghpull:`12168`) + +Video HTML attributes +--------------------- + +Add an option to `IPython.display.Video` to change the attributes of the HTML display of the video (:ghpull:`12212`) + + Pending deprecated imports -------------------------- From 6d868182949af86df9c042a995ae31b2cc368ef4 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 29 May 2020 15:39:32 -0700 Subject: [PATCH 34/57] Merge pull request #12354 from Carreau/docstats --- docs/source/whatsnew/github-stats-7.rst | 42 +++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/docs/source/whatsnew/github-stats-7.rst b/docs/source/whatsnew/github-stats-7.rst index 2c0c5d81cf3..d326771e8de 100644 --- a/docs/source/whatsnew/github-stats-7.rst +++ b/docs/source/whatsnew/github-stats-7.rst @@ -1,6 +1,25 @@ Issues closed in the 7.x development cycle ========================================== +Issues closed in 7.15 +--------------------- + +GitHub stats for 2020/05/01 - 2020/05/29 (tag: 7.14.0) + +These lists are automatically generated, and may be incomplete or contain duplicates. + +We closed 1 issues and merged 29 pull requests. +The full list can be seen `on GitHub `__ + +The following 6 authors contributed 31 commits. + +* Blake Griffin +* Inception95 +* Marcio Mazza +* Matthias Bussonnier +* Talley Lambert +* Thomas + Issues closed in 7.14 --------------------- @@ -27,6 +46,29 @@ The following 10 authors contributed 47 commits. Issues closed in 7.13 --------------------- +GitHub stats for 2020/02/29 - 2020/05/01 (tag: 7.13.0) + +These lists are automatically generated, and may be incomplete or contain duplicates. + +We closed 0 issues and merged 30 pull requests. +The full list can be seen `on GitHub `__ + +The following 10 authors contributed 47 commits. + +* Eric Wieser +* foobarbyte +* Ian Castleden +* Itamar Turner-Trauring +* Lumir Balhar +* Markus Wageringel +* Matthias Bussonnier +* Matthieu Ancellin +* Quentin Peter +* Theo Ouzhinski + +Issues closed in 7.13 +--------------------- + GitHub stats for 2020/02/01 - 2020/02/28 (tag: 7.12.0) These lists are automatically generated, and may be incomplete or contain duplicates. From 2922831acf6d7c185fb863b4436ffd7cfa665675 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 29 May 2020 15:46:12 -0700 Subject: [PATCH 35/57] release 7.15.0 --- IPython/core/release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index 1bc5595cac9..aae948bbb1b 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -24,7 +24,7 @@ _version_patch = 0 _version_extra = '.dev' # _version_extra = 'b1' -# _version_extra = '' # Uncomment this for full releases +_version_extra = '' # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] From 368973ccabcaf67824ea557e7428a871a5c62a5b Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 29 May 2020 15:47:19 -0700 Subject: [PATCH 36/57] back to dev --- IPython/core/release.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index aae948bbb1b..76bed55d664 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -20,11 +20,11 @@ # release. 'dev' as a _version_extra string means this is a development # version _version_major = 7 -_version_minor = 15 +_version_minor = 16 _version_patch = 0 _version_extra = '.dev' # _version_extra = 'b1' -_version_extra = '' # Uncomment this for full releases +# _version_extra = '' # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] From 2874a21a1d2efca947ca1721cb39e5adcc256b09 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sun, 31 May 2020 15:57:22 -0700 Subject: [PATCH 37/57] Backport PR #12301: Exceptions raised when executing notebooks via the %run magic command --- IPython/core/magics/execution.py | 19 +++++++++++-------- IPython/core/tests/test_run.py | 19 +++++++++++++++++++ 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/IPython/core/magics/execution.py b/IPython/core/magics/execution.py index 438d0b5f0ad..dc6cdf00e29 100644 --- a/IPython/core/magics/execution.py +++ b/IPython/core/magics/execution.py @@ -688,17 +688,16 @@ def run(self, parameter_s='', runner=None, modulename = opts["m"][0] modpath = find_mod(modulename) if modpath is None: - warn('%r is not a valid modulename on sys.path'%modulename) - return + msg = '%r is not a valid modulename on sys.path'%modulename + raise Exception(msg) arg_lst = [modpath] + arg_lst try: fpath = None # initialize to make sure fpath is in scope later fpath = arg_lst[0] filename = file_finder(fpath) except IndexError: - warn('you must provide at least a filename.') - print('\n%run:\n', oinspect.getdoc(self.run)) - return + msg = 'you must provide at least a filename.' + raise Exception(msg) except IOError as e: try: msg = str(e) @@ -706,13 +705,17 @@ def run(self, parameter_s='', runner=None, msg = e.message if os.name == 'nt' and re.match(r"^'.*'$",fpath): warn('For Windows, use double quotes to wrap a filename: %run "mypath\\myfile.py"') - error(msg) - return + raise Exception(msg) + except TypeError: + if fpath in sys.meta_path: + filename = "" + else: + raise if filename.lower().endswith(('.ipy', '.ipynb')): with preserve_keys(self.shell.user_ns, '__file__'): self.shell.user_ns['__file__'] = filename - self.shell.safe_execfile_ipy(filename) + self.shell.safe_execfile_ipy(filename, raise_exceptions=True) return # Control the response to exit() calls made by the script being run diff --git a/IPython/core/tests/test_run.py b/IPython/core/tests/test_run.py index 38d71b31740..eff832b3fc0 100644 --- a/IPython/core/tests/test_run.py +++ b/IPython/core/tests/test_run.py @@ -402,6 +402,25 @@ def test_run_nb(self): nt.assert_equal(_ip.user_ns['answer'], 42) + def test_run_nb_error(self): + """Test %run notebook.ipynb error""" + from nbformat import v4, writes + # %run when a file name isn't provided + nt.assert_raises(Exception, _ip.magic, "run") + + # %run when a file doesn't exist + nt.assert_raises(Exception, _ip.magic, "run foobar.ipynb") + + # %run on a notebook with an error + nb = v4.new_notebook( + cells=[ + v4.new_code_cell("0/0") + ] + ) + src = writes(nb, version=4) + self.mktmp(src, ext='.ipynb') + nt.assert_raises(Exception, _ip.magic, "run %s" % self.fname) + def test_file_options(self): src = ('import sys\n' 'a = " ".join(sys.argv[1:])\n') From 31133c3a5766f23a8b1c067319dc4d5c0c182355 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 2 Jun 2020 13:08:08 -0700 Subject: [PATCH 38/57] Backport PR #12360: use $SHELL in system_piped --- IPython/utils/_process_posix.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/IPython/utils/_process_posix.py b/IPython/utils/_process_posix.py index f3f93f774d6..a11cad7697c 100644 --- a/IPython/utils/_process_posix.py +++ b/IPython/utils/_process_posix.py @@ -59,11 +59,12 @@ class ProcessHandler(object): @property def sh(self): - if self._sh is None: - self._sh = pexpect.which('sh') + if self._sh is None: + shell_name = os.environ.get("SHELL", "sh") + self._sh = pexpect.which(shell_name) if self._sh is None: - raise OSError('"sh" shell not found') - + raise OSError('"{}" shell not found'.format(shell_name)) + return self._sh def __init__(self, logfile=None, read_timeout=None, terminate_timeout=None): From a7ca2e7881463080512976750d4c7f514745447d Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 2 Jun 2020 16:38:55 -0700 Subject: [PATCH 39/57] Backport PR #12358: Fix retaring process for twine. --- tools/retar.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/retar.py b/tools/retar.py index dbdca0ed7f5..9b5e73c6baa 100644 --- a/tools/retar.py +++ b/tools/retar.py @@ -47,10 +47,10 @@ m2.type = m.type m2.linkname = m.linkname if m.isdir(): + new.addfile(m2) + else: data = old.extractfile(m) new.addfile(m2, data) - else: - new.addfile(m2) new.close() old.close() @@ -58,3 +58,7 @@ with open(sys.argv[1], "wb") as f: with gzip.GzipFile('', "wb", fileobj=f, mtime=timestamp) as gzf: gzf.write(buf.read()) + +# checks the archive is valid. +archive = tarfile.open(sys.argv[1]) +names = archive.getnames() From ee4be875ffae4f1b0c71182eafa89976d0127b5e Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Wed, 3 Jun 2020 16:20:50 -0700 Subject: [PATCH 40/57] Backport PR #12339: Add instruction to list the API changes. --- tools/release_helper.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tools/release_helper.sh b/tools/release_helper.sh index 6e2e8bf201a..dd700b56329 100644 --- a/tools/release_helper.sh +++ b/tools/release_helper.sh @@ -81,6 +81,21 @@ then fi +if ask_section "Generate API difference (using frapuccino)" +then + echo $BLUE"Checking out $PREV_RELEASE"$NOR + git checkout $PREV_RELEASE + echo $BLUE"Saving API to file $PREV_RELEASE"$NOR + frappuccino IPython --save IPython-$PREV_RELEASE.json + echo $BLUE"comming back to $BRANCH"$NOR + git checkout $BRANCH + echo $BLUE"comparing ..."$NOR + frappuccino IPython --compare IPython-$PREV_RELEASE.json + echo $GREEN"Use the above guideline to write an API changelog ..."$NOR + echo $GREEN"Press any keys to continue"$NOR + read +fi + echo "Cleaning repository" git clean -xfdi From e769fcbd50ef648251128985232d4b04a86e839b Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 5 Jun 2020 10:31:16 -0700 Subject: [PATCH 41/57] Backport PR #12367: PR: Add handling for malformed pathext env var (Windows) --- IPython/core/magics/osm.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/IPython/core/magics/osm.py b/IPython/core/magics/osm.py index ddc9b6f5874..90da7e22803 100644 --- a/IPython/core/magics/osm.py +++ b/IPython/core/magics/osm.py @@ -25,6 +25,7 @@ from IPython.utils.process import abbrev_cwd from IPython.utils.terminal import set_term_title from traitlets import Bool +from warnings import warn @magics_class @@ -48,8 +49,15 @@ def __init__(self, shell=None, **kwargs): winext = os.environ['pathext'].replace(';','|').replace('.','') except KeyError: winext = 'exe|com|bat|py' - - self.execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE) + try: + self.execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE) + except re.error: + warn("Seems like your pathext environmental " + "variable is malformed. Please check it to " + "enable a proper handle of file extensions " + "managed for your system") + winext = 'exe|com|bat|py' + self.execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE) # call up the chain super().__init__(shell=shell, **kwargs) From 1d622dd149295be58bb9c8098cde8a8e9943b5e4 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 5 Jun 2020 11:16:27 -0700 Subject: [PATCH 42/57] Backport PR #12355: Fix GUI inputhook for qt 5.15.0 --- IPython/terminal/pt_inputhooks/qt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/terminal/pt_inputhooks/qt.py b/IPython/terminal/pt_inputhooks/qt.py index 1504c3bde72..9f5e1b41dcd 100644 --- a/IPython/terminal/pt_inputhooks/qt.py +++ b/IPython/terminal/pt_inputhooks/qt.py @@ -45,7 +45,7 @@ def inputhook(context): QtCore.QSocketNotifier.Read) try: # connect the callback we care about before we turn it on - notifier.activated.connect(event_loop.exit) + notifier.activated.connect(lambda: event_loop.exit()) notifier.setEnabled(True) # only start the event loop we are not already flipped if not context.input_is_ready(): From 4a742600e67590652dcf630425159f7fd8bd0182 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 8 Jun 2020 11:04:19 -0700 Subject: [PATCH 43/57] Backport PR #12375: BF: wx inputhook --- IPython/terminal/pt_inputhooks/wx.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/IPython/terminal/pt_inputhooks/wx.py b/IPython/terminal/pt_inputhooks/wx.py index 618f092b4f9..a0f4442c771 100644 --- a/IPython/terminal/pt_inputhooks/wx.py +++ b/IPython/terminal/pt_inputhooks/wx.py @@ -177,11 +177,13 @@ def inputhook_wxphoenix(context): # Use a wx.Timer to periodically check whether input is ready - as soon as # it is, we exit the main loop + timer = wx.Timer() + def poll(ev): if context.input_is_ready(): + timer.Stop() app.ExitMainLoop() - timer = wx.Timer() timer.Start(poll_interval) timer.Bind(wx.EVT_TIMER, poll) From 7238a928f1adc516b769028bd267fe00ef71a1d7 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 15 Jun 2020 15:18:00 -0700 Subject: [PATCH 44/57] Backport PR #12389: Minor: typos, grammar --- docs/source/interactive/autoawait.rst | 28 +++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/source/interactive/autoawait.rst b/docs/source/interactive/autoawait.rst index f87379d2ca0..e4ed965a13e 100644 --- a/docs/source/interactive/autoawait.rst +++ b/docs/source/interactive/autoawait.rst @@ -8,7 +8,7 @@ Asynchronous in REPL: Autoawait This feature is experimental and behavior can change between python and IPython version without prior deprecation. -Starting with IPython 7.0, and when user Python 3.6 and above, IPython offer the +Starting with IPython 7.0, and when using Python 3.6 and above, IPython offer the ability to run asynchronous code from the REPL. Constructs which are :exc:`SyntaxError` s in the Python REPL can be used seamlessly in IPython. @@ -19,7 +19,7 @@ will differ between IPython, IPykernel and their versions. When a supported library is used, IPython will automatically allow Futures and Coroutines in the REPL to be ``await`` ed. This will happen if an :ref:`await -` (or any other async constructs like async-with, async-for) is use at +` (or any other async constructs like async-with, async-for) is used at top level scope, or if any structure valid only in `async def `_ function context are present. For example, the following being a syntax error in the @@ -73,7 +73,7 @@ By default IPython will assume integration with Python's provided :mod:`asyncio`, but integration with other libraries is provided. In particular we provide experimental integration with the ``curio`` and ``trio`` library. -You can switch current integration by using the +You can switch the current integration by using the ``c.InteractiveShell.loop_runner`` option or the ``autoawait `` magic. @@ -118,7 +118,7 @@ to your code. When using command line IPython, the default loop (or runner) does not process in the background, so top level asynchronous code must finish for the REPL to -allow you to enter more code. As with usual Python semantic, the awaitables are +allow you to enter more code. As with usual Python semantics, the awaitables are started only when awaited for the first time. That is to say, in first example, no network request is done between ``In[1]`` and ``In[2]``. @@ -131,8 +131,8 @@ a loop to run. By default IPython will use a fake coroutine runner which should allow ``IPython.embed()`` to be nested. Though this will prevent usage of the :magic:`%autoawait` feature when using IPython embed. -You can set explicitly a coroutine runner for ``embed()`` if you desire to run -asynchronous code, the exact behavior is though undefined. +You can set a coroutine runner explicitly for ``embed()`` if you want to run +asynchronous code, though the exact behavior is undefined. Effects on Magics ----------------- @@ -140,14 +140,14 @@ Effects on Magics A couple of magics (``%%timeit``, ``%timeit``, ``%%time``, ``%%prun``) have not yet been updated to work with asynchronous code and will raise syntax errors when trying to use top-level ``await``. We welcome any contribution to help fix -those, and extra cases we haven't caught yet. We hope for better support in Cor +those, and extra cases we haven't caught yet. We hope for better support in Core Python for top-level Async code. Internals --------- As running asynchronous code is not supported in interactive REPL (as of Python -3.7) we have to rely to a number of complex workaround and heuristic to allow +3.7) we have to rely to a number of complex workarounds and heuristics to allow this to happen. It is interesting to understand how this works in order to comprehend potential bugs, or provide a custom runner. @@ -179,16 +179,16 @@ significant overhead to this kind of code. By default the generated coroutine function will be consumed by Asyncio's ``loop_runner = asyncio.get_evenloop().run_until_complete()`` method if ``async`` mode is deemed necessary, otherwise the coroutine will just be -exhausted in a simple runner. It is though possible to change the default +exhausted in a simple runner. It is possible, though, to change the default runner. A loop runner is a *synchronous* function responsible from running a coroutine object. -The runner is responsible from ensuring that ``coroutine`` run to completion, -and should return the result of executing the coroutine. Let's write a +The runner is responsible for ensuring that ``coroutine`` runs to completion, +and it should return the result of executing the coroutine. Let's write a runner for ``trio`` that print a message when used as an exercise, ``trio`` is -special as it usually prefer to run a function object and make a coroutine by +special as it usually prefers to run a function object and make a coroutine by itself, we can get around this limitation by wrapping it in an async-def without parameters and passing this value to ``trio``:: @@ -246,8 +246,8 @@ Difference between terminal IPython and IPykernel The exact asynchronous code running behavior varies between Terminal IPython and IPykernel. The root cause of this behavior is due to IPykernel having a *persistent* `asyncio` loop running, while Terminal IPython starts and stops a -loop for each code block. This can lead to surprising behavior in some case if -you are used to manipulate asyncio loop yourself, see for example +loop for each code block. This can lead to surprising behavior in some cases if +you are used to manipulating asyncio loop yourself, see for example :ghissue:`11303` for a longer discussion but here are some of the astonishing cases. From ad4453ed63be4914c1abbd2e90db3a5d68f765f4 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 16 Jun 2020 14:48:50 -0700 Subject: [PATCH 45/57] Backport PR #12359: Implement understanding on __tracebackhide__ --- IPython/core/debugger.py | 155 ++++++++++++++++-- IPython/core/interactiveshell.py | 16 +- IPython/core/magics/basic.py | 14 +- IPython/core/tests/test_debugger.py | 78 ++++++++- IPython/core/ultratb.py | 27 ++- IPython/terminal/debugger.py | 10 ++ appveyor.yml | 7 - .../source/whatsnew/pr/skip_tracebackhide.rst | 11 ++ 8 files changed, 290 insertions(+), 28 deletions(-) create mode 100644 docs/source/whatsnew/pr/skip_tracebackhide.rst diff --git a/IPython/core/debugger.py b/IPython/core/debugger.py index 89eae383dd2..a330baa450e 100644 --- a/IPython/core/debugger.py +++ b/IPython/core/debugger.py @@ -280,26 +280,31 @@ def __init__(self, color_scheme=None, completekey=None, # Set the prompt - the default prompt is '(Pdb)' self.prompt = prompt + self.skip_hidden = True def set_colors(self, scheme): """Shorthand access to the color table scheme selector method.""" self.color_scheme_table.set_active_scheme(scheme) self.parser.style = scheme + + def hidden_frames(self, stack): + """ + Given an index in the stack return wether it should be skipped. + + This is used in up/down and where to skip frames. + """ + ip_hide = [s[0].f_locals.get("__tracebackhide__", False) for s in stack] + ip_start = [i for i, s in enumerate(ip_hide) if s == "__ipython_bottom__"] + if ip_start: + ip_hide = [h if i > ip_start[0] else True for (i, h) in enumerate(ip_hide)] + return ip_hide + def interaction(self, frame, traceback): try: OldPdb.interaction(self, frame, traceback) except KeyboardInterrupt: - self.stdout.write('\n' + self.shell.get_exception_only()) - - def new_do_up(self, arg): - OldPdb.do_up(self, arg) - do_u = do_up = decorate_fn_with_doc(new_do_up, OldPdb.do_up) - - def new_do_down(self, arg): - OldPdb.do_down(self, arg) - - do_d = do_down = decorate_fn_with_doc(new_do_down, OldPdb.do_down) + self.stdout.write("\n" + self.shell.get_exception_only()) def new_do_frame(self, arg): OldPdb.do_frame(self, arg) @@ -320,6 +325,8 @@ def new_do_restart(self, arg): return self.do_quit(arg) def print_stack_trace(self, context=None): + Colors = self.color_scheme_table.active_colors + ColorsNormal = Colors.Normal if context is None: context = self.context try: @@ -329,8 +336,21 @@ def print_stack_trace(self, context=None): except (TypeError, ValueError): raise ValueError("Context must be a positive integer") try: - for frame_lineno in self.stack: + skipped = 0 + for hidden, frame_lineno in zip(self.hidden_frames(self.stack), self.stack): + if hidden and self.skip_hidden: + skipped += 1 + continue + if skipped: + print( + f"{Colors.excName} [... skipping {skipped} hidden frame(s)]{ColorsNormal}\n" + ) + skipped = 0 self.print_stack_entry(frame_lineno, context=context) + if skipped: + print( + f"{Colors.excName} [... skipping {skipped} hidden frame(s)]{ColorsNormal}\n" + ) except KeyboardInterrupt: pass @@ -487,6 +507,16 @@ def print_list_lines(self, filename, first, last): except KeyboardInterrupt: pass + def do_skip_hidden(self, arg): + """ + Change whether or not we should skip frames with the + __tracebackhide__ attribute. + """ + if arg.strip().lower() in ("true", "yes"): + self.skip_hidden = True + elif arg.strip().lower() in ("false", "no"): + self.skip_hidden = False + def do_list(self, arg): """Print lines of code from the current stack frame """ @@ -633,6 +663,109 @@ def do_where(self, arg): do_w = do_where + def stop_here(self, frame): + hidden = False + if self.skip_hidden: + hidden = frame.f_locals.get("__tracebackhide__", False) + if hidden: + Colors = self.color_scheme_table.active_colors + ColorsNormal = Colors.Normal + print(f"{Colors.excName} [... skipped 1 hidden frame]{ColorsNormal}\n") + + return super().stop_here(frame) + + def do_up(self, arg): + """u(p) [count] + Move the current frame count (default one) levels up in the + stack trace (to an older frame). + + Will skip hidden frames. + """ + ## modified version of upstream that skips + # frames with __tracebackide__ + if self.curindex == 0: + self.error("Oldest frame") + return + try: + count = int(arg or 1) + except ValueError: + self.error("Invalid frame count (%s)" % arg) + return + skipped = 0 + if count < 0: + _newframe = 0 + else: + _newindex = self.curindex + counter = 0 + hidden_frames = self.hidden_frames(self.stack) + for i in range(self.curindex - 1, -1, -1): + frame = self.stack[i][0] + if hidden_frames[i] and self.skip_hidden: + skipped += 1 + continue + counter += 1 + if counter >= count: + break + else: + # if no break occured. + self.error("all frames above hidden") + return + + Colors = self.color_scheme_table.active_colors + ColorsNormal = Colors.Normal + _newframe = i + self._select_frame(_newframe) + if skipped: + print( + f"{Colors.excName} [... skipped {skipped} hidden frame(s)]{ColorsNormal}\n" + ) + + def do_down(self, arg): + """d(own) [count] + Move the current frame count (default one) levels down in the + stack trace (to a newer frame). + + Will skip hidden frames. + """ + if self.curindex + 1 == len(self.stack): + self.error("Newest frame") + return + try: + count = int(arg or 1) + except ValueError: + self.error("Invalid frame count (%s)" % arg) + return + if count < 0: + _newframe = len(self.stack) - 1 + else: + _newindex = self.curindex + counter = 0 + skipped = 0 + hidden_frames = self.hidden_frames(self.stack) + for i in range(self.curindex + 1, len(self.stack)): + frame = self.stack[i][0] + if hidden_frames[i] and self.skip_hidden: + skipped += 1 + continue + counter += 1 + if counter >= count: + break + else: + self.error("all frames bellow hidden") + return + + Colors = self.color_scheme_table.active_colors + ColorsNormal = Colors.Normal + if skipped: + print( + f"{Colors.excName} [... skipped {skipped} hidden frame(s)]{ColorsNormal}\n" + ) + _newframe = i + + self._select_frame(_newframe) + + do_d = do_down + do_u = do_up class InterruptiblePdb(Pdb): """Version of debugger where KeyboardInterrupt exits the debugger altogether.""" diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index f5fdba6692b..ddb1b64ea78 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -2211,11 +2211,20 @@ def complete(self, text, line=None, cursor_pos=None): with self.builtin_trap: return self.Completer.complete(text, line, cursor_pos) - def set_custom_completer(self, completer, pos=0): + def set_custom_completer(self, completer, pos=0) -> None: """Adds a new custom completer function. The position argument (defaults to 0) is the index in the completers - list where you want the completer to be inserted.""" + list where you want the completer to be inserted. + + `completer` should have the following signature:: + + def completion(self: Completer, text: string) -> List[str]: + raise NotImplementedError + + It will be bound to the current Completer instance and pass some text + and return a list with current completions to suggest to the user. + """ newcomp = types.MethodType(completer, self.Completer) self.Completer.custom_matchers.insert(pos,newcomp) @@ -3310,6 +3319,9 @@ async def run_code(self, code_obj, result=None, *, async_=False): False : successful execution. True : an error occurred. """ + # special value to say that anything above is IPython and should be + # hidden. + __tracebackhide__ = "__ipython_bottom__" # Set our own excepthook in case the user code tries to call it # directly, so that the IPython crash handler doesn't get triggered old_excepthook, sys.excepthook = sys.excepthook, self.excepthook diff --git a/IPython/core/magics/basic.py b/IPython/core/magics/basic.py index 5c8649b5408..a8feb755386 100644 --- a/IPython/core/magics/basic.py +++ b/IPython/core/magics/basic.py @@ -364,13 +364,25 @@ def xmode(self, parameter_s=''): Valid modes: Plain, Context, Verbose, and Minimal. - If called without arguments, acts as a toggle.""" + If called without arguments, acts as a toggle. + + When in verbose mode the value --show (and --hide) + will respectively show (or hide) frames with ``__tracebackhide__ = + True`` value set. + """ def xmode_switch_err(name): warn('Error changing %s exception modes.\n%s' % (name,sys.exc_info()[1])) shell = self.shell + if parameter_s.strip() == "--show": + shell.InteractiveTB.skip_hidden = False + return + if parameter_s.strip() == "--hide": + shell.InteractiveTB.skip_hidden = True + return + new_mode = parameter_s.strip().capitalize() try: shell.InteractiveTB.set_mode(mode=new_mode) diff --git a/IPython/core/tests/test_debugger.py b/IPython/core/tests/test_debugger.py index 7f3720a5ab9..9fdc944e4d0 100644 --- a/IPython/core/tests/test_debugger.py +++ b/IPython/core/tests/test_debugger.py @@ -4,20 +4,24 @@ # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. +import bdb +import builtins +import os import signal +import subprocess import sys import time import warnings +from subprocess import PIPE, CalledProcessError, check_output from tempfile import NamedTemporaryFile -from subprocess import check_output, CalledProcessError, PIPE -import subprocess +from textwrap import dedent from unittest.mock import patch -import builtins -import bdb import nose.tools as nt from IPython.core import debugger +from IPython.testing import IPYTHON_TESTING_TIMEOUT_SCALE +from IPython.testing.decorators import skip_win32 #----------------------------------------------------------------------------- # Helper classes, from CPython's Pdb test suite @@ -254,3 +258,69 @@ def raising_input(msg="", called=[0]): # implementation would involve a subprocess, but that adds issues with # interrupting subprocesses that are rather complex, so it's simpler # just to do it this way. + +@skip_win32 +def test_xmode_skip(): + """that xmode skip frames + + Not as a doctest as pytest does not run doctests. + """ + import pexpect + env = os.environ.copy() + env["IPY_TEST_SIMPLE_PROMPT"] = "1" + + child = pexpect.spawn( + sys.executable, ["-m", "IPython", "--colors=nocolor"], env=env + ) + child.timeout = 15 * IPYTHON_TESTING_TIMEOUT_SCALE + + child.expect("IPython") + child.expect("\n") + child.expect_exact("In [1]") + + block = dedent( + """ +def f(): + __tracebackhide__ = True + g() + +def g(): + raise ValueError + +f() + """ + ) + + for line in block.splitlines(): + child.sendline(line) + child.expect_exact(line) + child.expect_exact("skipping") + + block = dedent( + """ +def f(): + __tracebackhide__ = True + g() + +def g(): + from IPython.core.debugger import set_trace + set_trace() + +f() + """ + ) + + for line in block.splitlines(): + child.sendline(line) + child.expect_exact(line) + + child.expect("ipdb>") + child.sendline("w") + child.expect("hidden") + child.expect("ipdb>") + child.sendline("skip_hidden false") + child.sendline("w") + child.expect("__traceba") + child.expect("ipdb>") + + child.close() diff --git a/IPython/core/ultratb.py b/IPython/core/ultratb.py index 9e7e8fb390d..45e22bd7b94 100644 --- a/IPython/core/ultratb.py +++ b/IPython/core/ultratb.py @@ -879,13 +879,36 @@ def __init__(self, color_scheme='Linux', call_pdb=False, ostream=None, self.check_cache = check_cache self.debugger_cls = debugger_cls or debugger.Pdb + self.skip_hidden = True def format_records(self, records, last_unique, recursion_repeat): """Format the stack frames of the traceback""" frames = [] + + skipped = 0 for r in records[:last_unique+recursion_repeat+1]: - #print '*** record:',file,lnum,func,lines,index # dbg + if self.skip_hidden: + if r[0].f_locals.get("__tracebackhide__", 0): + skipped += 1 + continue + if skipped: + Colors = self.Colors # just a shorthand + quicker name lookup + ColorsNormal = Colors.Normal # used a lot + frames.append( + " %s[... skipping hidden %s frame]%s\n" + % (Colors.excName, skipped, ColorsNormal) + ) + skipped = 0 + frames.append(self.format_record(*r)) + + if skipped: + Colors = self.Colors # just a shorthand + quicker name lookup + ColorsNormal = Colors.Normal # used a lot + frames.append( + " %s[... skipping hidden %s frame]%s\n" + % (Colors.excName, skipped, ColorsNormal) + ) if recursion_repeat: frames.append('... last %d frames repeated, from the frame below ...\n' % recursion_repeat) @@ -1123,8 +1146,6 @@ def format_exception_as_a_whole(self, etype, evalue, etb, number_of_lines_of_con head = self.prepare_header(etype, self.long_header) records = self.get_records(etb, number_of_lines_of_context, tb_offset) - if records is None: - return "" last_unique, recursion_repeat = find_recursion(orig_etype, evalue, records) diff --git a/IPython/terminal/debugger.py b/IPython/terminal/debugger.py index f3268e70bd6..ffd53c65e78 100644 --- a/IPython/terminal/debugger.py +++ b/IPython/terminal/debugger.py @@ -41,6 +41,16 @@ def get_prompt_tokens(): global_namespace={}, parent=self.shell, ) + # add a completer for all the do_ methods + methods_names = [m[3:] for m in dir(self) if m.startswith("do_")] + + def gen_comp(self, text): + return [m for m in methods_names if m.startswith(text)] + import types + newcomp = types.MethodType(gen_comp, compl) + compl.custom_matchers.insert(0, newcomp) + # end add completer. + self._ptcomp = IPythonPTCompleter(compl) options = dict( diff --git a/appveyor.yml b/appveyor.yml index b79c7f26ff4..d20effdf815 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -4,13 +4,6 @@ matrix: environment: matrix: - - PYTHON: "C:\\Python36" - PYTHON_VERSION: "3.6.x" - PYTHON_ARCH: "32" - - - PYTHON: "C:\\Python36-x64" - PYTHON_VERSION: "3.6.x" - PYTHON_ARCH: "64" - PYTHON: "C:\\Python37-x64" PYTHON_VERSION: "3.7.x" diff --git a/docs/source/whatsnew/pr/skip_tracebackhide.rst b/docs/source/whatsnew/pr/skip_tracebackhide.rst new file mode 100644 index 00000000000..0acfecf127d --- /dev/null +++ b/docs/source/whatsnew/pr/skip_tracebackhide.rst @@ -0,0 +1,11 @@ +The default tracebackmode will now skip frames that are marked with +``__tracebackhide__ = True`` and show how many traceback frames have been +skipped. This can be toggled by using :magic:`xmode` with the ``--show`` or +``--hide`` attribute. It will have no effect on non verbose traceback modes. + +The ipython debugger also now understand ``__tracebackhide__`` as well and will +skip hidden frames when displaying. Movement up and down the stack will skip the +hidden frames and will show how many frames were hidden. Internal IPython frames +are also now hidden by default. The behavior can be changed with the +``skip_hidden`` command and accepts "yes", "no", "true" and "false" case +insensitive parameters. From 737e62eb8f90c0412feaf53a5195c362d1ace462 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 26 Jun 2020 14:58:42 -0700 Subject: [PATCH 46/57] Backport PR #12404: Update what's new for 7.16 --- docs/source/whatsnew/development.rst | 1 + docs/source/whatsnew/github-stats-7.rst | 22 +++++++ .../source/whatsnew/pr/skip_tracebackhide.rst | 11 ---- docs/source/whatsnew/version7.rst | 61 +++++++++++++++++++ 4 files changed, 84 insertions(+), 11 deletions(-) delete mode 100644 docs/source/whatsnew/pr/skip_tracebackhide.rst diff --git a/docs/source/whatsnew/development.rst b/docs/source/whatsnew/development.rst index f2bde112f66..9e9b2984c6c 100644 --- a/docs/source/whatsnew/development.rst +++ b/docs/source/whatsnew/development.rst @@ -23,6 +23,7 @@ Need to be updated: pr/* + .. DO NOT EDIT THIS LINE BEFORE RELEASE. FEATURE INSERTION POINT. As a reminder, IPython master has diverged from the 7.x branch, thus master may diff --git a/docs/source/whatsnew/github-stats-7.rst b/docs/source/whatsnew/github-stats-7.rst index d326771e8de..6bc36c0789c 100644 --- a/docs/source/whatsnew/github-stats-7.rst +++ b/docs/source/whatsnew/github-stats-7.rst @@ -1,6 +1,28 @@ Issues closed in the 7.x development cycle ========================================== + +Issues closed in 7.16 +--------------------- + +GitHub stats for 2020/05/29 - 2020/06/26 (tag: 7.15.0) + +These lists are automatically generated, and may be incomplete or contain duplicates. + +We closed 0 issues and merged 18 pull requests. +The full list can be seen `on GitHub `__ + +The following 7 authors contributed 22 commits. + +* Benjamin Ragan-Kelley +* dalthviz +* Frank Tobia +* Matthias Bussonnier +* palewire +* Paul McCarthy +* Talley Lambert + + Issues closed in 7.15 --------------------- diff --git a/docs/source/whatsnew/pr/skip_tracebackhide.rst b/docs/source/whatsnew/pr/skip_tracebackhide.rst deleted file mode 100644 index 0acfecf127d..00000000000 --- a/docs/source/whatsnew/pr/skip_tracebackhide.rst +++ /dev/null @@ -1,11 +0,0 @@ -The default tracebackmode will now skip frames that are marked with -``__tracebackhide__ = True`` and show how many traceback frames have been -skipped. This can be toggled by using :magic:`xmode` with the ``--show`` or -``--hide`` attribute. It will have no effect on non verbose traceback modes. - -The ipython debugger also now understand ``__tracebackhide__`` as well and will -skip hidden frames when displaying. Movement up and down the stack will skip the -hidden frames and will show how many frames were hidden. Internal IPython frames -are also now hidden by default. The behavior can be changed with the -``skip_hidden`` command and accepts "yes", "no", "true" and "false" case -insensitive parameters. diff --git a/docs/source/whatsnew/version7.rst b/docs/source/whatsnew/version7.rst index 3d9abc96f6f..286f16fb9f4 100644 --- a/docs/source/whatsnew/version7.rst +++ b/docs/source/whatsnew/version7.rst @@ -2,6 +2,67 @@ 7.x Series ============ +.. _version 716: + +IPython 7.16 +============ + + +The default traceback mode will now skip frames that are marked with +``__tracebackhide__ = True`` and show how many traceback frames have been +skipped. This can be toggled by using :magic:`xmode` with the ``--show`` or +``--hide`` attribute. It will have no effect on non verbose traceback modes. + +The ipython debugger also now understands ``__tracebackhide__`` as well and will +skip hidden frames when displaying. Movement up and down the stack will skip the +hidden frames and will show how many frames were hidden. Internal IPython frames +are also now hidden by default. The behavior can be changed with the +``skip_hidden`` while in the debugger, command and accepts "yes", "no", "true" +and "false" case insensitive parameters. + + +Misc Noticeable changes: +------------------------ + +- Exceptions are now (re)raised when running notebooks via the :magic:`%run`, helping to catch issues in workflows and + pipelines. :ghpull:`12301` +- Fix inputhook for qt 5.15.0 :ghpull:`12355` +- Fix wx inputhook :ghpull:`12375` +- Add handling for malformed pathext env var (Windows) :ghpull:`12367` +- use $SHELL in system_piped :ghpull:`12360` for uniform behavior with + ipykernel. + +Reproducible Build +------------------ + +IPython 7.15 reproducible build did not work, so we try again this month +:ghpull:`12358`. + + +API Changes +----------- + +Change of API and exposed objects automatically detected using `frappuccino +`_ (still in beta): + + +The following items are new and mostly related to understanding ``__tracebackbhide__``:: + + + IPython.core.debugger.Pdb.do_down(self, arg) + + IPython.core.debugger.Pdb.do_skip_hidden(self, arg) + + IPython.core.debugger.Pdb.do_up(self, arg) + + IPython.core.debugger.Pdb.hidden_frames(self, stack) + + IPython.core.debugger.Pdb.stop_here(self, frame) + + +The following items have been removed:: + + - IPython.core.debugger.Pdb.new_do_down + - IPython.core.debugger.Pdb.new_do_up + +Those were implementation details. + + .. _version 715: IPython 7.15 From 329684047cfc82be54374e88f6f6785ac99e0bc2 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 26 Jun 2020 15:03:46 -0700 Subject: [PATCH 47/57] release 7.16.0 --- IPython/core/release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index 76bed55d664..4e50e18b838 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -24,7 +24,7 @@ _version_patch = 0 _version_extra = '.dev' # _version_extra = 'b1' -# _version_extra = '' # Uncomment this for full releases +_version_extra = '' # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] From 5ecc01a7d617abca756be8c2bacd0a315ffcb59a Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 26 Jun 2020 15:04:29 -0700 Subject: [PATCH 48/57] back to dev --- IPython/core/release.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index 4e50e18b838..51597dbf83b 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -20,11 +20,11 @@ # release. 'dev' as a _version_extra string means this is a development # version _version_major = 7 -_version_minor = 16 +_version_minor = 17 _version_patch = 0 _version_extra = '.dev' # _version_extra = 'b1' -_version_extra = '' # Uncomment this for full releases +# _version_extra = '' # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] From 20bdc6fc0db7f3cf1f0df4bad9e2f922ac5adc1e Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 26 Jun 2020 15:31:39 -0700 Subject: [PATCH 49/57] fix conda build --- tools/retar.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/retar.py b/tools/retar.py index 9b5e73c6baa..efd8c0c6bbd 100644 --- a/tools/retar.py +++ b/tools/retar.py @@ -46,6 +46,7 @@ m2.size = m.size m2.type = m.type m2.linkname = m.linkname + m2.mode = m.mode if m.isdir(): new.addfile(m2) else: From 2486838d9eaa73413505ec7e72b01d6e262f30f9 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 26 Jun 2020 15:42:20 -0700 Subject: [PATCH 50/57] release 7.16.1 --- IPython/core/release.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index 51597dbf83b..bba11bbaa50 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -20,11 +20,11 @@ # release. 'dev' as a _version_extra string means this is a development # version _version_major = 7 -_version_minor = 17 -_version_patch = 0 +_version_minor = 16 +_version_patch = 1 _version_extra = '.dev' # _version_extra = 'b1' -# _version_extra = '' # Uncomment this for full releases +_version_extra = '' # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] From 8fcdcd384b8b5e17f9b95a10e65a52f3ca78d63f Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 26 Nov 2021 11:19:25 -0800 Subject: [PATCH 51/57] Pin Jedi to <0.17.2. This may not completely fix the issue, in particular pip may decide to downgrade IPython, but maybe this will lead to less errors. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index daf5f55f000..3e734bafe14 100755 --- a/setup.py +++ b/setup.py @@ -186,7 +186,7 @@ install_requires = [ 'setuptools>=18.5', - 'jedi>=0.10', + 'jedi>=0.10,<=0.17.2', 'decorator', 'pickleshare', 'traitlets>=4.2', From bcae8e07c7eeef79d185522411d8b4cb014ee3be Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 26 Nov 2021 11:55:54 -0800 Subject: [PATCH 52/57] Backport PR #13335: What's new 7.16.2 --- docs/source/whatsnew/version7.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/source/whatsnew/version7.rst b/docs/source/whatsnew/version7.rst index 286f16fb9f4..6211783c099 100644 --- a/docs/source/whatsnew/version7.rst +++ b/docs/source/whatsnew/version7.rst @@ -4,6 +4,19 @@ .. _version 716: +IPython 7.16.1, 7.16.2 +====================== + +IPython 7.16.1 was release immediately after 7.16.0 to fix a conda packaging issue. +The source is identical to 7.16.0 but the file permissions in the tar are different. + +IPython 7.16.2 pins jedi dependency to "<=0.17.2" which should prevent some +issues for users still on python 3.6. This may not be sufficient as pip may +still allow to downgrade IPython. + +Compatibility with Jedi > 0.17.2 was not added as this would have meant bumping +the minimal version to >0.16. + IPython 7.16 ============ From 138f2664f1fbd3d04c24bf098732efd8c6d1f2c6 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 26 Nov 2021 14:23:39 -0800 Subject: [PATCH 53/57] bring back release helper from master branch --- tools/release_helper.sh | 45 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/tools/release_helper.sh b/tools/release_helper.sh index dd700b56329..de88d0f0a69 100644 --- a/tools/release_helper.sh +++ b/tools/release_helper.sh @@ -7,7 +7,7 @@ python -c 'import keyring' python -c 'import twine' python -c 'import sphinx' python -c 'import sphinx_rtd_theme' -python -c 'import nose' +python -c 'import pytest' BLACK=$(tput setaf 1) @@ -21,6 +21,7 @@ WHITE=$(tput setaf 7) NOR=$(tput sgr0) +echo "Will use $EDITOR to edit files when necessary" echo -n "PREV_RELEASE (X.y.z) [$PREV_RELEASE]: " read input PREV_RELEASE=${input:-$PREV_RELEASE} @@ -38,18 +39,40 @@ ask_section(){ echo echo $BLUE"$1"$NOR echo -n $GREEN"Press Enter to continue, S to skip: "$NOR - read -n1 value - echo - if [ -z $value ] || [ $value = 'y' ] ; then + if [ "$ZSH_NAME" = "zsh" ] ; then + read -k1 value + value=${value%$'\n'} + else + read -n1 value + fi + if [ -z "$value" ] || [ $value = 'y' ]; then return 0 fi return 1 } +maybe_edit(){ + echo + echo $BLUE"$1"$NOR + echo -n $GREEN"Press e to Edit $1, any other keys to skip: "$NOR + if [ "$ZSH_NAME" = "zsh" ] ; then + read -k1 value + value=${value%$'\n'} + else + read -n1 value + fi + + echo + if [ $value = 'e' ] ; then + $EDITOR $1 + fi +} + + echo -if ask_section "Updating what's new with informations from docs/source/whatsnew/pr" +if ask_section "Updating what's new with information from docs/source/whatsnew/pr" then python tools/update_whatsnew.py @@ -100,6 +123,11 @@ echo "Cleaning repository" git clean -xfdi echo $GREEN"please update version number in ${RED}IPython/core/release.py${NOR} , Do not commit yet – we'll do it later."$NOR +echo $GREEN"I tried ${RED}sed -i bkp -e '/Uncomment/s/^# //g' IPython/core/release.py${NOR}" +sed -i bkp -e '/Uncomment/s/^# //g' IPython/core/release.py +rm IPython/core/release.pybkp +git diff | cat +maybe_edit IPython/core/release.py echo $GREEN"Press enter to continue"$NOR read @@ -142,7 +170,14 @@ then echo $GREEN"please update version number and back to .dev in ${RED}IPython/core/release.py" + echo $GREEN"I tried ${RED}sed -i bkp -e '/Uncomment/s/^/# /g' IPython/core/release.py${NOR}" + sed -i bkp -e '/Uncomment/s/^/# /g' IPython/core/release.py + rm IPython/core/release.pybkp + git diff | cat + echo $GREEN"Please bump ${RED}the minor version number${NOR}" + maybe_edit IPython/core/release.py echo ${BLUE}"Do not commit yet – we'll do it later."$NOR + echo $GREEN"Press enter to continue"$NOR read From 9f477b701e3b7ba0680e24c0f94cd8bf75bca90d Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 26 Nov 2021 14:39:42 -0800 Subject: [PATCH 54/57] release 7.16.2 --- IPython/core/release.py | 2 +- tools/release_helper.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index bba11bbaa50..c7326fe1b5f 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -21,7 +21,7 @@ # version _version_major = 7 _version_minor = 16 -_version_patch = 1 +_version_patch = 2 _version_extra = '.dev' # _version_extra = 'b1' _version_extra = '' # Uncomment this for full releases diff --git a/tools/release_helper.sh b/tools/release_helper.sh index de88d0f0a69..cf053d3459a 100644 --- a/tools/release_helper.sh +++ b/tools/release_helper.sh @@ -21,7 +21,7 @@ WHITE=$(tput setaf 7) NOR=$(tput sgr0) -echo "Will use $EDITOR to edit files when necessary" +echo "Will use '$EDITOR' to edit files when necessary" echo -n "PREV_RELEASE (X.y.z) [$PREV_RELEASE]: " read input PREV_RELEASE=${input:-$PREV_RELEASE} @@ -65,7 +65,7 @@ maybe_edit(){ echo if [ $value = 'e' ] ; then - $EDITOR $1 + $=EDITOR $1 fi } From 8df8971999886a3804a2b9231a47f6d3764b9997 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 26 Nov 2021 14:44:01 -0800 Subject: [PATCH 55/57] back to dev --- IPython/core/release.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index c7326fe1b5f..431086851b7 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -21,10 +21,10 @@ # version _version_major = 7 _version_minor = 16 -_version_patch = 2 +_version_patch = 3 _version_extra = '.dev' # _version_extra = 'b1' -_version_extra = '' # Uncomment this for full releases +# _version_extra = '' # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] From 5fa1e409d2dc126c456510c16ece18e08b524e5b Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Wed, 19 Jan 2022 14:21:24 +0100 Subject: [PATCH 56/57] Merge pull request from GHSA-pq7m-3gw7-gq5x FIX CVE-2022-21699 --- IPython/__init__.py | 4 +++ IPython/core/application.py | 2 +- IPython/core/profileapp.py | 7 ++-- IPython/core/profiledir.py | 4 +-- IPython/tests/cve.py | 56 +++++++++++++++++++++++++++++++ docs/source/whatsnew/version7.rst | 8 +++++ 6 files changed, 75 insertions(+), 6 deletions(-) create mode 100644 IPython/tests/cve.py diff --git a/IPython/__init__.py b/IPython/__init__.py index 4fb77107680..c17ec76a602 100644 --- a/IPython/__init__.py +++ b/IPython/__init__.py @@ -65,6 +65,10 @@ __license__ = release.license __version__ = release.version version_info = release.version_info +# list of CVEs that should have been patched in this release. +# this is informational and should not be relied upon. +__patched_cves__ = {"CVE-2022-21699"} + def embed_kernel(module=None, local_ns=None, **kwargs): """Embed and start an IPython kernel in a given scope. diff --git a/IPython/core/application.py b/IPython/core/application.py index 93639d88e2c..4f679df18e3 100644 --- a/IPython/core/application.py +++ b/IPython/core/application.py @@ -133,7 +133,7 @@ def _config_file_name_changed(self, change): config_file_paths = List(Unicode()) @default('config_file_paths') def _config_file_paths_default(self): - return [os.getcwd()] + return [] extra_config_file = Unicode( help="""Path to an extra config file to load. diff --git a/IPython/core/profileapp.py b/IPython/core/profileapp.py index 97434e3d0b5..9a1bae55ac5 100644 --- a/IPython/core/profileapp.py +++ b/IPython/core/profileapp.py @@ -181,9 +181,10 @@ def list_profile_dirs(self): profiles = list_profiles_in(os.getcwd()) if profiles: print() - print("Available profiles in current directory (%s):" % os.getcwd()) - self._print_profiles(profiles) - + print( + "Profiles from CWD have been removed for security reason, see CVE-2022-21699:" + ) + print() print("To use any of the above profiles, start IPython with:") print(" ipython --profile=") diff --git a/IPython/core/profiledir.py b/IPython/core/profiledir.py index 3199dfd5d64..2c48e4c2f1c 100644 --- a/IPython/core/profiledir.py +++ b/IPython/core/profiledir.py @@ -186,7 +186,7 @@ def find_profile_dir_by_name(cls, ipython_dir, name=u'default', config=None): is not found, a :class:`ProfileDirError` exception will be raised. The search path algorithm is: - 1. ``os.getcwd()`` + 1. ``os.getcwd()`` # removed for security reason. 2. ``ipython_dir`` Parameters @@ -198,7 +198,7 @@ def find_profile_dir_by_name(cls, ipython_dir, name=u'default', config=None): will be "profile_". """ dirname = u'profile_' + name - paths = [os.getcwd(), ipython_dir] + paths = [ipython_dir] for p in paths: profile_dir = os.path.join(p, dirname) if os.path.isdir(profile_dir): diff --git a/IPython/tests/cve.py b/IPython/tests/cve.py new file mode 100644 index 00000000000..026415a57a4 --- /dev/null +++ b/IPython/tests/cve.py @@ -0,0 +1,56 @@ +""" +Test that CVEs stay fixed. +""" + +from IPython.utils.tempdir import TemporaryDirectory, TemporaryWorkingDirectory +from pathlib import Path +import random +import sys +import os +import string +import subprocess +import time + +def test_cve_2022_21699(): + """ + Here we test CVE-2022-21699. + + We create a temporary directory, cd into it. + Make a profile file that should not be executed and start IPython in a subprocess, + checking for the value. + + + + """ + + dangerous_profile_dir = Path('profile_default') + + dangerous_startup_dir = dangerous_profile_dir / 'startup' + dangerous_expected = 'CVE-2022-21699-'+''.join([random.choice(string.ascii_letters) for i in range(10)]) + + with TemporaryWorkingDirectory() as t: + dangerous_startup_dir.mkdir(parents=True) + (dangerous_startup_dir/ 'foo.py').write_text(f'print("{dangerous_expected}")') + # 1 sec to make sure FS is flushed. + #time.sleep(1) + cmd = [sys.executable,'-m', 'IPython'] + env = os.environ.copy() + env['IPY_TEST_SIMPLE_PROMPT'] = '1' + + + # First we fake old behavior, making sure the profile is/was actually dangerous + p_dangerous = subprocess.Popen(cmd + [f'--profile-dir={dangerous_profile_dir}'], env=env, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out_dangerous, err_dangerouns = p_dangerous.communicate(b"exit\r") + assert dangerous_expected in out_dangerous.decode() + + # Now that we know it _would_ have been dangerous, we test it's not loaded + p = subprocess.Popen(cmd, env=env, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate(b"exit\r") + assert b'IPython' in out + assert dangerous_expected not in out.decode() + assert err == b'' + + + diff --git a/docs/source/whatsnew/version7.rst b/docs/source/whatsnew/version7.rst index 6211783c099..f8c88ab844b 100644 --- a/docs/source/whatsnew/version7.rst +++ b/docs/source/whatsnew/version7.rst @@ -2,6 +2,14 @@ 7.x Series ============ +======= +.. _version 7.16.3: + +IPython 7.16.3 (CVE-2022-21699) +=============================== + +Fixed CVE-2022-21699, see IPython 8.0.1 release notes for informations. + .. _version 716: IPython 7.16.1, 7.16.2 From d43c7c79c1c8518f1917c3cd69e5384738ef0b37 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Wed, 19 Jan 2022 14:37:30 +0100 Subject: [PATCH 57/57] release 7.16.3 --- IPython/core/release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index 431086851b7..a6f3cf81f7d 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -24,7 +24,7 @@ _version_patch = 3 _version_extra = '.dev' # _version_extra = 'b1' -# _version_extra = '' # Uncomment this for full releases +_version_extra = "" # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch]