diff --git a/.travis.yml b/.travis.yml index 9bb3a9bb0a1..00c5e3f6bbc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,7 +38,7 @@ install: - pip install setuptools --upgrade - pip install -e file://$PWD#egg=ipython[test] --upgrade - pip install trio curio --upgrade --upgrade-strategy eager - - pip install pytest matplotlib + - pip install pytest 'matplotlib !=3.2.0' mypy - pip install codecov check-manifest --upgrade script: @@ -46,10 +46,11 @@ script: - | if [[ "$TRAVIS_PYTHON_VERSION" == "nightly" ]]; then # on nightly fake parso known the grammar - cp /home/travis/virtualenv/python3.8-dev/lib/python3.8/site-packages/parso/python/grammar37.txt /home/travis/virtualenv/python3.8-dev/lib/python3.8/site-packages/parso/python/grammar38.txt + cp /home/travis/virtualenv/python3.9-dev/lib/python3.9/site-packages/parso/python/grammar38.txt /home/travis/virtualenv/python3.9-dev/lib/python3.9/site-packages/parso/python/grammar39.txt fi - cd /tmp && iptest --coverage xml && cd - - pytest IPython + - mypy --ignore-missing-imports -m IPython.terminal.ptutils # On the latest Python (on Linux) only, make sure that the docs build. - | if [[ "$TRAVIS_PYTHON_VERSION" == "3.7" ]] && [[ "$TRAVIS_OS_NAME" == "linux" ]]; then @@ -69,11 +70,6 @@ matrix: python: "3.7" dist: xenial sudo: true - - arch: arm64 - python: "3.7" - dist: xenial - env: ARM64=True IPYTHON_TESTING_TIMEOUT_SCALE=2 - sudo: true - arch: amd64 python: "3.8-dev" dist: xenial diff --git a/IPython/__init__.py b/IPython/__init__.py index 4fb77107680..c17ec76a602 100644 --- a/IPython/__init__.py +++ b/IPython/__init__.py @@ -65,6 +65,10 @@ __license__ = release.license __version__ = release.version version_info = release.version_info +# list of CVEs that should have been patched in this release. +# this is informational and should not be relied upon. +__patched_cves__ = {"CVE-2022-21699"} + def embed_kernel(module=None, local_ns=None, **kwargs): """Embed and start an IPython kernel in a given scope. diff --git a/IPython/core/application.py b/IPython/core/application.py index 93639d88e2c..4f679df18e3 100644 --- a/IPython/core/application.py +++ b/IPython/core/application.py @@ -133,7 +133,7 @@ def _config_file_name_changed(self, change): config_file_paths = List(Unicode()) @default('config_file_paths') def _config_file_paths_default(self): - return [os.getcwd()] + return [] extra_config_file = Unicode( help="""Path to an extra config file to load. diff --git a/IPython/core/completer.py b/IPython/core/completer.py index 985ac5ab9eb..bc114f0f66b 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -1699,8 +1699,6 @@ def latex_matches(self, text): u"""Match Latex syntax for unicode characters. This does both ``\\alp`` -> ``\\alpha`` and ``\\alpha`` -> ``α`` - - Used on Python 3 only. """ slashpos = text.rfind('\\') if slashpos > -1: @@ -1713,7 +1711,8 @@ def latex_matches(self, text): # If a user has partially typed a latex symbol, give them # a full list of options \al -> [\aleph, \alpha] matches = [k for k in latex_symbols if k.startswith(s)] - return s, matches + if matches: + return s, matches return u'', [] def dispatch_custom_completer(self, text): @@ -1983,8 +1982,8 @@ def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None, # if text is either None or an empty string, rely on the line buffer if (not line_buffer) and full_text: line_buffer = full_text.split('\n')[cursor_line] - if not text: - text = self.splitter.split_line(line_buffer, cursor_pos) + if not text: # issue #11508: check line_buffer before calling split_line + text = self.splitter.split_line(line_buffer, cursor_pos) if line_buffer else '' if self.backslash_combining_completions: # allow deactivation of these on windows. diff --git a/IPython/core/completerlib.py b/IPython/core/completerlib.py index 9e592b0817e..7860cb67dcb 100644 --- a/IPython/core/completerlib.py +++ b/IPython/core/completerlib.py @@ -52,7 +52,7 @@ TIMEOUT_GIVEUP = 20 # Regular expression for the python import statement -import_re = re.compile(r'(?P[a-zA-Z_][a-zA-Z0-9_]*?)' +import_re = re.compile(r'(?P[^\W\d]\w*?)' r'(?P[/\\]__init__)?' r'(?P%s)$' % r'|'.join(re.escape(s) for s in _suffixes)) diff --git a/IPython/core/debugger.py b/IPython/core/debugger.py index ebb8dcac0d8..a330baa450e 100644 --- a/IPython/core/debugger.py +++ b/IPython/core/debugger.py @@ -280,26 +280,31 @@ def __init__(self, color_scheme=None, completekey=None, # Set the prompt - the default prompt is '(Pdb)' self.prompt = prompt + self.skip_hidden = True def set_colors(self, scheme): """Shorthand access to the color table scheme selector method.""" self.color_scheme_table.set_active_scheme(scheme) self.parser.style = scheme + + def hidden_frames(self, stack): + """ + Given an index in the stack return wether it should be skipped. + + This is used in up/down and where to skip frames. + """ + ip_hide = [s[0].f_locals.get("__tracebackhide__", False) for s in stack] + ip_start = [i for i, s in enumerate(ip_hide) if s == "__ipython_bottom__"] + if ip_start: + ip_hide = [h if i > ip_start[0] else True for (i, h) in enumerate(ip_hide)] + return ip_hide + def interaction(self, frame, traceback): try: OldPdb.interaction(self, frame, traceback) except KeyboardInterrupt: - self.stdout.write('\n' + self.shell.get_exception_only()) - - def new_do_up(self, arg): - OldPdb.do_up(self, arg) - do_u = do_up = decorate_fn_with_doc(new_do_up, OldPdb.do_up) - - def new_do_down(self, arg): - OldPdb.do_down(self, arg) - - do_d = do_down = decorate_fn_with_doc(new_do_down, OldPdb.do_down) + self.stdout.write("\n" + self.shell.get_exception_only()) def new_do_frame(self, arg): OldPdb.do_frame(self, arg) @@ -320,6 +325,8 @@ def new_do_restart(self, arg): return self.do_quit(arg) def print_stack_trace(self, context=None): + Colors = self.color_scheme_table.active_colors + ColorsNormal = Colors.Normal if context is None: context = self.context try: @@ -329,8 +336,21 @@ def print_stack_trace(self, context=None): except (TypeError, ValueError): raise ValueError("Context must be a positive integer") try: - for frame_lineno in self.stack: + skipped = 0 + for hidden, frame_lineno in zip(self.hidden_frames(self.stack), self.stack): + if hidden and self.skip_hidden: + skipped += 1 + continue + if skipped: + print( + f"{Colors.excName} [... skipping {skipped} hidden frame(s)]{ColorsNormal}\n" + ) + skipped = 0 self.print_stack_entry(frame_lineno, context=context) + if skipped: + print( + f"{Colors.excName} [... skipping {skipped} hidden frame(s)]{ColorsNormal}\n" + ) except KeyboardInterrupt: pass @@ -487,6 +507,16 @@ def print_list_lines(self, filename, first, last): except KeyboardInterrupt: pass + def do_skip_hidden(self, arg): + """ + Change whether or not we should skip frames with the + __tracebackhide__ attribute. + """ + if arg.strip().lower() in ("true", "yes"): + self.skip_hidden = True + elif arg.strip().lower() in ("false", "no"): + self.skip_hidden = False + def do_list(self, arg): """Print lines of code from the current stack frame """ @@ -622,13 +652,148 @@ def do_where(self, arg): Take a number as argument as an (optional) number of context line to print""" if arg: - context = int(arg) + try: + context = int(arg) + except ValueError as err: + self.error(err) + return self.print_stack_trace(context) else: self.print_stack_trace() do_w = do_where + def stop_here(self, frame): + hidden = False + if self.skip_hidden: + hidden = frame.f_locals.get("__tracebackhide__", False) + if hidden: + Colors = self.color_scheme_table.active_colors + ColorsNormal = Colors.Normal + print(f"{Colors.excName} [... skipped 1 hidden frame]{ColorsNormal}\n") + + return super().stop_here(frame) + + def do_up(self, arg): + """u(p) [count] + Move the current frame count (default one) levels up in the + stack trace (to an older frame). + + Will skip hidden frames. + """ + ## modified version of upstream that skips + # frames with __tracebackide__ + if self.curindex == 0: + self.error("Oldest frame") + return + try: + count = int(arg or 1) + except ValueError: + self.error("Invalid frame count (%s)" % arg) + return + skipped = 0 + if count < 0: + _newframe = 0 + else: + _newindex = self.curindex + counter = 0 + hidden_frames = self.hidden_frames(self.stack) + for i in range(self.curindex - 1, -1, -1): + frame = self.stack[i][0] + if hidden_frames[i] and self.skip_hidden: + skipped += 1 + continue + counter += 1 + if counter >= count: + break + else: + # if no break occured. + self.error("all frames above hidden") + return + + Colors = self.color_scheme_table.active_colors + ColorsNormal = Colors.Normal + _newframe = i + self._select_frame(_newframe) + if skipped: + print( + f"{Colors.excName} [... skipped {skipped} hidden frame(s)]{ColorsNormal}\n" + ) + + def do_down(self, arg): + """d(own) [count] + Move the current frame count (default one) levels down in the + stack trace (to a newer frame). + + Will skip hidden frames. + """ + if self.curindex + 1 == len(self.stack): + self.error("Newest frame") + return + try: + count = int(arg or 1) + except ValueError: + self.error("Invalid frame count (%s)" % arg) + return + if count < 0: + _newframe = len(self.stack) - 1 + else: + _newindex = self.curindex + counter = 0 + skipped = 0 + hidden_frames = self.hidden_frames(self.stack) + for i in range(self.curindex + 1, len(self.stack)): + frame = self.stack[i][0] + if hidden_frames[i] and self.skip_hidden: + skipped += 1 + continue + counter += 1 + if counter >= count: + break + else: + self.error("all frames bellow hidden") + return + + Colors = self.color_scheme_table.active_colors + ColorsNormal = Colors.Normal + if skipped: + print( + f"{Colors.excName} [... skipped {skipped} hidden frame(s)]{ColorsNormal}\n" + ) + _newframe = i + + self._select_frame(_newframe) + + do_d = do_down + do_u = do_up + +class InterruptiblePdb(Pdb): + """Version of debugger where KeyboardInterrupt exits the debugger altogether.""" + + def cmdloop(self): + """Wrap cmdloop() such that KeyboardInterrupt stops the debugger.""" + try: + return OldPdb.cmdloop(self) + except KeyboardInterrupt: + self.stop_here = lambda frame: False + self.do_quit("") + sys.settrace(None) + self.quitting = False + raise + + def _cmdloop(self): + while True: + try: + # keyboard interrupts allow for an easy way to cancel + # the current command, so allow them during interactive input + self.allow_kbdint = True + self.cmdloop() + self.allow_kbdint = False + break + except KeyboardInterrupt: + self.message('--KeyboardInterrupt--') + raise + def set_trace(frame=None): """ diff --git a/IPython/core/display.py b/IPython/core/display.py index 465c000c55a..424414a662f 100644 --- a/IPython/core/display.py +++ b/IPython/core/display.py @@ -615,9 +615,12 @@ def __init__(self, data=None, url=None, filename=None, metadata=None): filename = data data = None - self.data = data self.url = url self.filename = filename + # because of @data.setter methods in + # subclasses ensure url and filename are set + # before assigning to self.data + self.data = data if metadata is not None: self.metadata = metadata @@ -652,23 +655,36 @@ def reload(self): with open(self.filename, self._read_flags) as f: self.data = f.read() elif self.url is not None: - try: - # Deferred import - from urllib.request import urlopen - response = urlopen(self.url) - self.data = response.read() - # extract encoding from header, if there is one: - encoding = None + # Deferred import + from urllib.request import urlopen + response = urlopen(self.url) + data = response.read() + # extract encoding from header, if there is one: + encoding = None + if 'content-type' in response.headers: for sub in response.headers['content-type'].split(';'): sub = sub.strip() if sub.startswith('charset'): encoding = sub.split('=')[-1].strip() break - # decode data, if an encoding was specified - if encoding: - self.data = self.data.decode(encoding, 'replace') - except: - self.data = None + if 'content-encoding' in response.headers: + # TODO: do deflate? + if 'gzip' in response.headers['content-encoding']: + import gzip + from io import BytesIO + with gzip.open(BytesIO(data), 'rt', encoding=encoding) as fp: + encoding = None + data = fp.read() + + # decode data, if an encoding was specified + # We only touch self.data once since + # subclasses such as SVG have @data.setter methods + # that transform self.data into ... well svg. + if encoding: + self.data = data.decode(encoding, 'replace') + else: + self.data = data + class TextDisplayObject(DisplayObject): """Validate that display data is text""" @@ -736,6 +752,11 @@ def _repr_latex_(self): class SVG(DisplayObject): + """Embed an SVG into the display. + + Note if you just want to view a svg image via a URL use `:class:Image` with + a url=URL keyword argument. + """ _read_flags = 'rb' # wrap data in a property, which extracts the tag, discarding @@ -879,7 +900,7 @@ def data(self, data): data = str(data) if isinstance(data, str): - if getattr(self, 'filename', None) is None: + if self.filename is None and self.url is None: warnings.warn("JSON expects JSONable dict or list, not JSON strings") data = json.loads(data) self._data = data @@ -1308,7 +1329,7 @@ def _find_ext(self, s): class Video(DisplayObject): def __init__(self, data=None, url=None, filename=None, embed=False, - mimetype=None, width=None, height=None): + mimetype=None, width=None, height=None, html_attributes="controls"): """Create a video object given raw data or an URL. When this object is returned by an input cell or passed to the @@ -1346,14 +1367,22 @@ def __init__(self, data=None, url=None, filename=None, embed=False, height : int Height in pixels to which to constrain the video in html. If not supplied, defaults to the height of the video. + html_attributes : str + Attributes for the HTML `video element. - """.format(url, width, height) + """.format(url, self.html_attributes, width, height) return output # Embedded videos are base64-encoded. @@ -1411,10 +1441,10 @@ def _repr_html_(self): else: b64_video = b2a_base64(video).decode('ascii').rstrip() - output = """""".format(self.html_attributes, width, height, mimetype, b64_video) return output def reload(self): diff --git a/IPython/core/inputtransformer.py b/IPython/core/inputtransformer.py index 1c35eb64f32..afeca93cc0e 100644 --- a/IPython/core/inputtransformer.py +++ b/IPython/core/inputtransformer.py @@ -278,8 +278,8 @@ def escaped_commands(line): _initial_space_re = re.compile(r'\s*') _help_end_re = re.compile(r"""(%{0,2} - [a-zA-Z_*][\w*]* # Variable name - (\.[a-zA-Z_*][\w*]*)* # .etc.etc + (?!\d)[\w*]+ # Variable name + (\.(?!\d)[\w*]+)* # .etc.etc ) (\?\??)$ # ? or ?? """, diff --git a/IPython/core/inputtransformer2.py b/IPython/core/inputtransformer2.py index 4562fe01d2c..0443e6829b4 100644 --- a/IPython/core/inputtransformer2.py +++ b/IPython/core/inputtransformer2.py @@ -405,8 +405,8 @@ def transform(self, lines): return lines_before + [new_line] + lines_after _help_end_re = re.compile(r"""(%{0,2} - [a-zA-Z_*][\w*]* # Variable name - (\.[a-zA-Z_*][\w*]*)* # .etc.etc + (?!\d)[\w*]+ # Variable name + (\.(?!\d)[\w*]+)* # .etc.etc ) (\?\??)$ # ? or ?? """, diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 848a14a481c..ddb1b64ea78 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -2211,13 +2211,22 @@ def complete(self, text, line=None, cursor_pos=None): with self.builtin_trap: return self.Completer.complete(text, line, cursor_pos) - def set_custom_completer(self, completer, pos=0): + def set_custom_completer(self, completer, pos=0) -> None: """Adds a new custom completer function. The position argument (defaults to 0) is the index in the completers - list where you want the completer to be inserted.""" + list where you want the completer to be inserted. - newcomp = types.MethodType(completer,self.Completer) + `completer` should have the following signature:: + + def completion(self: Completer, text: string) -> List[str]: + raise NotImplementedError + + It will be bound to the current Completer instance and pass some text + and return a list with current completions to suggest to the user. + """ + + newcomp = types.MethodType(completer, self.Completer) self.Completer.custom_matchers.insert(pos,newcomp) def set_completer_frame(self, frame=None): @@ -3310,6 +3319,9 @@ async def run_code(self, code_obj, result=None, *, async_=False): False : successful execution. True : an error occurred. """ + # special value to say that anything above is IPython and should be + # hidden. + __tracebackhide__ = "__ipython_bottom__" # Set our own excepthook in case the user code tries to call it # directly, so that the IPython crash handler doesn't get triggered old_excepthook, sys.excepthook = sys.excepthook, self.excepthook diff --git a/IPython/core/magics/basic.py b/IPython/core/magics/basic.py index 5c8649b5408..a8feb755386 100644 --- a/IPython/core/magics/basic.py +++ b/IPython/core/magics/basic.py @@ -364,13 +364,25 @@ def xmode(self, parameter_s=''): Valid modes: Plain, Context, Verbose, and Minimal. - If called without arguments, acts as a toggle.""" + If called without arguments, acts as a toggle. + + When in verbose mode the value --show (and --hide) + will respectively show (or hide) frames with ``__tracebackhide__ = + True`` value set. + """ def xmode_switch_err(name): warn('Error changing %s exception modes.\n%s' % (name,sys.exc_info()[1])) shell = self.shell + if parameter_s.strip() == "--show": + shell.InteractiveTB.skip_hidden = False + return + if parameter_s.strip() == "--hide": + shell.InteractiveTB.skip_hidden = True + return + new_mode = parameter_s.strip().capitalize() try: shell.InteractiveTB.set_mode(mode=new_mode) diff --git a/IPython/core/magics/execution.py b/IPython/core/magics/execution.py index 438d0b5f0ad..dc6cdf00e29 100644 --- a/IPython/core/magics/execution.py +++ b/IPython/core/magics/execution.py @@ -688,17 +688,16 @@ def run(self, parameter_s='', runner=None, modulename = opts["m"][0] modpath = find_mod(modulename) if modpath is None: - warn('%r is not a valid modulename on sys.path'%modulename) - return + msg = '%r is not a valid modulename on sys.path'%modulename + raise Exception(msg) arg_lst = [modpath] + arg_lst try: fpath = None # initialize to make sure fpath is in scope later fpath = arg_lst[0] filename = file_finder(fpath) except IndexError: - warn('you must provide at least a filename.') - print('\n%run:\n', oinspect.getdoc(self.run)) - return + msg = 'you must provide at least a filename.' + raise Exception(msg) except IOError as e: try: msg = str(e) @@ -706,13 +705,17 @@ def run(self, parameter_s='', runner=None, msg = e.message if os.name == 'nt' and re.match(r"^'.*'$",fpath): warn('For Windows, use double quotes to wrap a filename: %run "mypath\\myfile.py"') - error(msg) - return + raise Exception(msg) + except TypeError: + if fpath in sys.meta_path: + filename = "" + else: + raise if filename.lower().endswith(('.ipy', '.ipynb')): with preserve_keys(self.shell.user_ns, '__file__'): self.shell.user_ns['__file__'] = filename - self.shell.safe_execfile_ipy(filename) + self.shell.safe_execfile_ipy(filename, raise_exceptions=True) return # Control the response to exit() calls made by the script being run diff --git a/IPython/core/magics/namespace.py b/IPython/core/magics/namespace.py index cef6ddba8d7..acc4620549b 100644 --- a/IPython/core/magics/namespace.py +++ b/IPython/core/magics/namespace.py @@ -208,12 +208,6 @@ def psearch(self, parameter_s=''): %psearch -l list all available object types """ - try: - parameter_s.encode('ascii') - except UnicodeEncodeError: - print('Python identifiers can only contain ascii characters.') - return - # default namespaces to be searched def_search = ['user_local', 'user_global', 'builtin'] diff --git a/IPython/core/magics/osm.py b/IPython/core/magics/osm.py index ddc9b6f5874..90da7e22803 100644 --- a/IPython/core/magics/osm.py +++ b/IPython/core/magics/osm.py @@ -25,6 +25,7 @@ from IPython.utils.process import abbrev_cwd from IPython.utils.terminal import set_term_title from traitlets import Bool +from warnings import warn @magics_class @@ -48,8 +49,15 @@ def __init__(self, shell=None, **kwargs): winext = os.environ['pathext'].replace(';','|').replace('.','') except KeyError: winext = 'exe|com|bat|py' - - self.execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE) + try: + self.execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE) + except re.error: + warn("Seems like your pathext environmental " + "variable is malformed. Please check it to " + "enable a proper handle of file extensions " + "managed for your system") + winext = 'exe|com|bat|py' + self.execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE) # call up the chain super().__init__(shell=shell, **kwargs) diff --git a/IPython/core/prefilter.py b/IPython/core/prefilter.py index dbf185e6a42..bf801f999c4 100644 --- a/IPython/core/prefilter.py +++ b/IPython/core/prefilter.py @@ -37,7 +37,7 @@ class PrefilterError(Exception): # RegExp to identify potential function names -re_fun_name = re.compile(r'[a-zA-Z_]([a-zA-Z0-9_.]*) *$') +re_fun_name = re.compile(r'[^\W\d]([\w.]*) *$') # RegExp to exclude strings with this start from autocalling. In # particular, all binary operators should be excluded, so that if foo is diff --git a/IPython/core/profileapp.py b/IPython/core/profileapp.py index 97434e3d0b5..9a1bae55ac5 100644 --- a/IPython/core/profileapp.py +++ b/IPython/core/profileapp.py @@ -181,9 +181,10 @@ def list_profile_dirs(self): profiles = list_profiles_in(os.getcwd()) if profiles: print() - print("Available profiles in current directory (%s):" % os.getcwd()) - self._print_profiles(profiles) - + print( + "Profiles from CWD have been removed for security reason, see CVE-2022-21699:" + ) + print() print("To use any of the above profiles, start IPython with:") print(" ipython --profile=") diff --git a/IPython/core/profiledir.py b/IPython/core/profiledir.py index 3199dfd5d64..2c48e4c2f1c 100644 --- a/IPython/core/profiledir.py +++ b/IPython/core/profiledir.py @@ -186,7 +186,7 @@ def find_profile_dir_by_name(cls, ipython_dir, name=u'default', config=None): is not found, a :class:`ProfileDirError` exception will be raised. The search path algorithm is: - 1. ``os.getcwd()`` + 1. ``os.getcwd()`` # removed for security reason. 2. ``ipython_dir`` Parameters @@ -198,7 +198,7 @@ def find_profile_dir_by_name(cls, ipython_dir, name=u'default', config=None): will be "profile_". """ dirname = u'profile_' + name - paths = [os.getcwd(), ipython_dir] + paths = [ipython_dir] for p in paths: profile_dir = os.path.join(p, dirname) if os.path.isdir(profile_dir): diff --git a/IPython/core/release.py b/IPython/core/release.py index d818b071dc5..a6f3cf81f7d 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -20,11 +20,11 @@ # release. 'dev' as a _version_extra string means this is a development # version _version_major = 7 -_version_minor = 13 -_version_patch = 0 +_version_minor = 16 +_version_patch = 3 _version_extra = '.dev' # _version_extra = 'b1' -_version_extra = '' # Uncomment this for full releases +_version_extra = "" # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] diff --git a/IPython/core/shellapp.py b/IPython/core/shellapp.py index 33052c44d15..9e8bfbfbb81 100644 --- a/IPython/core/shellapp.py +++ b/IPython/core/shellapp.py @@ -418,7 +418,8 @@ def _run_cmd_line_code(self): fname = os.path.join(fname, "__main__.py") if not os.path.exists(fname): self.log.warning("File '%s' doesn't exist", fname) - self.exit(2) + if not self.interact: + self.exit(2) try: self._exec_file(fname, shell_futures=True) except: diff --git a/IPython/core/tests/test_async_helpers.py b/IPython/core/tests/test_async_helpers.py index 86a516ccc1f..11c475874d7 100644 --- a/IPython/core/tests/test_async_helpers.py +++ b/IPython/core/tests/test_async_helpers.py @@ -8,7 +8,7 @@ from textwrap import dedent, indent from unittest import TestCase from IPython.testing.decorators import skip_without - +import sys iprc = lambda x: ip.run_cell(dedent(x)).raise_error() iprc_nr = lambda x: ip.run_cell(dedent(x)) @@ -275,10 +275,13 @@ def test_autoawait(self): await sleep(0.1) """ ) - - def test_memory_error(self): - with self.assertRaises(MemoryError): - iprc("(" * 200 + ")" * 200) + + if sys.version_info < (3,9): + # new pgen parser in 3.9 does not raise MemoryError on too many nested + # parens anymore + def test_memory_error(self): + with self.assertRaises(MemoryError): + iprc("(" * 200 + ")" * 200) @skip_without('curio') def test_autoawait_curio(self): diff --git a/IPython/core/tests/test_completer.py b/IPython/core/tests/test_completer.py index 2920d453936..2c19e2e0187 100644 --- a/IPython/core/tests/test_completer.py +++ b/IPython/core/tests/test_completer.py @@ -224,20 +224,27 @@ def test_latex_completions(self): nt.assert_in("\\alpha", matches) nt.assert_in("\\aleph", matches) + def test_latex_no_results(self): + """ + forward latex should really return nothing in either field if nothing is found. + """ + ip = get_ipython() + text, matches = ip.Completer.latex_matches("\\really_i_should_match_nothing") + nt.assert_equal(text, "") + nt.assert_equal(matches, []) + def test_back_latex_completion(self): ip = get_ipython() # do not return more than 1 matches fro \beta, only the latex one. name, matches = ip.complete("\\β") - nt.assert_equal(len(matches), 1) - nt.assert_equal(matches[0], "\\beta") + nt.assert_equal(matches, ['\\beta']) def test_back_unicode_completion(self): ip = get_ipython() name, matches = ip.complete("\\Ⅴ") - nt.assert_equal(len(matches), 1) - nt.assert_equal(matches[0], "\\ROMAN NUMERAL FIVE") + nt.assert_equal(matches, ["\\ROMAN NUMERAL FIVE"]) def test_forward_unicode_completion(self): ip = get_ipython() diff --git a/IPython/core/tests/test_debugger.py b/IPython/core/tests/test_debugger.py index dcfd9a42438..9fdc944e4d0 100644 --- a/IPython/core/tests/test_debugger.py +++ b/IPython/core/tests/test_debugger.py @@ -4,12 +4,24 @@ # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. +import bdb +import builtins +import os +import signal +import subprocess import sys +import time import warnings +from subprocess import PIPE, CalledProcessError, check_output +from tempfile import NamedTemporaryFile +from textwrap import dedent +from unittest.mock import patch import nose.tools as nt from IPython.core import debugger +from IPython.testing import IPYTHON_TESTING_TIMEOUT_SCALE +from IPython.testing.decorators import skip_win32 #----------------------------------------------------------------------------- # Helper classes, from CPython's Pdb test suite @@ -223,3 +235,92 @@ def can_exit(): >>> sys.settrace(old_trace) ''' + + +def test_interruptible_core_debugger(): + """The debugger can be interrupted. + + The presumption is there is some mechanism that causes a KeyboardInterrupt + (this is implemented in ipykernel). We want to ensure the + KeyboardInterrupt cause debugging to cease. + """ + def raising_input(msg="", called=[0]): + called[0] += 1 + if called[0] == 1: + raise KeyboardInterrupt() + else: + raise AssertionError("input() should only be called once!") + + with patch.object(builtins, "input", raising_input): + debugger.InterruptiblePdb().set_trace() + # The way this test will fail is by set_trace() never exiting, + # resulting in a timeout by the test runner. The alternative + # implementation would involve a subprocess, but that adds issues with + # interrupting subprocesses that are rather complex, so it's simpler + # just to do it this way. + +@skip_win32 +def test_xmode_skip(): + """that xmode skip frames + + Not as a doctest as pytest does not run doctests. + """ + import pexpect + env = os.environ.copy() + env["IPY_TEST_SIMPLE_PROMPT"] = "1" + + child = pexpect.spawn( + sys.executable, ["-m", "IPython", "--colors=nocolor"], env=env + ) + child.timeout = 15 * IPYTHON_TESTING_TIMEOUT_SCALE + + child.expect("IPython") + child.expect("\n") + child.expect_exact("In [1]") + + block = dedent( + """ +def f(): + __tracebackhide__ = True + g() + +def g(): + raise ValueError + +f() + """ + ) + + for line in block.splitlines(): + child.sendline(line) + child.expect_exact(line) + child.expect_exact("skipping") + + block = dedent( + """ +def f(): + __tracebackhide__ = True + g() + +def g(): + from IPython.core.debugger import set_trace + set_trace() + +f() + """ + ) + + for line in block.splitlines(): + child.sendline(line) + child.expect_exact(line) + + child.expect("ipdb>") + child.sendline("w") + child.expect("hidden") + child.expect("ipdb>") + child.sendline("skip_hidden false") + child.sendline("w") + child.expect("__traceba") + child.expect("ipdb>") + + child.close() diff --git a/IPython/core/tests/test_display.py b/IPython/core/tests/test_display.py index 0ce4ad2b429..95f1eb622e4 100644 --- a/IPython/core/tests/test_display.py +++ b/IPython/core/tests/test_display.py @@ -72,6 +72,40 @@ def test_retina_png(): nt.assert_equal(md['width'], 1) nt.assert_equal(md['height'], 1) +def test_embed_svg_url(): + import gzip + from io import BytesIO + svg_data = b'' + url = 'http://test.com/circle.svg' + + gzip_svg = BytesIO() + with gzip.open(gzip_svg, 'wb') as fp: + fp.write(svg_data) + gzip_svg = gzip_svg.getvalue() + + def mocked_urlopen(*args, **kwargs): + class MockResponse: + def __init__(self, svg): + self._svg_data = svg + self.headers = {'content-type': 'image/svg+xml'} + + def read(self): + return self._svg_data + + if args[0] == url: + return MockResponse(svg_data) + elif args[0] == url + 'z': + ret= MockResponse(gzip_svg) + ret.headers['content-encoding']= 'gzip' + return ret + return MockResponse(None) + + with mock.patch('urllib.request.urlopen', side_effect=mocked_urlopen): + svg = display.SVG(url=url) + nt.assert_true(svg._repr_svg_().startswith('=3.9 + '''\ +long_function( + a_really_long_parameter: int, + and_another_long_one: bool = False, + let_us_make_sure_this_is_looong: Optional[str] = None, +) -> bool\ +''', # Python >=3.7 '''\ long_function( diff --git a/IPython/core/tests/test_prefilter.py b/IPython/core/tests/test_prefilter.py index 0e61b4693f7..ca447b3d0b7 100644 --- a/IPython/core/tests/test_prefilter.py +++ b/IPython/core/tests/test_prefilter.py @@ -115,3 +115,13 @@ def __call__(self, x): finally: del ip.user_ns['x'] ip.magic('autocall 0') + + +def test_autocall_should_support_unicode(): + ip.magic('autocall 2') + ip.user_ns['π'] = lambda x: x + try: + nt.assert_equal(ip.prefilter('π 3'),'π(3)') + finally: + ip.magic('autocall 0') + del ip.user_ns['π'] diff --git a/IPython/core/tests/test_pylabtools.py b/IPython/core/tests/test_pylabtools.py index 7b724000f58..7b64aab111a 100644 --- a/IPython/core/tests/test_pylabtools.py +++ b/IPython/core/tests/test_pylabtools.py @@ -61,7 +61,7 @@ def test_figure_to_jpeg(): ax = fig.add_subplot(1,1,1) ax.plot([1,2,3]) plt.draw() - jpeg = pt.print_figure(fig, 'jpeg', quality=50)[:100].lower() + jpeg = pt.print_figure(fig, 'jpeg', pil_kwargs={'optimize': 50})[:100].lower() assert jpeg.startswith(_JPEG) def test_retina_figure(): diff --git a/IPython/core/tests/test_run.py b/IPython/core/tests/test_run.py index 38d71b31740..eff832b3fc0 100644 --- a/IPython/core/tests/test_run.py +++ b/IPython/core/tests/test_run.py @@ -402,6 +402,25 @@ def test_run_nb(self): nt.assert_equal(_ip.user_ns['answer'], 42) + def test_run_nb_error(self): + """Test %run notebook.ipynb error""" + from nbformat import v4, writes + # %run when a file name isn't provided + nt.assert_raises(Exception, _ip.magic, "run") + + # %run when a file doesn't exist + nt.assert_raises(Exception, _ip.magic, "run foobar.ipynb") + + # %run on a notebook with an error + nb = v4.new_notebook( + cells=[ + v4.new_code_cell("0/0") + ] + ) + src = writes(nb, version=4) + self.mktmp(src, ext='.ipynb') + nt.assert_raises(Exception, _ip.magic, "run %s" % self.fname) + def test_file_options(self): src = ('import sys\n' 'a = " ".join(sys.argv[1:])\n') diff --git a/IPython/core/tests/test_ultratb.py b/IPython/core/tests/test_ultratb.py index 3ab0ce3cf00..3751117b692 100644 --- a/IPython/core/tests/test_ultratb.py +++ b/IPython/core/tests/test_ultratb.py @@ -252,12 +252,16 @@ def test_non_syntaxerror(self): with tt.AssertPrints('QWERTY'): ip.showsyntaxerror() - -class MemoryErrorTest(unittest.TestCase): - def test_memoryerror(self): - memoryerror_code = "(" * 200 + ")" * 200 - with tt.AssertPrints("MemoryError"): - ip.run_cell(memoryerror_code) +import sys +if sys.version_info < (3,9): + """ + New 3.9 Pgen Parser does not raise Memory error, except on failed malloc. + """ + class MemoryErrorTest(unittest.TestCase): + def test_memoryerror(self): + memoryerror_code = "(" * 200 + ")" * 200 + with tt.AssertPrints("MemoryError"): + ip.run_cell(memoryerror_code) class Python3ChainedExceptionsTest(unittest.TestCase): diff --git a/IPython/core/ultratb.py b/IPython/core/ultratb.py index 9e7e8fb390d..45e22bd7b94 100644 --- a/IPython/core/ultratb.py +++ b/IPython/core/ultratb.py @@ -879,13 +879,36 @@ def __init__(self, color_scheme='Linux', call_pdb=False, ostream=None, self.check_cache = check_cache self.debugger_cls = debugger_cls or debugger.Pdb + self.skip_hidden = True def format_records(self, records, last_unique, recursion_repeat): """Format the stack frames of the traceback""" frames = [] + + skipped = 0 for r in records[:last_unique+recursion_repeat+1]: - #print '*** record:',file,lnum,func,lines,index # dbg + if self.skip_hidden: + if r[0].f_locals.get("__tracebackhide__", 0): + skipped += 1 + continue + if skipped: + Colors = self.Colors # just a shorthand + quicker name lookup + ColorsNormal = Colors.Normal # used a lot + frames.append( + " %s[... skipping hidden %s frame]%s\n" + % (Colors.excName, skipped, ColorsNormal) + ) + skipped = 0 + frames.append(self.format_record(*r)) + + if skipped: + Colors = self.Colors # just a shorthand + quicker name lookup + ColorsNormal = Colors.Normal # used a lot + frames.append( + " %s[... skipping hidden %s frame]%s\n" + % (Colors.excName, skipped, ColorsNormal) + ) if recursion_repeat: frames.append('... last %d frames repeated, from the frame below ...\n' % recursion_repeat) @@ -1123,8 +1146,6 @@ def format_exception_as_a_whole(self, etype, evalue, etb, number_of_lines_of_con head = self.prepare_header(etype, self.long_header) records = self.get_records(etb, number_of_lines_of_context, tb_offset) - if records is None: - return "" last_unique, recursion_repeat = find_recursion(orig_etype, evalue, records) diff --git a/IPython/lib/pretty.py b/IPython/lib/pretty.py index 3115d3f6dbf..1cb46b1413d 100644 --- a/IPython/lib/pretty.py +++ b/IPython/lib/pretty.py @@ -648,6 +648,22 @@ def _re_pattern_pprint(obj, p, cycle): p.text(')') +def _types_simplenamespace_pprint(obj, p, cycle): + """The pprint function for types.SimpleNamespace.""" + name = 'namespace' + with p.group(len(name) + 1, name + '(', ')'): + if cycle: + p.text('...') + else: + for idx, (attr, value) in enumerate(obj.__dict__.items()): + if idx: + p.text(',') + p.breakable() + attr_kwarg = '{}='.format(attr) + with p.group(len(attr_kwarg), attr_kwarg): + p.pretty(value) + + def _type_pprint(obj, p, cycle): """The pprint for classes and types.""" # Heap allocated types might not have the module attribute, @@ -741,6 +757,7 @@ def _exception_pprint(obj, p, cycle): types.FunctionType: _function_pprint, types.BuiltinFunctionType: _function_pprint, types.MethodType: _repr_pprint, + types.SimpleNamespace: _types_simplenamespace_pprint, datetime.datetime: _repr_pprint, datetime.timedelta: _repr_pprint, _exception_base: _exception_pprint diff --git a/IPython/lib/tests/test_pretty.py b/IPython/lib/tests/test_pretty.py index 695012d6fc9..ba4c3296694 100644 --- a/IPython/lib/tests/test_pretty.py +++ b/IPython/lib/tests/test_pretty.py @@ -407,6 +407,26 @@ def test_mappingproxy(): nt.assert_equal(pretty.pretty(obj), expected) +def test_simplenamespace(): + SN = types.SimpleNamespace + + sn_recursive = SN() + sn_recursive.first = sn_recursive + sn_recursive.second = sn_recursive + cases = [ + (SN(), "namespace()"), + (SN(x=SN()), "namespace(x=namespace())"), + (SN(a_long_name=[SN(s=string.ascii_lowercase)]*3, a_short_name=None), + "namespace(a_long_name=[namespace(s='abcdefghijklmnopqrstuvwxyz'),\n" + " namespace(s='abcdefghijklmnopqrstuvwxyz'),\n" + " namespace(s='abcdefghijklmnopqrstuvwxyz')],\n" + " a_short_name=None)"), + (sn_recursive, "namespace(first=namespace(...), second=namespace(...))"), + ] + for obj, expected in cases: + nt.assert_equal(pretty.pretty(obj), expected) + + def test_pretty_environ(): dict_repr = pretty.pretty(dict(os.environ)) # reindent to align with 'environ' prefix diff --git a/IPython/sphinxext/ipython_directive.py b/IPython/sphinxext/ipython_directive.py index 0481624c42c..3d5b3768230 100644 --- a/IPython/sphinxext/ipython_directive.py +++ b/IPython/sphinxext/ipython_directive.py @@ -419,8 +419,8 @@ def process_image(self, decorator): # insert relative path to image file in source # as absolute path for Sphinx # sphinx expects a posix path, even on Windows - posix_path = pathlib.Path(savefig_dir,filename).as_posix() - outfile = '/' + os.path.relpath(posix_path, source_dir) + path = pathlib.Path(savefig_dir, filename) + outfile = '/' + path.relative_to(source_dir).as_posix() imagerows = ['.. image:: %s' % outfile] diff --git a/IPython/terminal/debugger.py b/IPython/terminal/debugger.py index f3268e70bd6..ffd53c65e78 100644 --- a/IPython/terminal/debugger.py +++ b/IPython/terminal/debugger.py @@ -41,6 +41,16 @@ def get_prompt_tokens(): global_namespace={}, parent=self.shell, ) + # add a completer for all the do_ methods + methods_names = [m[3:] for m in dir(self) if m.startswith("do_")] + + def gen_comp(self, text): + return [m for m in methods_names if m.startswith(text)] + import types + newcomp = types.MethodType(gen_comp, compl) + compl.custom_matchers.insert(0, newcomp) + # end add completer. + self._ptcomp = IPythonPTCompleter(compl) options = dict( diff --git a/IPython/terminal/interactiveshell.py b/IPython/terminal/interactiveshell.py index 9f7d335ead8..b3bb5777450 100644 --- a/IPython/terminal/interactiveshell.py +++ b/IPython/terminal/interactiveshell.py @@ -102,7 +102,11 @@ class TerminalInteractiveShell(InteractiveShell): mime_renderers = Dict().tag(config=True) space_for_menu = Integer(6, help='Number of line at the bottom of the screen ' - 'to reserve for the completion menu' + 'to reserve for the tab completion menu, ' + 'search history, ...etc, the height of ' + 'these menus will at most this value. ' + 'Increase it is you prefer long and skinny ' + 'menus, decrease for short and wide.' ).tag(config=True) pt_app = None @@ -322,6 +326,7 @@ def prompt(): mouse_support=self.mouse_support, enable_open_in_editor=self.extra_open_editor_shortcuts, color_depth=self.color_depth, + tempfile_suffix=".py", **self._extra_prompt_options()) def _make_style_from_name_or_cls(self, name_or_cls): diff --git a/IPython/terminal/pt_inputhooks/qt.py b/IPython/terminal/pt_inputhooks/qt.py index ce0a9da86bd..9f5e1b41dcd 100644 --- a/IPython/terminal/pt_inputhooks/qt.py +++ b/IPython/terminal/pt_inputhooks/qt.py @@ -25,6 +25,7 @@ def inputhook(context): 'variable. Deactivate Qt5 code.' ) return + QtCore.QCoreApplication.setAttribute(QtCore.Qt.AA_EnableHighDpiScaling) _appref = app = QtGui.QApplication([" "]) event_loop = QtCore.QEventLoop(app) @@ -44,7 +45,7 @@ def inputhook(context): QtCore.QSocketNotifier.Read) try: # connect the callback we care about before we turn it on - notifier.activated.connect(event_loop.exit) + notifier.activated.connect(lambda: event_loop.exit()) notifier.setEnabled(True) # only start the event loop we are not already flipped if not context.input_is_ready(): diff --git a/IPython/terminal/pt_inputhooks/wx.py b/IPython/terminal/pt_inputhooks/wx.py index 618f092b4f9..a0f4442c771 100644 --- a/IPython/terminal/pt_inputhooks/wx.py +++ b/IPython/terminal/pt_inputhooks/wx.py @@ -177,11 +177,13 @@ def inputhook_wxphoenix(context): # Use a wx.Timer to periodically check whether input is ready - as soon as # it is, we exit the main loop + timer = wx.Timer() + def poll(ev): if context.input_is_ready(): + timer.Stop() app.ExitMainLoop() - timer = wx.Timer() timer.Start(poll_interval) timer.Bind(wx.EVT_TIMER, poll) diff --git a/IPython/terminal/ptutils.py b/IPython/terminal/ptutils.py index 47fd6f40612..ed7ad45eb45 100644 --- a/IPython/terminal/ptutils.py +++ b/IPython/terminal/ptutils.py @@ -23,7 +23,7 @@ _completion_sentinel = object() -def _elide(string, *, min_elide=30): +def _elide_point(string:str, *, min_elide=30)->str: """ If a string is long enough, and has at least 3 dots, replace the middle part with ellipses. @@ -53,6 +53,26 @@ def _elide(string, *, min_elide=30): return string +def _elide_typed(string:str, typed:str, *, min_elide:int=30)->str: + """ + Elide the middle of a long string if the beginning has already been typed. + """ + + if len(string) < min_elide: + return string + cut_how_much = len(typed)-3 + if cut_how_much < 7: + return string + if string.startswith(typed) and len(string)> len(typed): + return f"{string[:3]}\N{HORIZONTAL ELLIPSIS}{string[cut_how_much:]}" + return string + +def _elide(string:str, typed:str, min_elide=30)->str: + return _elide_typed( + _elide_point(string, min_elide=min_elide), + typed, min_elide=min_elide) + + def _adjust_completion_text_based_on_context(text, body, offset): if text.endswith('=') and len(body) > offset and body[offset] == '=': @@ -89,7 +109,11 @@ def get_completions(self, document, complete_event): cursor_col = document.cursor_position_col cursor_position = document.cursor_position offset = cursor_to_position(body, cursor_row, cursor_col) - yield from self._get_completions(body, offset, cursor_position, self.ipy_completer) + try: + yield from self._get_completions(body, offset, cursor_position, self.ipy_completer) + except Exception as e: + from traceback import print_tb + print_tb(e) @staticmethod def _get_completions(body, offset, cursor_position, ipyc): @@ -128,9 +152,9 @@ def _get_completions(body, offset, cursor_position, ipyc): adjusted_text = _adjust_completion_text_based_on_context(c.text, body, offset) if c.type == 'function': - yield Completion(adjusted_text, start_position=c.start - offset, display=_elide(display_text+'()'), display_meta=c.type+c.signature) + yield Completion(adjusted_text, start_position=c.start - offset, display=_elide(display_text+'()', body[c.start:c.end]), display_meta=c.type+c.signature) else: - yield Completion(adjusted_text, start_position=c.start - offset, display=_elide(display_text), display_meta=c.type) + yield Completion(adjusted_text, start_position=c.start - offset, display=_elide(display_text, body[c.start:c.end]), display_meta=c.type) class IPythonPTLexer(Lexer): """ diff --git a/IPython/terminal/shortcuts.py b/IPython/terminal/shortcuts.py index e44e34277e2..a23fa091a0e 100644 --- a/IPython/terminal/shortcuts.py +++ b/IPython/terminal/shortcuts.py @@ -130,8 +130,10 @@ def newline_or_execute(event): # if all we have after the cursor is whitespace: reformat current text # before cursor after_cursor = d.text[d.cursor_position:] + reformatted = False if not after_cursor.strip(): reformat_text_before_cursor(b, d, shell) + reformatted = True if not (d.on_last_line or d.cursor_position_row >= d.line_count - d.empty_line_count_at_the_end() ): @@ -142,7 +144,8 @@ def newline_or_execute(event): return if (status != 'incomplete') and b.accept_handler: - reformat_text_before_cursor(b, d, shell) + if not reformatted: + reformat_text_before_cursor(b, d, shell) b.validate_and_handle() else: if shell.autoindent: @@ -250,7 +253,6 @@ def newline_autoindent(event): def open_input_in_editor(event): - event.app.current_buffer.tempfile_suffix = ".py" event.app.current_buffer.open_in_editor() diff --git a/IPython/terminal/tests/test_interactivshell.py b/IPython/terminal/tests/test_interactivshell.py index 6bacc8ea48a..640e5d482ae 100644 --- a/IPython/terminal/tests/test_interactivshell.py +++ b/IPython/terminal/tests/test_interactivshell.py @@ -17,14 +17,32 @@ class TestElide(unittest.TestCase): def test_elide(self): - _elide('concatenate((a1, a2, ...), axis') # do not raise - _elide('concatenate((a1, a2, ..), . axis') # do not raise - nt.assert_equal(_elide('aaaa.bbbb.ccccc.dddddd.eeeee.fffff.gggggg.hhhhhh'), 'aaaa.b…g.hhhhhh') - + _elide('concatenate((a1, a2, ...), axis', '') # do not raise + _elide('concatenate((a1, a2, ..), . axis', '') # do not raise + nt.assert_equal(_elide('aaaa.bbbb.ccccc.dddddd.eeeee.fffff.gggggg.hhhhhh',''), 'aaaa.b…g.hhhhhh') + test_string = os.sep.join(['', 10*'a', 10*'b', 10*'c', '']) expect_stirng = os.sep + 'a' + '\N{HORIZONTAL ELLIPSIS}' + 'b' + os.sep + 10*'c' - nt.assert_equal(_elide(test_string), expect_stirng) - + nt.assert_equal(_elide(test_string, ''), expect_stirng) + + def test_elide_typed_normal(self): + nt.assert_equal(_elide('the quick brown fox jumped over the lazy dog', 'the quick brown fox', min_elide=10), 'the…fox jumped over the lazy dog') + + + def test_elide_typed_short_match(self): + """ + if the match is too short we don't elide. + avoid the "the...the" + """ + nt.assert_equal(_elide('the quick brown fox jumped over the lazy dog', 'the', min_elide=10), 'the quick brown fox jumped over the lazy dog') + + def test_elide_typed_no_match(self): + """ + if the match is too short we don't elide. + avoid the "the...the" + """ + # here we typed red instead of brown + nt.assert_equal(_elide('the quick brown fox jumped over the lazy dog', 'the quick red fox', min_elide=10), 'the quick brown fox jumped over the lazy dog') class TestContextAwareCompletion(unittest.TestCase): diff --git a/IPython/tests/cve.py b/IPython/tests/cve.py new file mode 100644 index 00000000000..026415a57a4 --- /dev/null +++ b/IPython/tests/cve.py @@ -0,0 +1,56 @@ +""" +Test that CVEs stay fixed. +""" + +from IPython.utils.tempdir import TemporaryDirectory, TemporaryWorkingDirectory +from pathlib import Path +import random +import sys +import os +import string +import subprocess +import time + +def test_cve_2022_21699(): + """ + Here we test CVE-2022-21699. + + We create a temporary directory, cd into it. + Make a profile file that should not be executed and start IPython in a subprocess, + checking for the value. + + + + """ + + dangerous_profile_dir = Path('profile_default') + + dangerous_startup_dir = dangerous_profile_dir / 'startup' + dangerous_expected = 'CVE-2022-21699-'+''.join([random.choice(string.ascii_letters) for i in range(10)]) + + with TemporaryWorkingDirectory() as t: + dangerous_startup_dir.mkdir(parents=True) + (dangerous_startup_dir/ 'foo.py').write_text(f'print("{dangerous_expected}")') + # 1 sec to make sure FS is flushed. + #time.sleep(1) + cmd = [sys.executable,'-m', 'IPython'] + env = os.environ.copy() + env['IPY_TEST_SIMPLE_PROMPT'] = '1' + + + # First we fake old behavior, making sure the profile is/was actually dangerous + p_dangerous = subprocess.Popen(cmd + [f'--profile-dir={dangerous_profile_dir}'], env=env, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out_dangerous, err_dangerouns = p_dangerous.communicate(b"exit\r") + assert dangerous_expected in out_dangerous.decode() + + # Now that we know it _would_ have been dangerous, we test it's not loaded + p = subprocess.Popen(cmd, env=env, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate(b"exit\r") + assert b'IPython' in out + assert dangerous_expected not in out.decode() + assert err == b'' + + + diff --git a/IPython/utils/_process_posix.py b/IPython/utils/_process_posix.py index f3f93f774d6..a11cad7697c 100644 --- a/IPython/utils/_process_posix.py +++ b/IPython/utils/_process_posix.py @@ -59,11 +59,12 @@ class ProcessHandler(object): @property def sh(self): - if self._sh is None: - self._sh = pexpect.which('sh') + if self._sh is None: + shell_name = os.environ.get("SHELL", "sh") + self._sh = pexpect.which(shell_name) if self._sh is None: - raise OSError('"sh" shell not found') - + raise OSError('"{}" shell not found'.format(shell_name)) + return self._sh def __init__(self, logfile=None, read_timeout=None, terminate_timeout=None): diff --git a/appveyor.yml b/appveyor.yml index b79c7f26ff4..d20effdf815 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -4,13 +4,6 @@ matrix: environment: matrix: - - PYTHON: "C:\\Python36" - PYTHON_VERSION: "3.6.x" - PYTHON_ARCH: "32" - - - PYTHON: "C:\\Python36-x64" - PYTHON_VERSION: "3.6.x" - PYTHON_ARCH: "64" - PYTHON: "C:\\Python37-x64" PYTHON_VERSION: "3.7.x" diff --git a/docs/source/conf.py b/docs/source/conf.py index afdff72b5c4..5012da2694e 100755 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -121,6 +121,29 @@ def is_stable(extra): numpydoc_class_members_toctree = False warning_is_error = True +import logging + +class ConfigtraitFilter(logging.Filter): + """ + This is a filter to remove in sphinx 3+ the error about config traits being duplicated. + + As we autogenerate configuration traits from, subclasses have lots of + duplication and we want to silence them. Indeed we build on travis with + warnings-as-error set to True, so those duplicate items make the build fail. + """ + + def filter(self, record): + if record.args and record.args[0] == 'configtrait' and 'duplicate' in record.msg: + return False + return True + +ct_filter = ConfigtraitFilter() + +import sphinx.util +logger = sphinx.util.logging.getLogger('sphinx.domains.std').logger + +logger.addFilter(ct_filter) + # The default replacements for |version| and |release|, also used in various # other places throughout the built documents. # @@ -223,7 +246,7 @@ def is_stable(extra): htmlhelp_basename = 'ipythondoc' intersphinx_mapping = {'python': ('https://docs.python.org/3/', None), - 'rpy2': ('https://rpy2.readthedocs.io/en/latest/', None), + 'rpy2': ('https://rpy2.github.io/doc/latest/html/', None), 'jupyterclient': ('https://jupyter-client.readthedocs.io/en/latest/', None), 'jupyter': ('https://jupyter.readthedocs.io/en/latest/', None), 'jedi': ('https://jedi.readthedocs.io/en/latest/', None), diff --git a/docs/source/config/details.rst b/docs/source/config/details.rst index 6685b14d8f6..9e63232d81d 100644 --- a/docs/source/config/details.rst +++ b/docs/source/config/details.rst @@ -247,7 +247,7 @@ VI input mode to ``Normal`` when in insert mode:: For more information on filters and what you can do with the ``event`` object, `see the prompt_toolkit docs -`__. +`__. Enter to execute diff --git a/docs/source/config/intro.rst b/docs/source/config/intro.rst index 115315c9ddb..6dccde1348c 100644 --- a/docs/source/config/intro.rst +++ b/docs/source/config/intro.rst @@ -11,48 +11,49 @@ Many of IPython's classes have configurable attributes (see :doc:`options/index` for the list). These can be configured in several ways. -Python config files -------------------- +Python configuration files +-------------------------- -To create the blank config files, run:: +To create the blank configuration files, run:: ipython profile create [profilename] If you leave out the profile name, the files will be created for the -``default`` profile (see :ref:`profiles`). These will typically be -located in :file:`~/.ipython/profile_default/`, and will be named -:file:`ipython_config.py`, :file:`ipython_notebook_config.py`, etc. -The settings in :file:`ipython_config.py` apply to all IPython commands. +``default`` profile (see :ref:`profiles`). These will typically be located in +:file:`~/.ipython/profile_default/`, and will be named +:file:`ipython_config.py`, for historical reasons you may also find files +named with IPython prefix instead of Jupyter: +:file:`ipython_notebook_config.py`, etc. The settings in +:file:`ipython_config.py` apply to all IPython commands. -The files typically start by getting the root config object:: - - c = get_config() +By default, configuration files are fully featured Python scripts that can +execute arbitrary code, the main usage is to set value on the configuration +object ``c`` which exist in your configuration file. You can then configure class attributes like this:: c.InteractiveShell.automagic = False Be careful with spelling--incorrect names will simply be ignored, with -no error. +no error. -To add to a collection which may have already been defined elsewhere, -you can use methods like those found on lists, dicts and sets: append, -extend, :meth:`~traitlets.config.LazyConfigValue.prepend` (like -extend, but at the front), add and update (which works both for dicts -and sets):: +To add to a collection which may have already been defined elsewhere or have +default values, you can use methods like those found on lists, dicts and +sets: append, extend, :meth:`~traitlets.config.LazyConfigValue.prepend` (like +extend, but at the front), add and update (which works both for dicts and +sets):: c.InteractiveShellApp.extensions.append('Cython') .. versionadded:: 2.0 list, dict and set methods for config values -Example config file -``````````````````` +Example configuration file +`````````````````````````` :: # sample ipython_config.py - c = get_config() c.TerminalIPythonApp.display_banner = True c.InteractiveShellApp.log_level = 20 @@ -78,6 +79,38 @@ Example config file ('la', 'ls -al') ] +JSON Configuration files +------------------------ + +In case where executability of configuration can be problematic, or +configurations need to be modified programmatically, IPython also support a +limited set of functionalities via ``.json`` configuration files. + +You can defined most of the configuration options via a json object which +hierarchy represent the value you would normally set on the ``c`` object of +``.py`` configuration files. The following ``ipython_config.json`` file:: + + { + "InteractiveShell": { + "colors": "LightBG", + "editor": "nano" + }, + "InteractiveShellApp": { + "extensions": [ + "myextension" + ] + } + } + +Is equivalent to the following ``ipython_config.py``:: + + c.InteractiveShellApp.extensions = [ + 'myextension' + ] + + c.InteractiveShell.colors = 'LightBG' + c.InteractiveShell.editor = 'nano' + Command line arguments ---------------------- @@ -94,7 +127,7 @@ Many frequently used options have short aliases and flags, such as To see all of these abbreviated options, run:: ipython --help - ipython notebook --help + jupyter notebook --help # etc. Options specified at the command line, in either format, override @@ -163,3 +196,38 @@ the directory :file:`~/.ipython/` by default. To see where IPython is looking for the IPython directory, use the command ``ipython locate``, or the Python function :func:`IPython.paths.get_ipython_dir`. + + +Systemwide configuration +======================== + +It can be useful to deploy systemwide ipython or ipykernel configuration +when managing environment for many users. At startup time IPython and +IPykernel will search for configuration file in multiple systemwide +locations, mainly: + + - ``/etc/ipython/`` + - ``/usr/local/etc/ipython/`` + +When the global install is a standalone python distribution it may also +search in distribution specific location, for example: + + - ``$ANACONDA_LOCATION/etc/ipython/`` + +In those locations, Terminal IPython will look for a file called +``ipython_config.py`` and ``ipython_config.json``, ipykernel will look for +``ipython_kernel_config.py`` and ``ipython_kernel.json``. + +Configuration files are loaded in order and merged with configuration on +later location taking precedence on earlier locations (that is to say a user +can overwrite a systemwide configuration option). + +You can see all locations in which IPython is looking for configuration files +by starting ipython in debug mode:: + + $ ipython --debug -c 'exit()' + +Identically with ipykernel though the command is currently blocking until +this process is killed with ``Ctrl-\``:: + + $ python -m ipykernel --debug diff --git a/docs/source/coredev/index.rst b/docs/source/coredev/index.rst index 60f1cb0fdfd..ee1eadb9b1e 100644 --- a/docs/source/coredev/index.rst +++ b/docs/source/coredev/index.rst @@ -80,6 +80,13 @@ for the release you are actually making:: VERSION=5.0.0 BRANCH=master +For `reproducibility of builds `_, +we recommend setting ``SOURCE_DATE_EPOCH`` prior to running the build; record the used value +of ``SOURCE_DATE_EPOCH`` as it may not be available from build artifact. You +should be able to use ``date +%s`` to get a formatted timestamp:: + + SOURCE_DATE_EPOCH=$(date +%s) + 2. Create GitHub stats and finish release note ---------------------------------------------- @@ -229,6 +236,16 @@ uploading them to PyPI. We do not use an universal wheel as each wheel installs an ``ipython2`` or ``ipython3`` script, depending on the version of Python it is built for. Using an universal wheel would prevent this. +Check the shasum of files with:: + + shasum -a 256 dist/* + +and takes notes of them you might need them to update the conda-forge recipes. +Rerun the command and check the hash have not changed:: + + ./tools/release + shasum -a 256 dist/* + Use the following to actually upload the result of the build:: ./tools/release upload diff --git a/docs/source/development/wrapperkernels.rst b/docs/source/development/wrapperkernels.rst index eb0a0488807..d734c30ee99 100644 --- a/docs/source/development/wrapperkernels.rst +++ b/docs/source/development/wrapperkernels.rst @@ -116,7 +116,7 @@ You can override a number of other methods to improve the functionality of your kernel. All of these methods should return a dictionary as described in the relevant section of the :doc:`messaging spec `. -.. class:: MyKernel +.. class:: MyBetterKernel .. method:: do_complete(code, cusor_pos) diff --git a/docs/source/interactive/autoawait.rst b/docs/source/interactive/autoawait.rst index f87379d2ca0..e4ed965a13e 100644 --- a/docs/source/interactive/autoawait.rst +++ b/docs/source/interactive/autoawait.rst @@ -8,7 +8,7 @@ Asynchronous in REPL: Autoawait This feature is experimental and behavior can change between python and IPython version without prior deprecation. -Starting with IPython 7.0, and when user Python 3.6 and above, IPython offer the +Starting with IPython 7.0, and when using Python 3.6 and above, IPython offer the ability to run asynchronous code from the REPL. Constructs which are :exc:`SyntaxError` s in the Python REPL can be used seamlessly in IPython. @@ -19,7 +19,7 @@ will differ between IPython, IPykernel and their versions. When a supported library is used, IPython will automatically allow Futures and Coroutines in the REPL to be ``await`` ed. This will happen if an :ref:`await -` (or any other async constructs like async-with, async-for) is use at +` (or any other async constructs like async-with, async-for) is used at top level scope, or if any structure valid only in `async def `_ function context are present. For example, the following being a syntax error in the @@ -73,7 +73,7 @@ By default IPython will assume integration with Python's provided :mod:`asyncio`, but integration with other libraries is provided. In particular we provide experimental integration with the ``curio`` and ``trio`` library. -You can switch current integration by using the +You can switch the current integration by using the ``c.InteractiveShell.loop_runner`` option or the ``autoawait `` magic. @@ -118,7 +118,7 @@ to your code. When using command line IPython, the default loop (or runner) does not process in the background, so top level asynchronous code must finish for the REPL to -allow you to enter more code. As with usual Python semantic, the awaitables are +allow you to enter more code. As with usual Python semantics, the awaitables are started only when awaited for the first time. That is to say, in first example, no network request is done between ``In[1]`` and ``In[2]``. @@ -131,8 +131,8 @@ a loop to run. By default IPython will use a fake coroutine runner which should allow ``IPython.embed()`` to be nested. Though this will prevent usage of the :magic:`%autoawait` feature when using IPython embed. -You can set explicitly a coroutine runner for ``embed()`` if you desire to run -asynchronous code, the exact behavior is though undefined. +You can set a coroutine runner explicitly for ``embed()`` if you want to run +asynchronous code, though the exact behavior is undefined. Effects on Magics ----------------- @@ -140,14 +140,14 @@ Effects on Magics A couple of magics (``%%timeit``, ``%timeit``, ``%%time``, ``%%prun``) have not yet been updated to work with asynchronous code and will raise syntax errors when trying to use top-level ``await``. We welcome any contribution to help fix -those, and extra cases we haven't caught yet. We hope for better support in Cor +those, and extra cases we haven't caught yet. We hope for better support in Core Python for top-level Async code. Internals --------- As running asynchronous code is not supported in interactive REPL (as of Python -3.7) we have to rely to a number of complex workaround and heuristic to allow +3.7) we have to rely to a number of complex workarounds and heuristics to allow this to happen. It is interesting to understand how this works in order to comprehend potential bugs, or provide a custom runner. @@ -179,16 +179,16 @@ significant overhead to this kind of code. By default the generated coroutine function will be consumed by Asyncio's ``loop_runner = asyncio.get_evenloop().run_until_complete()`` method if ``async`` mode is deemed necessary, otherwise the coroutine will just be -exhausted in a simple runner. It is though possible to change the default +exhausted in a simple runner. It is possible, though, to change the default runner. A loop runner is a *synchronous* function responsible from running a coroutine object. -The runner is responsible from ensuring that ``coroutine`` run to completion, -and should return the result of executing the coroutine. Let's write a +The runner is responsible for ensuring that ``coroutine`` runs to completion, +and it should return the result of executing the coroutine. Let's write a runner for ``trio`` that print a message when used as an exercise, ``trio`` is -special as it usually prefer to run a function object and make a coroutine by +special as it usually prefers to run a function object and make a coroutine by itself, we can get around this limitation by wrapping it in an async-def without parameters and passing this value to ``trio``:: @@ -246,8 +246,8 @@ Difference between terminal IPython and IPykernel The exact asynchronous code running behavior varies between Terminal IPython and IPykernel. The root cause of this behavior is due to IPykernel having a *persistent* `asyncio` loop running, while Terminal IPython starts and stops a -loop for each code block. This can lead to surprising behavior in some case if -you are used to manipulate asyncio loop yourself, see for example +loop for each code block. This can lead to surprising behavior in some cases if +you are used to manipulating asyncio loop yourself, see for example :ghissue:`11303` for a longer discussion but here are some of the astonishing cases. diff --git a/docs/source/whatsnew/development.rst b/docs/source/whatsnew/development.rst index 9d4493a3a53..9e9b2984c6c 100644 --- a/docs/source/whatsnew/development.rst +++ b/docs/source/whatsnew/development.rst @@ -22,8 +22,12 @@ Need to be updated: pr/* + + .. DO NOT EDIT THIS LINE BEFORE RELEASE. FEATURE INSERTION POINT. +As a reminder, IPython master has diverged from the 7.x branch, thus master may +have more feature and API changes. Backwards incompatible changes ------------------------------ diff --git a/docs/source/whatsnew/github-stats-7.rst b/docs/source/whatsnew/github-stats-7.rst index 91955bb2afa..6bc36c0789c 100644 --- a/docs/source/whatsnew/github-stats-7.rst +++ b/docs/source/whatsnew/github-stats-7.rst @@ -1,9 +1,95 @@ Issues closed in the 7.x development cycle ========================================== -Issues closed in 8.12 + +Issues closed in 7.16 +--------------------- + +GitHub stats for 2020/05/29 - 2020/06/26 (tag: 7.15.0) + +These lists are automatically generated, and may be incomplete or contain duplicates. + +We closed 0 issues and merged 18 pull requests. +The full list can be seen `on GitHub `__ + +The following 7 authors contributed 22 commits. + +* Benjamin Ragan-Kelley +* dalthviz +* Frank Tobia +* Matthias Bussonnier +* palewire +* Paul McCarthy +* Talley Lambert + + +Issues closed in 7.15 +--------------------- + +GitHub stats for 2020/05/01 - 2020/05/29 (tag: 7.14.0) + +These lists are automatically generated, and may be incomplete or contain duplicates. + +We closed 1 issues and merged 29 pull requests. +The full list can be seen `on GitHub `__ + +The following 6 authors contributed 31 commits. + +* Blake Griffin +* Inception95 +* Marcio Mazza +* Matthias Bussonnier +* Talley Lambert +* Thomas + +Issues closed in 7.14 --------------------- +GitHub stats for 2020/02/29 - 2020/05/01 (tag: 7.13.0) + +These lists are automatically generated, and may be incomplete or contain duplicates. + +We closed 0 issues and merged 30 pull requests. +The full list can be seen `on GitHub `__ + +The following 10 authors contributed 47 commits. + +* Eric Wieser +* foobarbyte +* Ian Castleden +* Itamar Turner-Trauring +* Lumir Balhar +* Markus Wageringel +* Matthias Bussonnier +* Matthieu Ancellin +* Quentin Peter +* Theo Ouzhinski + +Issues closed in 7.13 +--------------------- + +GitHub stats for 2020/02/29 - 2020/05/01 (tag: 7.13.0) + +These lists are automatically generated, and may be incomplete or contain duplicates. + +We closed 0 issues and merged 30 pull requests. +The full list can be seen `on GitHub `__ + +The following 10 authors contributed 47 commits. + +* Eric Wieser +* foobarbyte +* Ian Castleden +* Itamar Turner-Trauring +* Lumir Balhar +* Markus Wageringel +* Matthias Bussonnier +* Matthieu Ancellin +* Quentin Peter +* Theo Ouzhinski + +Issues closed in 7.13 +--------------------- GitHub stats for 2020/02/01 - 2020/02/28 (tag: 7.12.0) @@ -27,6 +113,7 @@ The following 12 authors contributed 108 commits. * Nathan Goldbaum * Terry Davis + Issues closed in 7.12 --------------------- diff --git a/docs/source/whatsnew/version7.rst b/docs/source/whatsnew/version7.rst index f54f0e5ef0c..f8c88ab844b 100644 --- a/docs/source/whatsnew/version7.rst +++ b/docs/source/whatsnew/version7.rst @@ -2,14 +2,234 @@ 7.x Series ============ +======= +.. _version 7.16.3: + +IPython 7.16.3 (CVE-2022-21699) +=============================== + +Fixed CVE-2022-21699, see IPython 8.0.1 release notes for informations. + +.. _version 716: + +IPython 7.16.1, 7.16.2 +====================== + +IPython 7.16.1 was release immediately after 7.16.0 to fix a conda packaging issue. +The source is identical to 7.16.0 but the file permissions in the tar are different. + +IPython 7.16.2 pins jedi dependency to "<=0.17.2" which should prevent some +issues for users still on python 3.6. This may not be sufficient as pip may +still allow to downgrade IPython. + +Compatibility with Jedi > 0.17.2 was not added as this would have meant bumping +the minimal version to >0.16. + +IPython 7.16 +============ + + +The default traceback mode will now skip frames that are marked with +``__tracebackhide__ = True`` and show how many traceback frames have been +skipped. This can be toggled by using :magic:`xmode` with the ``--show`` or +``--hide`` attribute. It will have no effect on non verbose traceback modes. + +The ipython debugger also now understands ``__tracebackhide__`` as well and will +skip hidden frames when displaying. Movement up and down the stack will skip the +hidden frames and will show how many frames were hidden. Internal IPython frames +are also now hidden by default. The behavior can be changed with the +``skip_hidden`` while in the debugger, command and accepts "yes", "no", "true" +and "false" case insensitive parameters. + + +Misc Noticeable changes: +------------------------ + +- Exceptions are now (re)raised when running notebooks via the :magic:`%run`, helping to catch issues in workflows and + pipelines. :ghpull:`12301` +- Fix inputhook for qt 5.15.0 :ghpull:`12355` +- Fix wx inputhook :ghpull:`12375` +- Add handling for malformed pathext env var (Windows) :ghpull:`12367` +- use $SHELL in system_piped :ghpull:`12360` for uniform behavior with + ipykernel. + +Reproducible Build +------------------ + +IPython 7.15 reproducible build did not work, so we try again this month +:ghpull:`12358`. + + +API Changes +----------- + +Change of API and exposed objects automatically detected using `frappuccino +`_ (still in beta): + + +The following items are new and mostly related to understanding ``__tracebackbhide__``:: + + + IPython.core.debugger.Pdb.do_down(self, arg) + + IPython.core.debugger.Pdb.do_skip_hidden(self, arg) + + IPython.core.debugger.Pdb.do_up(self, arg) + + IPython.core.debugger.Pdb.hidden_frames(self, stack) + + IPython.core.debugger.Pdb.stop_here(self, frame) + + +The following items have been removed:: + + - IPython.core.debugger.Pdb.new_do_down + - IPython.core.debugger.Pdb.new_do_up + +Those were implementation details. + + +.. _version 715: + +IPython 7.15 +============ + +IPython 7.15 brings a number of bug fixes and user facing improvements. + +Misc Noticeable changes: +------------------------ + + - Long completion name have better elision in terminal :ghpull:`12284` + - I've started to test on Python 3.9 :ghpull:`12307` and fix some errors. + - Hi DPI scaling of figures when using qt eventloop :ghpull:`12314` + - Document the ability to have systemwide configuration for IPython. + :ghpull:`12328` + - Fix issues with input autoformatting :ghpull:`12336` + - ``IPython.core.debugger.Pdb`` is now interruptible (:ghpull:`12168`, in 7.14 + but forgotten in release notes) + - Video HTML attributes (:ghpull:`12212`, in 7.14 but forgotten in release + notes) + +Reproducible Build +------------------ + +Starting with IPython 7.15, I am attempting to provide reproducible builds, +that is to say you should be able from the source tree to generate an sdist +and wheel that are identical byte for byte with the publish version on PyPI. + +I've only tested on a couple of machines so far and the process is relatively +straightforward, so this mean that IPython not only have a deterministic build +process, but also I have either removed, or put under control all effects of +the build environments on the final artifact. I encourage you to attempt the +build process on your machine as documented in :ref:`core_developer_guide` +and let me know if you do not obtain an identical artifact. + +While reproducible builds is critical to check that the supply chain of (open +source) software has not been compromised, it can also help to speedup many +of the build processes in large environment (conda, apt...) by allowing +better caching of intermediate build steps. + +Learn more on ``_. `Reflections on trusting +trust `_ is also one of the +cornerstone and recommended reads on this subject. + +.. note:: + + The build commit from which the sdist is generated is also `signed + `_, so you should be able to + check it has not been compromised, and the git repository is a `merkle-tree + `_, you can check the consistency + with `git-fsck `_ which you likely `want + to enable by default + `_. + +NEP29: Last version to support Python 3.6 +----------------------------------------- + +IPython 7.15 will be the Last IPython version to officially support Python +3.6, as stated by `NumPy Enhancement Proposal 29 +`_. Starting with +next minor version of IPython I may stop testing on Python 3.6 and may stop +publishing release artifacts that install on Python 3.6 + +Highlighted features +-------------------- + +Highlighted features are not new, but seem to not be widely known, this +section will help you discover in more narrative form what you can do with +IPython. + +Increase Tab Completion Menu Height +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In terminal IPython it is possible to increase the hight of the tab-completion +menu. To do so set the value of +:configtrait:`TerminalInteractiveShell.space_for_menu`, this will reserve more +space at the bottom of the screen for various kind of menus in IPython including +tab completion and searching in history. + +Autoformat Code in the terminal +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you have a preferred code formatter, you can configure IPython to +reformat your code. Set the value of +:configtrait:`TerminalInteractiveShell.autoformatter` to for example ``'black'`` +and IPython will auto format your code when possible. + + +.. _version 714: + +IPython 7.14 +============ + +IPython 7.14 is a minor release that fix a couple of bugs and prepare +compatibility with new or future versions of some libraries. + +Important changes: +------------------ + + - Fix compatibility with Sphinx 3+ :ghpull:`12235` + - Remove deprecated matplotlib parameter usage, compatibility with matplotlib + 3.3+ :`122250` + +Misc Changes +------------ + + - set ``.py`` extension when editing current buffer in vi/emacs. :ghpull:`12167` + - support for unicode identifiers in ``?``/``??`` :ghpull:`12208` + - add extra options to the ``Video`` Rich objects :ghpull:`12212` + - add pretty-printing to ``SimpleNamespace`` :ghpull:`12230` + +IPython.core.debugger.Pdb is now interruptible +---------------------------------------------- + +A ``KeyboardInterrupt`` will now interrupt IPython's extended debugger, in order to make Jupyter able to interrupt it. (:ghpull:`12168`) + +Video HTML attributes +--------------------- + +Add an option to `IPython.display.Video` to change the attributes of the HTML display of the video (:ghpull:`12212`) + + +Pending deprecated imports +-------------------------- + +Many object present in ``IPython.core.display`` are there for internal use only, +and should already been imported from ``IPython.display`` by users and external +libraries. Trying to import those from ``IPython.core.display`` is still possible +but will trigger a +deprecation warning in later versions of IPython and will become errors in the +future. + +This will simplify compatibility with other Python kernels (like Xeus-Python), +and simplify code base. + + + + .. _version 713: IPython 7.13 ============ -IPython 7.13 is the first release of the 7.x branch since master is diverging +IPython 7.13 is the final release of the 7.x branch since master is diverging toward an 8.0. Exiting new features have already been merged in 8.0 and will -not be available on the 7.x branch. All the changes bellow have been backported +not be available on the 7.x branch. All the changes below have been backported from the master branch. diff --git a/docs/sphinxext/configtraits.py b/docs/sphinxext/configtraits.py index 4e767694097..2b05d2bf6c8 100644 --- a/docs/sphinxext/configtraits.py +++ b/docs/sphinxext/configtraits.py @@ -8,8 +8,7 @@ Cross reference like this: :configtrait:`Application.log_datefmt`. """ -from sphinx.locale import l_ -from sphinx.util.docfields import Field + def setup(app): app.add_object_type('configtrait', 'configtrait', objname='Config option') diff --git a/setup.py b/setup.py index 593e3a6a0af..3e734bafe14 100755 --- a/setup.py +++ b/setup.py @@ -186,7 +186,7 @@ install_requires = [ 'setuptools>=18.5', - 'jedi>=0.10', + 'jedi>=0.10,<=0.17.2', 'decorator', 'pickleshare', 'traitlets>=4.2', @@ -225,7 +225,7 @@ for key, deps in extras_require.items(): if ':' not in key: everything.update(deps) -extras_require['all'] = everything +extras_require['all'] = list(sorted(everything)) if 'setuptools' in sys.modules: setuptools_extra_args['python_requires'] = '>=3.6' diff --git a/tools/build_release b/tools/build_release index 26dc9ec874a..51fd87d54d9 100755 --- a/tools/build_release +++ b/tools/build_release @@ -2,6 +2,7 @@ """IPython release build script. """ import os +import sys from shutil import rmtree from toollib import sh, pjoin, get_ipdir, cd, sdists, buildwheels @@ -12,15 +13,10 @@ def build_release(): ipdir = get_ipdir() cd(ipdir) - # Cleanup - for d in ['build', 'dist', pjoin('docs', 'build'), pjoin('docs', 'dist'), - pjoin('docs', 'source', 'api', 'generated')]: - if os.path.isdir(d): - rmtree(d) - # Build source and binary distros sh(sdists) buildwheels() + sh(' '.join([sys.executable, 'tools/retar.py', 'dist/*.gz'])) if __name__ == '__main__': build_release() diff --git a/tools/make_tarball.py b/tools/make_tarball.py index bdce25ba804..fb639f61f61 100755 --- a/tools/make_tarball.py +++ b/tools/make_tarball.py @@ -3,7 +3,6 @@ """ import subprocess -import os from toollib import cd, sh diff --git a/tools/release b/tools/release index 5c8686b3145..2de8e120070 100755 --- a/tools/release +++ b/tools/release @@ -81,13 +81,10 @@ else: sh('mv ipython-*.tgz %s' % ipbackupdir) # Build release files - sh('./build_release %s' % ipdir) + sh('./build_release') cd(ipdir) - # Upload all files - sh(sdists) - buildwheels() print("`./release upload` to upload source distribution on PyPI and ipython archive") sys.exit(0) diff --git a/tools/release_helper.sh b/tools/release_helper.sh index 7489c16d756..cf053d3459a 100644 --- a/tools/release_helper.sh +++ b/tools/release_helper.sh @@ -7,7 +7,7 @@ python -c 'import keyring' python -c 'import twine' python -c 'import sphinx' python -c 'import sphinx_rtd_theme' -python -c 'import nose' +python -c 'import pytest' BLACK=$(tput setaf 1) @@ -21,6 +21,7 @@ WHITE=$(tput setaf 7) NOR=$(tput sgr0) +echo "Will use '$EDITOR' to edit files when necessary" echo -n "PREV_RELEASE (X.y.z) [$PREV_RELEASE]: " read input PREV_RELEASE=${input:-$PREV_RELEASE} @@ -38,18 +39,40 @@ ask_section(){ echo echo $BLUE"$1"$NOR echo -n $GREEN"Press Enter to continue, S to skip: "$NOR - read -n1 value - echo - if [ -z $value ] || [ $value = 'y' ] ; then + if [ "$ZSH_NAME" = "zsh" ] ; then + read -k1 value + value=${value%$'\n'} + else + read -n1 value + fi + if [ -z "$value" ] || [ $value = 'y' ]; then return 0 fi return 1 } +maybe_edit(){ + echo + echo $BLUE"$1"$NOR + echo -n $GREEN"Press e to Edit $1, any other keys to skip: "$NOR + if [ "$ZSH_NAME" = "zsh" ] ; then + read -k1 value + value=${value%$'\n'} + else + read -n1 value + fi + + echo + if [ $value = 'e' ] ; then + $=EDITOR $1 + fi +} + + echo -if ask_section "Updating what's new with informations from docs/source/whatsnew/pr" +if ask_section "Updating what's new with information from docs/source/whatsnew/pr" then python tools/update_whatsnew.py @@ -81,10 +104,30 @@ then fi +if ask_section "Generate API difference (using frapuccino)" +then + echo $BLUE"Checking out $PREV_RELEASE"$NOR + git checkout $PREV_RELEASE + echo $BLUE"Saving API to file $PREV_RELEASE"$NOR + frappuccino IPython --save IPython-$PREV_RELEASE.json + echo $BLUE"comming back to $BRANCH"$NOR + git checkout $BRANCH + echo $BLUE"comparing ..."$NOR + frappuccino IPython --compare IPython-$PREV_RELEASE.json + echo $GREEN"Use the above guideline to write an API changelog ..."$NOR + echo $GREEN"Press any keys to continue"$NOR + read +fi + echo "Cleaning repository" git clean -xfdi echo $GREEN"please update version number in ${RED}IPython/core/release.py${NOR} , Do not commit yet – we'll do it later."$NOR +echo $GREEN"I tried ${RED}sed -i bkp -e '/Uncomment/s/^# //g' IPython/core/release.py${NOR}" +sed -i bkp -e '/Uncomment/s/^# //g' IPython/core/release.py +rm IPython/core/release.pybkp +git diff | cat +maybe_edit IPython/core/release.py echo $GREEN"Press enter to continue"$NOR read @@ -98,12 +141,6 @@ then fi -echo -echo $BLUE"Attempting to build package..."$NOR - -tools/build_release -rm dist/* - if ask_section "Should we commit, tag, push... etc ? " then echo @@ -133,7 +170,14 @@ then echo $GREEN"please update version number and back to .dev in ${RED}IPython/core/release.py" + echo $GREEN"I tried ${RED}sed -i bkp -e '/Uncomment/s/^/# /g' IPython/core/release.py${NOR}" + sed -i bkp -e '/Uncomment/s/^/# /g' IPython/core/release.py + rm IPython/core/release.pybkp + git diff | cat + echo $GREEN"Please bump ${RED}the minor version number${NOR}" + maybe_edit IPython/core/release.py echo ${BLUE}"Do not commit yet – we'll do it later."$NOR + echo $GREEN"Press enter to continue"$NOR read @@ -160,14 +204,41 @@ fi if ask_section "Should we build and release ?" then + + echo $BLUE"going to set SOURCE_DATE_EPOCH"$NOR + echo $BLUE'export SOURCE_DATE_EPOCH=$(git show -s --format=%ct HEAD)'$NOR + echo $GREEN"Press enter to continue"$NOR + read + + export SOURCE_DATE_EPOCH=$(git show -s --format=%ct HEAD) + + echo $BLUE"SOURCE_DATE_EPOCH set to $SOURCE_DATE_EPOCH"$NOR + echo $GREEN"Press enter to continue"$NOR + read + + + + echo + echo $BLUE"Attempting to build package..."$NOR + + tools/release + + + echo $RED'$ shasum -a 256 dist/*' + shasum -a 256 dist/* + echo $NOR + + echo $BLUE"We are going to rebuild, node the hash above, and compare them to the rebuild"$NOR + echo $GREEN"Press enter to continue"$NOR + read echo echo $BLUE"Attempting to build package..."$NOR tools/release - echo $RED - echo '$ shasum -a 256 dist/*' + echo $RED"Check the shasum for SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH" + echo $RED'$ shasum -a 256 dist/*' shasum -a 256 dist/* echo $NOR diff --git a/tools/retar.py b/tools/retar.py new file mode 100644 index 00000000000..efd8c0c6bbd --- /dev/null +++ b/tools/retar.py @@ -0,0 +1,65 @@ +""" +Un-targz and retargz a targz file to ensure reproducible build. + +usage: + + $ export SOURCE_DATE_EPOCH=$(date +%s) + ... + $ python retar.py + +The process of creating an sdist can be non-reproducible: + - directory created during the process get a mtime of the creation date; + - gziping files embed the timestamp of fo zip creation. + +This will untar-retar; ensuring that all mtime > SOURCE_DATE_EPOCH will be set +equal to SOURCE_DATE_EPOCH. + +""" + +import tarfile +import sys +import os +import gzip +import io + +if len(sys.argv) > 2: + raise ValueError("Too many arguments") + + +timestamp = int(os.environ["SOURCE_DATE_EPOCH"]) + +old_buf = io.BytesIO() +with open(sys.argv[1], "rb") as f: + old_buf.write(f.read()) +old_buf.seek(0) +old = tarfile.open(fileobj=old_buf, mode="r:gz") + +buf = io.BytesIO() +new = tarfile.open(fileobj=buf, mode="w", format=tarfile.GNU_FORMAT) +for i, m in enumerate(old): + data = None + # mutation does not work, copy + if m.name.endswith('.DS_Store'): + continue + m2 = tarfile.TarInfo(m.name) + m2.mtime = min(timestamp, m.mtime) + m2.size = m.size + m2.type = m.type + m2.linkname = m.linkname + m2.mode = m.mode + if m.isdir(): + new.addfile(m2) + else: + data = old.extractfile(m) + new.addfile(m2, data) +new.close() +old.close() + +buf.seek(0) +with open(sys.argv[1], "wb") as f: + with gzip.GzipFile('', "wb", fileobj=f, mtime=timestamp) as gzf: + gzf.write(buf.read()) + +# checks the archive is valid. +archive = tarfile.open(sys.argv[1]) +names = archive.getnames()