generator raised StopIteration in python 3.7 #5870
Description
Subject: generator raised StopIteration in python 3.7
Problem
- Encountering what appears to be an internal Sphinx error during the course of the normal development.
The rst text is the following:
Types
=========
.. module:: dagster.core.types
Dagster type system.
Type definitions
-----------------
.. autoclass:: DagsterBuiltinScalarType
:members: is_python_valid_value
.. autoclass:: DagsterScalarType
:members: is_python_valid_value
.. autoclass:: DagsterStringType
:members: is_python_valid_value
.. autoclass:: DagsterType
:members:
.. autofunction:: Dict
.. autoclass:: PythonObjectType
.. autodata:: Any
.. autofunction:: Nullable
.. autofunction:: List
.. .. autodata:: String
..
.. .. autodata:: Path
..
.. autodata:: Int
..
.. .. autodata:: Bool
Utilities
------------
.. autoclass:: Configurable
:members:
.. autoclass:: ConfigurableFromAny
:members:
.. autoclass:: ConfigurableFromList
:members:
.. autoclass:: ConfigurableFromNullable
:members:
.. autoclass:: ConfigurableFromScalar
:members:
.. autoclass:: ConfigurableObjectFromDict
:members:
.. autoclass:: ConfigurableSelectorFromDict
:members:
.. autoclass:: Field
:members:
The code it is trying to introspect is a locally installed module. dagster/core.types.py:
from collections import namedtuple
import json
import os
import pickle
from six import integer_types, string_types
from dagster import check
from dagster.core.errors import (
DagsterRuntimeCoercionError,
)
from .configurable import (
Configurable,
ConfigurableFromAny,
ConfigurableFromList,
ConfigurableSelectorFromDict,
ConfigurableObjectFromDict,
ConfigurableFromScalar,
ConfigurableFromNullable,
Field,
)
from .materializable import MaterializeableBuiltinScalar
SerializedTypeValue = namedtuple('SerializedTypeValue', 'name value')
class DagsterTypeAttributes(
namedtuple(
'_DagsterTypeAttributes',
'is_builtin is_system_config is_named',
)
):
def __new__(cls, is_builtin=False, is_system_config=False, is_named=True):
return super(DagsterTypeAttributes, cls).__new__(
cls,
is_builtin=check.bool_param(is_builtin, 'is_builtin'),
is_system_config=check.bool_param(is_system_config, 'is_system_config'),
is_named=check.bool_param(is_named, 'is_named')
)
DEFAULT_TYPE_ATTRIBUTES = DagsterTypeAttributes()
class DagsterType(object):
'''Base class for Dagster Type system. Should be inherited by a subclass.
Subclass must implement `evaluate_value`
Attributes:
name (str): Name of the type
description (str): Description of the type
'''
def __init__(self, name, type_attributes=DEFAULT_TYPE_ATTRIBUTES, description=None):
self.name = check.str_param(name, 'name')
self.description = check.opt_str_param(description, 'description')
self.type_attributes = check.inst_param(
type_attributes,
'type_attributes',
DagsterTypeAttributes,
)
self.__doc__ = description
@property
def is_any(self):
return isinstance(self, _DagsterAnyType)
@property
def is_configurable(self):
return isinstance(self, Configurable)
@property
def is_system_config(self):
return self.type_attributes.is_system_config
@property
def is_named(self):
return self.type_attributes.is_named
@property
def configurable_from_scalar(self):
check.invariant(not isinstance(self, Configurable))
return False
@property
def configurable_from_dict(self):
check.invariant(not isinstance(self, Configurable))
return False
@property
def configurable_from_nullable(self):
check.invariant(not isinstance(self, Configurable))
return False
@property
def configurable_from_list(self):
check.invariant(not isinstance(self, Configurable))
return False
def coerce_runtime_value(self, _value):
check.not_implemented('Must implement in subclass')
def iterate_types(self):
yield self
def serialize_value(self, output_dir, value):
type_value = self.create_serializable_type_value(
self.coerce_runtime_value(value),
output_dir,
)
output_path = os.path.join(output_dir, 'type_value')
with open(output_path, 'w') as ff:
json.dump(
{
'type': type_value.name,
'value': type_value.value,
},
ff,
)
return type_value
def deserialize_value(self, output_dir):
with open(os.path.join(output_dir, 'type_value'), 'r') as ff:
type_value_dict = json.load(ff)
type_value = SerializedTypeValue(
name=type_value_dict['type'],
value=type_value_dict['value'],
)
if type_value.name != self.name:
raise Exception('type mismatch')
return self.deserialize_from_type_value(type_value, output_dir)
# Override these in subclasses for customizable serialization
def create_serializable_type_value(self, value, _output_dir):
return SerializedTypeValue(self.name, value)
# Override these in subclasses for customizable serialization
def deserialize_from_type_value(self, type_value, _output_dir):
return type_value.value
class UncoercedTypeMixin(object):
'''This is a helper mixin used when you only want to do a type check
against an in-memory value and then leave that value uncoerced. Only
is_python_valid_value must be implemented for these classes.
evaluate_value is implemented for you.
'''
def is_python_valid_value(self, _value):
'''Subclasses must implement this method. Check if the value and output a boolean.
Returns:
bool: Whether the value is valid.
'''
check.failed('must implement')
def coerce_runtime_value(self, value):
if not self.is_python_valid_value(value):
raise DagsterRuntimeCoercionError(
'Expected valid value for {type_name} but got {value}'.format(
type_name=self.name,
value=repr(value),
),
)
return value
class DagsterScalarType(UncoercedTypeMixin, DagsterType):
'''Base class for dagster types that are scalar python values.
Attributes:
name (str): Name of the type
description (str): Description of the type
'''
# All builtins are configurable
class DagsterBuiltinScalarType(
ConfigurableFromScalar,
DagsterScalarType,
MaterializeableBuiltinScalar,
):
def __init__(self, name, description=None):
super(DagsterBuiltinScalarType, self).__init__(
name=name,
type_attributes=DagsterTypeAttributes(is_builtin=True),
description=None,
)
class _DagsterAnyType(ConfigurableFromAny, UncoercedTypeMixin, DagsterType):
def __init__(self):
super(_DagsterAnyType, self).__init__(
name='Any',
type_attributes=DagsterTypeAttributes(is_builtin=True),
description='The type that allows any value, including no value.',
)
def is_python_valid_value(self, _value):
return True
class PythonObjectType(UncoercedTypeMixin, DagsterType):
'''Dagster Type that checks if the value is an instance of some `python_type`'''
def __init__(
self,
name,
python_type,
type_attributes=DEFAULT_TYPE_ATTRIBUTES,
description=None,
):
super(PythonObjectType, self).__init__(
name=name,
type_attributes=type_attributes,
description=description,
)
self.python_type = check.type_param(python_type, 'python_type')
def is_python_valid_value(self, value):
return isinstance(value, self.python_type)
def serialize_value(self, output_dir, value):
type_value = self.create_serializable_type_value(
self.coerce_runtime_value(value), output_dir
)
output_path = os.path.join(output_dir, 'type_value')
with open(output_path, 'w') as ff:
json.dump(
{
'type': type_value.name,
'path': 'pickle'
},
ff,
)
pickle_path = os.path.join(output_dir, 'pickle')
with open(pickle_path, 'wb') as pf:
pickle.dump(value, pf)
return type_value
# If python had final methods, these would be final
def deserialize_value(self, output_dir):
with open(os.path.join(output_dir, 'type_value'), 'r') as ff:
type_value_dict = json.load(ff)
if type_value_dict['type'] != self.name:
raise Exception('type mismatch')
path = type_value_dict['path']
with open(os.path.join(output_dir, path), 'rb') as pf:
return pickle.load(pf)
class DagsterStringType(DagsterBuiltinScalarType):
def is_python_valid_value(self, value):
return isinstance(value, string_types)
class _DagsterIntType(DagsterBuiltinScalarType):
def __init__(self):
super(_DagsterIntType, self).__init__('Int', description='An integer.')
def is_python_valid_value(self, value):
if isinstance(value, bool):
return False
return isinstance(value, integer_types)
class _DagsterBoolType(DagsterBuiltinScalarType):
def __init__(self):
super(_DagsterBoolType, self).__init__('Bool', description='A boolean.')
def is_python_valid_value(self, value):
return isinstance(value, bool)
def Nullable(inner_type):
return _DagsterNullableType(inner_type)
class _DagsterNullableType(ConfigurableFromNullable, DagsterType):
def __init__(self, inner_type):
self.inner_type = check.inst_param(inner_type, 'inner_type', DagsterType)
super(_DagsterNullableType, self).__init__(
inner_configurable=inner_type,
name='Nullable.{inner_type}'.format(inner_type=inner_type.name),
type_attributes=DagsterTypeAttributes(is_builtin=True, is_named=False),
)
def coerce_runtime_value(self, value):
return None if value is None else self.inner_type.coerce_runtime_value(value)
def iterate_types(self):
yield self.inner_type
def List(inner_type):
return _DagsterListType(inner_type)
class _DagsterListType(ConfigurableFromList, DagsterType):
def __init__(self, inner_type):
self.inner_type = check.inst_param(inner_type, 'inner_type', DagsterType)
super(_DagsterListType, self).__init__(
inner_configurable=inner_type,
name='List.{inner_type}'.format(inner_type=inner_type.name),
description='List of {inner_type}'.format(inner_type=inner_type.name),
type_attributes=DagsterTypeAttributes(is_builtin=True, is_named=False),
)
def coerce_runtime_value(self, value):
if not isinstance(value, list):
raise DagsterRuntimeCoercionError('Must be a list')
return list(map(self.inner_type.coerce_runtime_value, value))
def iterate_types(self):
yield self.inner_type
# HACK HACK HACK
#
# This is not good and a better solution needs to be found. In order
# for the client-side typeahead in dagit to work as currently structured,
# dictionaries need names. While we deal with that we're going to automatically
# name dictionaries. This will cause odd behavior and bugs is you restart
# the server-side process, the type names changes, and you do not refresh the client.
#
# A possible short term mitigation would to name the dictionary based on the hash
# of its member fields to provide stability in between process restarts.
#
class DictCounter:
_count = 0
@staticmethod
def get_next_count():
DictCounter._count += 1
return DictCounter._count
def Dict(fields):
return _Dict('Dict.' + str(DictCounter.get_next_count()), fields)
def NamedDict(name, fields):
return _Dict(name, fields)
class _Dict(ConfigurableObjectFromDict, DagsterType):
'''Configuration dictionary.
Typed-checked but then passed to implementations as a python dict
Arguments:
fields (dict): dictonary of :py:class:`Field` objects keyed by name'''
def __init__(self, name, fields):
super(_Dict, self).__init__(
name=name,
fields=fields,
description='A configuration dictionary with typed fields',
type_attributes=DagsterTypeAttributes(is_named=True, is_builtin=True),
)
def coerce_runtime_value(self, value):
return value
String = DagsterStringType(name='String', description='A string.')
Path = DagsterStringType(
name='Path',
description='''
A string the represents a path. It is very useful for some tooling
to know that a string indeed represents a file path. That way they
can, for example, make the paths relative to a different location
for a particular execution environment.
''',
)
Int = _DagsterIntType()
Bool = _DagsterBoolType()
Any = _DagsterAnyType()
# TO DISCUSS: Consolidate with Dict?
PythonDict = PythonObjectType('Dict', dict, type_attributes=DagsterTypeAttributes(is_builtin=True))
Procedure to reproduce the problem
sphinx-build -b html -d _build/doctrees . _build/html
Error logs / results
# Sphinx version: 1.7.5
# Python version: 3.7.1 (CPython)
# Docutils version: 0.14
# Jinja2 version: 2.10
# Last messages:
# building [mo]: targets for 0 po files that are out of date
#
# building [html]: targets for 1 source files that are out of date
#
# updating environment:
#
# 0 added, 1 changed, 0 removed
#
# reading sources... [100%] apidocs/types
#
# Loaded extensions:
# alabaster (0.7.12) from /Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/alabaster/__init__.py
# sphinx.ext.autodoc (1.7.5) from /Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/sphinx/ext/autodoc/__init__.py
# sphinx.ext.napoleon (1.7.5) from /Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/sphinx/ext/napoleon/__init__.py
Traceback (most recent call last):
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/sphinx/ext/autodoc/__init__.py", line 497, in process_doc
self.options, docstringlines)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/sphinx/application.py", line 444, in emit
return self.events.emit(event, self, *args)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/sphinx/events.py", line 79, in emit
results.append(callback(*args))
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/sphinx/ext/napoleon/__init__.py", line 367, in _process_docstring
obj, options)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/sphinx/ext/napoleon/docstring.py", line 885, in __init__
name, obj, options)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/sphinx/ext/napoleon/docstring.py", line 165, in __init__
self._parse()
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/sphinx/ext/napoleon/docstring.py", line 530, in _parse
self._parsed_lines.extend(self._parse_attribute_docstring())
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/sphinx/ext/napoleon/docstring.py", line 555, in _parse_attribute_docstring
_type, _desc = self._consume_inline_attribute()
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/sphinx/ext/napoleon/docstring.py", line 253, in _consume_inline_attribute
line = next(self._line_iter)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/sphinx/ext/napoleon/iterators.py", line 72, in __next__
return getattr(self, 'next')(n)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/sphinx/ext/napoleon/iterators.py", line 127, in next
raise StopIteration
StopIteration
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/sphinx/cmdline.py", line 304, in main
app.build(args.force_all, filenames)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/sphinx/application.py", line 331, in build
self.builder.build_update()
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/sphinx/builders/__init__.py", line 342, in build_update
'out of date' % len(to_build))
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/sphinx/builders/__init__.py", line 355, in build
updated_docnames = set(self.env.update(self.config, self.srcdir, self.doctreedir))
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/sphinx/environment/__init__.py", line 565, in update
self._read_serial(docnames, self.app)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/sphinx/environment/__init__.py", line 584, in _read_serial
self.read_doc(docname, app)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/sphinx/environment/__init__.py", line 659, in read_doc
doctree = read_doc(self.app, self, self.doc2path(docname))
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/sphinx/io.py", line 294, in read_doc
pub.publish()
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/docutils/core.py", line 217, in publish
self.settings)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/docutils/readers/__init__.py", line 72, in read
self.parse()
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/docutils/readers/__init__.py", line 78, in parse
self.parser.parse(self.input, document)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/sphinx/parsers.py", line 85, in parse
self.statemachine.run(inputstring, document, inliner=self.inliner)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/docutils/parsers/rst/states.py", line 171, in run
input_source=document['source'])
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/docutils/statemachine.py", line 239, in run
context, state, transitions)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/docutils/statemachine.py", line 460, in check_line
return method(match, context, next_state)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/docutils/parsers/rst/states.py", line 2753, in underline
self.section(title, source, style, lineno - 1, messages)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/docutils/parsers/rst/states.py", line 327, in section
self.new_subsection(title, lineno, messages)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/docutils/parsers/rst/states.py", line 395, in new_subsection
node=section_node, match_titles=True)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/docutils/parsers/rst/states.py", line 282, in nested_parse
node=node, match_titles=match_titles)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/docutils/parsers/rst/states.py", line 196, in run
results = StateMachineWS.run(self, input_lines, input_offset)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/docutils/statemachine.py", line 239, in run
context, state, transitions)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/docutils/statemachine.py", line 460, in check_line
return method(match, context, next_state)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/docutils/parsers/rst/states.py", line 2753, in underline
self.section(title, source, style, lineno - 1, messages)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/docutils/parsers/rst/states.py", line 327, in section
self.new_subsection(title, lineno, messages)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/docutils/parsers/rst/states.py", line 395, in new_subsection
node=section_node, match_titles=True)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/docutils/parsers/rst/states.py", line 282, in nested_parse
node=node, match_titles=match_titles)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/docutils/parsers/rst/states.py", line 196, in run
results = StateMachineWS.run(self, input_lines, input_offset)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/docutils/statemachine.py", line 239, in run
context, state, transitions)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/docutils/statemachine.py", line 460, in check_line
return method(match, context, next_state)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/docutils/parsers/rst/states.py", line 2328, in explicit_markup
self.explicit_list(blank_finish)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/docutils/parsers/rst/states.py", line 2358, in explicit_list
match_titles=self.state_machine.match_titles)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/docutils/parsers/rst/states.py", line 319, in nested_list_parse
node=node, match_titles=match_titles)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/docutils/parsers/rst/states.py", line 196, in run
results = StateMachineWS.run(self, input_lines, input_offset)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/docutils/statemachine.py", line 239, in run
context, state, transitions)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/docutils/statemachine.py", line 460, in check_line
return method(match, context, next_state)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/docutils/parsers/rst/states.py", line 2631, in explicit_markup
nodelist, blank_finish = self.explicit_construct(match)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/docutils/parsers/rst/states.py", line 2338, in explicit_construct
return method(self, expmatch)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/docutils/parsers/rst/states.py", line 2081, in directive
directive_class, match, type_name, option_presets)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/docutils/parsers/rst/states.py", line 2130, in run_directive
result = directive_instance.run()
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/sphinx/ext/autodoc/directive.py", line 133, in run
documenter.generate(more_content=self.content)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/sphinx/ext/autodoc/__init__.py", line 796, in generate
self.add_content(more_content)
File "/Users/schrockn/code/venvs/dagster-3.7.1/lib/python3.7/site-packages/sphinx/ext/autodoc/__init__.py", line 537, in add_content
for i, line in enumerate(self.process_doc(docstrings)):
RuntimeError: generator raised StopIteration
Expected results
This should build the project. Works in pre 3.7 python versions.
Reproducible project / your project
Pretty difficult to produce a totally reproducible build in this situation. I figured this stack would be useful even without that.
Environment info
- OS: Mac 10.14.2
- Python version: Python 3.7.1
(Note this works fine in other python versions_ - Sphinx version: 1.7.5
sphinx-autobuild 0.7.1