From 3179018b4a47b31dfffbb7b12c32f2ed211c55b7 Mon Sep 17 00:00:00 2001 From: Coen van der Kamp Date: Thu, 11 Feb 2021 00:54:13 +0100 Subject: [PATCH 1/6] WIP Add any-of --- openapi_core/schema/schemas/factories.py | 11 ++- openapi_core/schema/schemas/models.py | 3 +- .../unmarshalling/schemas/unmarshallers.py | 41 +++++++++- tests/unit/unmarshalling/test_validate.py | 74 +++++++++++++++++++ 4 files changed, 126 insertions(+), 3 deletions(-) diff --git a/openapi_core/schema/schemas/factories.py b/openapi_core/schema/schemas/factories.py index 55b48fe1..0ffb02eb 100644 --- a/openapi_core/schema/schemas/factories.py +++ b/openapi_core/schema/schemas/factories.py @@ -31,6 +31,7 @@ def create(self, schema_spec): deprecated = schema_deref.get('deprecated', False) all_of_spec = schema_deref.get('allOf', None) one_of_spec = schema_deref.get('oneOf', None) + any_of_spec = schema_deref.get('anyOf', None) additional_properties_spec = schema_deref.get('additionalProperties', True) min_items = schema_deref.get('minItems', None) @@ -63,6 +64,10 @@ def create(self, schema_spec): if one_of_spec: one_of = list(map(self.create, one_of_spec)) + any_of = [] + if any_of_spec: + any_of = list(map(self.create, any_of_spec)) + items = None if items_spec: items = self._create_items(items_spec) @@ -75,7 +80,7 @@ def create(self, schema_spec): schema_type=schema_type, properties=properties, items=items, schema_format=schema_format, required=required, default=default, nullable=nullable, enum=enum, - deprecated=deprecated, all_of=all_of, one_of=one_of, + deprecated=deprecated, all_of=all_of, one_of=one_of, any_of=any_of, additional_properties=additional_properties, min_items=min_items, max_items=max_items, min_length=min_length, max_length=max_length, pattern=pattern, unique_items=unique_items, @@ -118,6 +123,10 @@ class SchemaDictFactory(object): 'one_of', dest_prop_name='oneOf', is_list=True, dest_default=[], ), + Contribution( + 'any_of', + dest_prop_name='anyOf', is_list=True, dest_default=[], + ), Contribution( 'additional_properties', dest_prop_name='additionalProperties', dest_default=True, diff --git a/openapi_core/schema/schemas/models.py b/openapi_core/schema/schemas/models.py index a4109c4d..4a187ae4 100644 --- a/openapi_core/schema/schemas/models.py +++ b/openapi_core/schema/schemas/models.py @@ -21,7 +21,7 @@ class Schema(object): def __init__( self, schema_type=None, properties=None, items=None, schema_format=None, required=None, default=NoValue, nullable=False, - enum=None, deprecated=False, all_of=None, one_of=None, + enum=None, deprecated=False, all_of=None, one_of=None, any_of=None, additional_properties=True, min_items=None, max_items=None, min_length=None, max_length=None, pattern=None, unique_items=False, minimum=None, maximum=None, multiple_of=None, @@ -40,6 +40,7 @@ def __init__( self.deprecated = deprecated self.all_of = all_of and list(all_of) or [] self.one_of = one_of and list(one_of) or [] + self.any_of = any_of and list(any_of) or [] self.additional_properties = additional_properties self.min_items = int(min_items) if min_items is not None else None self.max_items = int(max_items) if max_items is not None else None diff --git a/openapi_core/unmarshalling/schemas/unmarshallers.py b/openapi_core/unmarshalling/schemas/unmarshallers.py index ef0fdb70..19c6a66b 100644 --- a/openapi_core/unmarshalling/schemas/unmarshallers.py +++ b/openapi_core/unmarshalling/schemas/unmarshallers.py @@ -187,6 +187,23 @@ def _unmarshal_object(self, value=NoValue): if properties is None: log.warning("valid oneOf schema not found") + if self.schema.any_of: + properties = None + for any_of_schema in self.schema.any_of: + try: + unmarshalled = self._unmarshal_properties( + value, any_of_schema) + except (UnmarshalError, ValueError): + pass + else: + if properties is not None: + log.warning("multiple valid anyOf schemas found") + continue + properties = unmarshalled + + if properties is None: + log.warning("valid anyOf schema not found") + else: properties = self._unmarshal_properties(value) @@ -196,7 +213,8 @@ def _unmarshal_object(self, value=NoValue): return properties - def _unmarshal_properties(self, value=NoValue, one_of_schema=None): + def _unmarshal_properties(self, value=NoValue, one_of_schema=None, + any_of_schema=None): all_props = self.schema.get_all_properties() all_props_names = self.schema.get_all_properties_names() @@ -205,6 +223,11 @@ def _unmarshal_properties(self, value=NoValue, one_of_schema=None): all_props_names |= one_of_schema.\ get_all_properties_names() + if any_of_schema is not None: + all_props.update(any_of_schema.get_all_properties()) + all_props_names |= any_of_schema.\ + get_all_properties_names() + value_props_names = value.keys() extra_props = set(value_props_names) - set(all_props_names) @@ -253,6 +276,10 @@ def __call__(self, value=NoValue): if one_of_schema: return self.unmarshallers_factory.create(one_of_schema)(value) + any_of_schema = self._get_any_of_schema(value) + if any_of_schema: + return self.unmarshallers_factory.create(any_of_schema)(value) + all_of_schema = self._get_all_of_schema(value) if all_of_schema: return self.unmarshallers_factory.create(all_of_schema)(value) @@ -283,6 +310,18 @@ def _get_one_of_schema(self, value): else: return subschema + def _get_any_of_schema(self, value): + if not self.schema.any_of: + return + for subschema in self.schema.any_of: + unmarshaller = self.unmarshallers_factory.create(subschema) + try: + unmarshaller.validate(value) + except ValidateError: + continue + else: + return subschema + def _get_all_of_schema(self, value): if not self.schema.all_of: return diff --git a/tests/unit/unmarshalling/test_validate.py b/tests/unit/unmarshalling/test_validate.py index fdb5d950..9dafb6a7 100644 --- a/tests/unit/unmarshalling/test_validate.py +++ b/tests/unit/unmarshalling/test_validate.py @@ -529,6 +529,80 @@ def test_unambiguous_one_of(self, value, validator_factory): assert result is None + @pytest.mark.parametrize('value', [Model(), ]) + def test_object_multiple_any_of(self, value, validator_factory): + any_of = [ + Schema('object'), Schema('object'), + ] + schema = Schema('object', any_of=any_of) + + with pytest.raises(InvalidSchemaValue): + validator_factory(schema).validate(value) + + @pytest.mark.parametrize('value', [{}, ]) + def test_object_different_type_any_of(self, value, validator_factory): + any_of = [ + Schema('integer'), Schema('string'), + ] + schema = Schema('object', any_of=any_of) + + with pytest.raises(InvalidSchemaValue): + validator_factory(schema).validate(value) + + @pytest.mark.parametrize('value', [{}, ]) + def test_object_no_any_of(self, value, validator_factory): + any_of = [ + Schema( + 'object', + properties={'test1': Schema('string')}, + required=['test1', ], + ), + Schema( + 'object', + properties={'test2': Schema('string')}, + required=['test2', ], + ), + ] + schema = Schema('object', any_of=any_of) + + with pytest.raises(InvalidSchemaValue): + validator_factory(schema).validate(value) + + @pytest.mark.parametrize('value', [ + { + 'foo': u("FOO"), + }, + { + 'foo': u("FOO"), + 'bar': u("BAR"), + }, + ]) + def test_unambiguous_any_of(self, value, validator_factory): + any_of = [ + Schema( + 'object', + properties={ + 'foo': Schema('string'), + }, + additional_properties=False, + required=['foo'], + ), + Schema( + 'object', + properties={ + 'foo': Schema('string'), + 'bar': Schema('string'), + }, + additional_properties=False, + required=['foo', 'bar'], + ), + ] + schema = Schema('object', any_of=any_of) + + result = validator_factory(schema).validate(value) + + assert result is None + @pytest.mark.parametrize('value', [{}, ]) def test_object_default_property(self, value, validator_factory): schema = Schema('object', default='value1') From a265b99b446c3741fd20d1d788989a8625531b5b Mon Sep 17 00:00:00 2001 From: Sigurd Spieckermann Date: Mon, 7 Jun 2021 19:15:07 +0200 Subject: [PATCH 2/6] Use "elif" instead of "if" --- openapi_core/unmarshalling/schemas/unmarshallers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openapi_core/unmarshalling/schemas/unmarshallers.py b/openapi_core/unmarshalling/schemas/unmarshallers.py index 16897aeb..de4c4a8f 100644 --- a/openapi_core/unmarshalling/schemas/unmarshallers.py +++ b/openapi_core/unmarshalling/schemas/unmarshallers.py @@ -200,7 +200,7 @@ def _unmarshal_object(self, value): if properties is None: log.warning("valid oneOf schema not found") - if 'anyOf' in self.schema: + elif 'anyOf' in self.schema: properties = None for any_of_schema in self.schema / 'anyOf': try: From 6d45c4aabe100b00647ec79c0c935f1081b4bad7 Mon Sep 17 00:00:00 2001 From: Sigurd Spieckermann Date: Tue, 8 Jun 2021 10:57:00 +0200 Subject: [PATCH 3/6] Fix unmarshaling for "anyOf" --- .../unmarshalling/schemas/unmarshallers.py | 61 +++++++------------ tests/unit/unmarshalling/test_unmarshal.py | 17 ++++++ tests/unit/unmarshalling/test_validate.py | 16 +++-- 3 files changed, 51 insertions(+), 43 deletions(-) diff --git a/openapi_core/unmarshalling/schemas/unmarshallers.py b/openapi_core/unmarshalling/schemas/unmarshallers.py index de4c4a8f..f302d451 100644 --- a/openapi_core/unmarshalling/schemas/unmarshallers.py +++ b/openapi_core/unmarshalling/schemas/unmarshallers.py @@ -10,9 +10,7 @@ from openapi_schema_validator._format import oas30_format_checker from openapi_core.extensions.models.factories import ModelFactory -from openapi_core.schema.schemas import ( - get_all_properties, get_all_properties_names -) +from openapi_core.schema.schemas import get_all_properties from openapi_core.unmarshalling.schemas.enums import UnmarshalContext from openapi_core.unmarshalling.schemas.exceptions import ( UnmarshalError, ValidateError, InvalidSchemaValue, @@ -187,8 +185,7 @@ def _unmarshal_object(self, value): properties = None for one_of_schema in self.schema / 'oneOf': try: - unmarshalled = self._unmarshal_properties( - value, one_of_schema=one_of_schema) + unmarshalled = self._unmarshal_properties(value) except (UnmarshalError, ValueError): pass else: @@ -204,15 +201,12 @@ def _unmarshal_object(self, value): properties = None for any_of_schema in self.schema / 'anyOf': try: - unmarshalled = self._unmarshal_properties( - value, any_of_schema=any_of_schema) + unmarshalled = self._unmarshal_properties(value) except (UnmarshalError, ValueError): pass else: - if properties is not None: - log.warning("multiple valid anyOf schemas found") - continue properties = unmarshalled + break if properties is None: log.warning("valid anyOf schema not found") @@ -226,37 +220,10 @@ def _unmarshal_object(self, value): return properties - def _unmarshal_properties( - self, value, one_of_schema=None, any_of_schema=None): - all_props = get_all_properties(self.schema) - all_props_names = get_all_properties_names(self.schema) - - if one_of_schema is not None: - all_props.update(get_all_properties(one_of_schema)) - all_props_names |= get_all_properties_names(one_of_schema) - - if any_of_schema is not None: - all_props.update(get_all_properties(any_of_schema)) - all_props_names |= get_all_properties_names(any_of_schema) - - value_props_names = list(value.keys()) - extra_props = set(value_props_names) - set(all_props_names) - + def _unmarshal_properties(self, value): properties = {} - additional_properties = self.schema.getkey( - 'additionalProperties', True) - if isinstance(additional_properties, dict): - additional_prop_schema = self.schema / 'additionalProperties' - for prop_name in extra_props: - prop_value = value[prop_name] - properties[prop_name] = self.unmarshallers_factory.create( - additional_prop_schema)(prop_value) - elif additional_properties is True: - for prop_name in extra_props: - prop_value = value[prop_name] - properties[prop_name] = prop_value - for prop_name, prop in list(all_props.items()): + for prop_name, prop in get_all_properties(self.schema).items(): read_only = prop.getkey('readOnly', False) if self.context == UnmarshalContext.REQUEST and read_only: continue @@ -273,6 +240,22 @@ def _unmarshal_properties( properties[prop_name] = self.unmarshallers_factory.create( prop)(prop_value) + additional_properties = self.schema.getkey( + 'additionalProperties', True) + if isinstance(additional_properties, dict): + additional_prop_schema = self.schema / 'additionalProperties' + additional_prop_unmarshaler = self.unmarshallers_factory.create( + additional_prop_schema) + for prop_name, prop_value in value.items(): + if prop_name in properties: + continue + properties[prop_name] = additional_prop_unmarshaler(prop_value) + elif additional_properties is True: + for prop_name, prop_value in value.items(): + if prop_name in properties: + continue + properties[prop_name] = prop_value + return properties diff --git a/tests/unit/unmarshalling/test_unmarshal.py b/tests/unit/unmarshalling/test_unmarshal.py index 8d88b1f0..af830895 100644 --- a/tests/unit/unmarshalling/test_unmarshal.py +++ b/tests/unit/unmarshalling/test_unmarshal.py @@ -542,6 +542,23 @@ def test_schema_any_one_of(self, unmarshaller_factory): schema = SpecPath.from_spec(spec) assert unmarshaller_factory(schema)(['hello']) == ['hello'] + def test_schema_any_any_of(self, unmarshaller_factory): + spec = { + 'anyOf': [ + { + 'type': 'string', + }, + { + 'type': 'array', + 'items': { + 'type': 'string', + } + } + ], + } + schema = SpecPath.from_spec(spec) + assert unmarshaller_factory(schema)(['hello']) == ['hello'] + def test_schema_any_all_of(self, unmarshaller_factory): spec = { 'allOf': [ diff --git a/tests/unit/unmarshalling/test_validate.py b/tests/unit/unmarshalling/test_validate.py index 1168b5ff..cc9b8274 100644 --- a/tests/unit/unmarshalling/test_validate.py +++ b/tests/unit/unmarshalling/test_validate.py @@ -761,17 +761,25 @@ def test_unambiguous_one_of(self, value, validator_factory): assert result is None - @pytest.mark.parametrize('value', [Model(), ]) + @pytest.mark.parametrize('value', [{}, ]) def test_object_multiple_any_of(self, value, validator_factory): - any_of = [{'type': 'object'}, {'type': 'object'}] + any_of = [ + { + 'type': 'object', + }, + { + 'type': 'object', + }, + ] spec = { 'type': 'object', 'anyOf': any_of, } schema = SpecPath.from_spec(spec) - with pytest.raises(InvalidSchemaValue): - validator_factory(schema).validate(value) + result = validator_factory(schema).validate(value) + + assert result is None @pytest.mark.parametrize('value', [{}, ]) def test_object_different_type_any_of(self, value, validator_factory): From 0ef039742e4a996db3722730ba2ee5d0178b2753 Mon Sep 17 00:00:00 2001 From: Sigurd Spieckermann Date: Tue, 8 Jun 2021 12:33:31 +0200 Subject: [PATCH 4/6] Add more tests --- tests/unit/unmarshalling/test_unmarshal.py | 55 ++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/unit/unmarshalling/test_unmarshal.py b/tests/unit/unmarshalling/test_unmarshal.py index af830895..5e117af8 100644 --- a/tests/unit/unmarshalling/test_unmarshal.py +++ b/tests/unit/unmarshalling/test_unmarshal.py @@ -559,6 +559,61 @@ def test_schema_any_any_of(self, unmarshaller_factory): schema = SpecPath.from_spec(spec) assert unmarshaller_factory(schema)(['hello']) == ['hello'] + def test_schema_object_any_of(self, unmarshaller_factory): + spec = { + 'type': 'object', + 'anyOf': [ + { + 'type': 'object', + 'required': ['someint'], + 'properties': { + 'someint': { + 'type': 'integer' + } + } + }, + { + 'type': 'object', + 'required': ['somestr'], + 'properties': { + 'somestr': { + 'type': 'string' + } + } + } + ], + } + schema = SpecPath.from_spec(spec) + assert unmarshaller_factory(schema)({'someint': 1}) == {'someint': 1} + + def test_schema_object_any_of_invalid(self, unmarshaller_factory): + spec = { + 'type': 'object', + 'anyOf': [ + { + 'type': 'object', + 'required': ['someint'], + 'properties': { + 'someint': { + 'type': 'integer' + } + } + }, + { + 'type': 'object', + 'required': ['somestr'], + 'properties': { + 'somestr': { + 'type': 'string' + } + } + } + ], + } + schema = SpecPath.from_spec(spec) + with pytest.raises(UnmarshalError): + unmarshaller_factory(schema)({'someint': '1'}) + def test_schema_any_all_of(self, unmarshaller_factory): spec = { 'allOf': [ From c6dd04bdb1a5c78d4c4779a0a84a620063df76a7 Mon Sep 17 00:00:00 2001 From: Sigurd Spieckermann Date: Tue, 8 Jun 2021 12:36:35 +0200 Subject: [PATCH 5/6] Remove unused code --- openapi_core/schema/schemas.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/openapi_core/schema/schemas.py b/openapi_core/schema/schemas.py index 43919cb3..2a696adf 100644 --- a/openapi_core/schema/schemas.py +++ b/openapi_core/schema/schemas.py @@ -10,8 +10,3 @@ def get_all_properties(schema): properties_dict.update(subschema_props) return properties_dict - - -def get_all_properties_names(schema): - all_properties = get_all_properties(schema) - return set(all_properties.keys()) From 2b05067476e982a65a1a41fdc7ff7baf7d59e660 Mon Sep 17 00:00:00 2001 From: Sigurd Spieckermann Date: Wed, 30 Jun 2021 17:21:55 +0200 Subject: [PATCH 6/6] Fix object oneOf/anyOf unmarshaling and model creation --- .../unmarshalling/schemas/unmarshallers.py | 53 +++++++++++-------- 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/openapi_core/unmarshalling/schemas/unmarshallers.py b/openapi_core/unmarshalling/schemas/unmarshallers.py index f302d451..4f2bbeed 100644 --- a/openapi_core/unmarshalling/schemas/unmarshallers.py +++ b/openapi_core/unmarshalling/schemas/unmarshallers.py @@ -172,6 +172,16 @@ def model_factory(self): return ModelFactory() def unmarshal(self, value): + properties = self.unmarshal_raw(value) + + if 'x-model' in self.schema: + name = self.schema['x-model'] + model = self.model_factory.create(properties, name=name) + return model + + return properties + + def unmarshal_raw(self, value): try: value = self.formatter.unmarshal(value) except ValueError as exc: @@ -180,48 +190,49 @@ def unmarshal(self, value): else: return self._unmarshal_object(value) + def _clone(self, schema): + return ObjectUnmarshaller( + schema, self.formatter, self.validator, self.unmarshallers_factory, + self.context) + def _unmarshal_object(self, value): + properties = {} + if 'oneOf' in self.schema: - properties = None + one_of_properties = None for one_of_schema in self.schema / 'oneOf': try: - unmarshalled = self._unmarshal_properties(value) + unmarshalled = self._clone(one_of_schema).unmarshal_raw( + value) except (UnmarshalError, ValueError): pass else: - if properties is not None: + if one_of_properties is not None: log.warning("multiple valid oneOf schemas found") continue - properties = unmarshalled + one_of_properties = unmarshalled - if properties is None: + if one_of_properties is None: log.warning("valid oneOf schema not found") + else: + properties.update(one_of_properties) elif 'anyOf' in self.schema: - properties = None + any_of_properties = None for any_of_schema in self.schema / 'anyOf': try: - unmarshalled = self._unmarshal_properties(value) + unmarshalled = self._clone(any_of_schema).unmarshal_raw( + value) except (UnmarshalError, ValueError): pass else: - properties = unmarshalled + any_of_properties = unmarshalled break - if properties is None: + if any_of_properties is None: log.warning("valid anyOf schema not found") - - else: - properties = self._unmarshal_properties(value) - - if 'x-model' in self.schema: - name = self.schema['x-model'] - return self.model_factory.create(properties, name=name) - - return properties - - def _unmarshal_properties(self, value): - properties = {} + else: + properties.update(any_of_properties) for prop_name, prop in get_all_properties(self.schema).items(): read_only = prop.getkey('readOnly', False)