From 26b34d36fa14379a2d57c1bbc5fa4468723274ac Mon Sep 17 00:00:00 2001 From: Jonathan Wylie Date: Tue, 15 Mar 2016 02:20:14 +0000 Subject: [PATCH 1/5] #88 Support the addition of new fields to an existing schema. - Especially ListFields and DictFields --- couchdb/mapping.py | 29 +++++++++++++- couchdb/tests/mapping.py | 81 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+), 1 deletion(-) diff --git a/couchdb/mapping.py b/couchdb/mapping.py index bedddc13..cf6fcff3 100644 --- a/couchdb/mapping.py +++ b/couchdb/mapping.py @@ -177,7 +177,34 @@ def build(cls, **d): @classmethod def wrap(cls, data): - instance = cls() + """ Wrap the data object with cls, so that any data changes are made to + the data object + + cls may define schema for data that is not in data yet, + eg a document is saved to the DB + the document schema is changed to contain another kind of data + load the old document with the new schema + we want to be able to + write the data for the new schema + get a default value for the new schema + + :param data: A dictionary object of data + :return: an instance of class which provides a nice interface to the + information in the data object + """ + + # Build the instance, this means defaults are set for any new fields + instance = cls(**data) + # instance now has a _data it's own data object which has been created + # based on data, and any new schema which may not have been in + # the original doc + + # We have to wrap the data object, but we don't want to loose the + # defaults created in instance._data + # Merge any changes of data from instance back into the main data object + data.update(instance._data) + + # make the instance wrap the data object instance._data = data return instance diff --git a/couchdb/tests/mapping.py b/couchdb/tests/mapping.py index 94387cb3..e9816157 100644 --- a/couchdb/tests/mapping.py +++ b/couchdb/tests/mapping.py @@ -6,6 +6,7 @@ # This software is licensed as described in the file COPYING, which # you should have received as part of this distribution. +import copy from decimal import Decimal import unittest @@ -250,6 +251,86 @@ def test_query(self): self.assertEqual(type(results.rows[0]), self.Item) +class TestWrap(testutil.TempDatabaseMixin, unittest.TestCase): + + def test_simple_schema_change(self): + my_mapping = mapping.Mapping.build( + name=mapping.TextField(), + ) + my_instance = my_mapping.wrap({}) + my_instance.name = 'Harry' + + data = copy.copy(my_instance._data) + + my_new_mapping = mapping.Mapping.build( + name=mapping.TextField(), + age=mapping.IntegerField(), + ) + + instance = my_new_mapping.wrap(data) + instance.age = 32 + assert data['age'] == 32 + + def test_defaults_for_new_schema_should_be_set(self): + my_mapping = mapping.Mapping.build( + name=mapping.TextField(), + ) + my_instance = my_mapping.wrap({}) + my_instance.name = 'Harry' + + data = copy.copy(my_instance._data) + + my_new_mapping = mapping.Mapping.build( + name=mapping.TextField(), + address=mapping.DictField(mapping.Mapping.build( + number=mapping.IntegerField(default=1), + street=mapping.TextField(default='street') + )), + pets=mapping.ListField(mapping.TextField(), default=[]) + ) + + my_new_instance = my_new_mapping.wrap(data) + + assert data['pets'] == [] + assert data['address']['number'] == 1 + assert data['address']['street'] == 'street' + assert my_new_instance.address.number == 1 + assert my_new_instance.address.street == 'street' + + def test_can_set_new_schema_data(self): + my_mapping = mapping.Mapping.build( + name=mapping.TextField(), + ) + my_instance = my_mapping.wrap({}) + my_instance.name = 'Sherlock' + + data = copy.copy(my_instance._data) + + my_new_mapping = mapping.Mapping.build( + name=mapping.TextField(), + address=mapping.DictField(mapping.Mapping.build( + number=mapping.TextField(), + street=mapping.TextField() + )), + pets=mapping.ListField(mapping.TextField(), default=[]), + ) + + my_new_instance = my_new_mapping.wrap(data) + + my_new_instance.pets.append('dog') + my_new_instance.pets.append('cat') + my_new_instance.address.number = '221B' + my_new_instance.address.street = 'Baker Street' + + assert data['address']['number'] == '221B' + assert data['address']['street'] == 'Baker Street' + assert data['pets'] == ['dog', 'cat'] + assert my_new_instance.address.number == '221B' + assert my_new_instance.address.street == 'Baker Street' + assert my_new_instance.pets[0] == 'dog' + assert my_new_instance.pets[1] == 'cat' + + def suite(): suite = unittest.TestSuite() suite.addTest(testutil.doctest_suite(mapping)) From 41dc78f44a1f51b65aff5fe9c1ad5ff1c79007bf Mon Sep 17 00:00:00 2001 From: Jonathan Wylie Date: Wed, 16 Mar 2016 05:59:34 +0000 Subject: [PATCH 2/5] #88 Support the addition of new fields to an existing schema, by setting the default values for fields in the schema when wrapping data loaded from the DB. - Especially ListFields and DictFields --- .idea/vcs.xml | 6 +++++ couchdb/mapping.py | 21 ++++++++--------- couchdb/tests/mapping.py | 51 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 11 deletions(-) create mode 100644 .idea/vcs.xml diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 00000000..94a25f7f --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/couchdb/mapping.py b/couchdb/mapping.py index cf6fcff3..ba1c36a9 100644 --- a/couchdb/mapping.py +++ b/couchdb/mapping.py @@ -192,20 +192,19 @@ def wrap(cls, data): :return: an instance of class which provides a nice interface to the information in the data object """ + instance = cls() - # Build the instance, this means defaults are set for any new fields - instance = cls(**data) - # instance now has a _data it's own data object which has been created - # based on data, and any new schema which may not have been in - # the original doc + def add_defaults(default_values, data_dict): + for key, defaultValue in default_values.iteritems(): + if key not in data_dict: + data_dict[key] = defaultValue + if isinstance(defaultValue, dict): + add_defaults(defaultValue, data_dict[key]) - # We have to wrap the data object, but we don't want to loose the - # defaults created in instance._data - # Merge any changes of data from instance back into the main data object - data.update(instance._data) - - # make the instance wrap the data object + # Recursively set the defaults if they are not in the doc already + add_defaults(instance._data, data) instance._data = data + return instance def _to_python(self, value): diff --git a/couchdb/tests/mapping.py b/couchdb/tests/mapping.py index e9816157..a2ca6fb8 100644 --- a/couchdb/tests/mapping.py +++ b/couchdb/tests/mapping.py @@ -297,6 +297,57 @@ def test_defaults_for_new_schema_should_be_set(self): assert my_new_instance.address.number == 1 assert my_new_instance.address.street == 'street' + def test_defaults_for_new_schema_should_be_set_more_complex(self): + my_mapping = mapping.Mapping.build( + name=mapping.TextField(), + ) + my_instance = my_mapping.wrap({}) + my_instance.name = 'Harry' + + data = copy.copy(my_instance._data) + + my_new_mapping = mapping.Mapping.build( + name=mapping.TextField(), + address=mapping.DictField(mapping.Mapping.build( + number=mapping.IntegerField(default=1), + street=mapping.TextField(default='street') + )), + pets=mapping.ListField(mapping.TextField(), default=[]) + ) + + my_new_instance = my_new_mapping.wrap(data) + + assert data['pets'] == [] + assert data['address']['number'] == 1 + assert data['address']['street'] == 'street' + assert my_new_instance.address.number == 1 + assert my_new_instance.address.street == 'street' + + + data = copy.copy(my_new_instance._data) + + my_newer_mapping = mapping.Mapping.build( + name=mapping.TextField(), + address=mapping.DictField(mapping.Mapping.build( + number=mapping.IntegerField(default=1), + street=mapping.TextField(default='street'), + cars=mapping.ListField( + mapping.DictField( + mapping.Mapping.build( + registration=mapping.TextField() + ) + ) + ) + )), + pets=mapping.ListField(mapping.TextField(), default=[]) + ) + + my_newer_instance = my_newer_mapping.wrap(data) + + assert data['address']['cars'] == [] + + + def test_can_set_new_schema_data(self): my_mapping = mapping.Mapping.build( name=mapping.TextField(), From 505cf343f6cdca06da69824cdec33b721b55a35f Mon Sep 17 00:00:00 2001 From: Jonathan Wylie Date: Wed, 16 Mar 2016 06:16:32 +0000 Subject: [PATCH 3/5] #88 Handle getting the default value for a field explicitly. Instead of overloading the descriptor __get__ method. - The _get_ method should return values in python types, eg a date time is a datetime.datetime instance - The default values need to be in basic json types, eg [1,2,3] , {}, str, int which can be serialized to the doc - We don't need to get the default values in __get__ anymore because the defaults are set on initialisation of the mapping, or on wrap in a mapping. --- couchdb/mapping.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/couchdb/mapping.py b/couchdb/mapping.py index ba1c36a9..27c871e5 100644 --- a/couchdb/mapping.py +++ b/couchdb/mapping.py @@ -94,7 +94,11 @@ def __get__(self, instance, owner): value = instance._data.get(self.name) if value is not None: value = self._to_python(value) - elif self.default is not None: + return value + + def get_default(self): + value = None + if self.default is not None: default = self.default if callable(default): default = default() @@ -139,7 +143,8 @@ def __init__(self, **values): if attrname in values: setattr(self, attrname, values.pop(attrname)) else: - setattr(self, attrname, getattr(self, attrname)) + field = getattr(self.__class__, attrname) + setattr(self, attrname, field.get_default()) def __iter__(self): return iter(self._data) From 2f2eecd23f2be9b59e8bf18829daf8b4acced32f Mon Sep 17 00:00:00 2001 From: Jonathan Wylie Date: Wed, 16 Mar 2016 06:31:26 +0000 Subject: [PATCH 4/5] #88 Python 3 does not have iteritems so just use bog standard items. These aren't massive so it doesn't really matter. --- couchdb/mapping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchdb/mapping.py b/couchdb/mapping.py index 27c871e5..56e88876 100644 --- a/couchdb/mapping.py +++ b/couchdb/mapping.py @@ -200,7 +200,7 @@ def wrap(cls, data): instance = cls() def add_defaults(default_values, data_dict): - for key, defaultValue in default_values.iteritems(): + for key, defaultValue in default_values.items(): if key not in data_dict: data_dict[key] = defaultValue if isinstance(defaultValue, dict): From 9cdf6085a5940ad3308f8792d696a8ba4f5e4de6 Mon Sep 17 00:00:00 2001 From: Jonathan Wylie Date: Wed, 16 Mar 2016 08:25:14 +0000 Subject: [PATCH 5/5] #88 Support the addition of new fields to an existing schema. - Especially ListFields and DictFields - The _to_json need to be idempotent to support string values coming from wraps, and python type values eg DateTime coming from user code. --- couchdb/mapping.py | 17 ++++++++++++++++- couchdb/tests/mapping.py | 26 +++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/couchdb/mapping.py b/couchdb/mapping.py index 56e88876..7ee3f91b 100644 --- a/couchdb/mapping.py +++ b/couchdb/mapping.py @@ -197,6 +197,11 @@ def wrap(cls, data): :return: an instance of class which provides a nice interface to the information in the data object """ + instance = cls(**data) + data.update(instance._data) + instance._data = data + return instance + instance = cls() def add_defaults(default_values, data_dict): @@ -205,6 +210,10 @@ def add_defaults(default_values, data_dict): data_dict[key] = defaultValue if isinstance(defaultValue, dict): add_defaults(defaultValue, data_dict[key]) + # # If it is a list of DictField then the DictField schema may have updated + # if isinstance(defaultValue, list) and : + + # Recursively set the defaults if they are not in the doc already add_defaults(instance._data, data) @@ -496,6 +505,8 @@ def _to_python(self, value): return value def _to_json(self, value): + # Make sure it is the right format (it might have come from wrap) + value = self._to_python(value) if isinstance(value, datetime): value = value.date() return value.isoformat() @@ -524,6 +535,8 @@ def _to_python(self, value): return value def _to_json(self, value): + # Make sure it is the right format (it might have come from wrap) + value = self._to_python(value) if isinstance(value, struct_time): value = datetime.utcfromtimestamp(timegm(value)) elif not isinstance(value, datetime): @@ -553,6 +566,8 @@ def _to_python(self, value): return value def _to_json(self, value): + # Make sure it is the right format (it might have come from wrap) + value = self._to_python(value) if isinstance(value, datetime): value = value.time() return value.replace(microsecond=0).isoformat() @@ -642,7 +657,7 @@ class ListField(Field): >>> comment['content'] u'Bla bla' >>> comment['time'] #doctest: +ELLIPSIS - u'...T...Z' + '...T...Z' >>> del server['python-tests'] """ diff --git a/couchdb/tests/mapping.py b/couchdb/tests/mapping.py index a2ca6fb8..1d9996f0 100644 --- a/couchdb/tests/mapping.py +++ b/couchdb/tests/mapping.py @@ -323,7 +323,6 @@ def test_defaults_for_new_schema_should_be_set_more_complex(self): assert my_new_instance.address.number == 1 assert my_new_instance.address.street == 'street' - data = copy.copy(my_new_instance._data) my_newer_mapping = mapping.Mapping.build( @@ -345,8 +344,33 @@ def test_defaults_for_new_schema_should_be_set_more_complex(self): my_newer_instance = my_newer_mapping.wrap(data) assert data['address']['cars'] == [] + my_newer_instance.address.cars.append({"registration": "AJ54 VSE"}) + assert data['address']['cars'] == [{"registration": "AJ54 VSE"}] + assert my_newer_instance.address.cars[0].registration == "AJ54 VSE" + + data = copy.copy(my_newer_instance._data) + + my_newest_mapping = mapping.Mapping.build( + name=mapping.TextField(), + address=mapping.DictField(mapping.Mapping.build( + number=mapping.IntegerField(default=1), + street=mapping.TextField(default='street'), + cars=mapping.ListField( + mapping.DictField( + mapping.Mapping.build( + registration=mapping.TextField(), + wheels=mapping.IntegerField(default=4) + ) + ) + ) + )), + pets=mapping.ListField(mapping.TextField(), default=[]) + ) + my_newest_instance = my_newest_mapping.wrap(data) + assert data['address']['cars'] == [{"registration": "AJ54 VSE", "wheels": 4}] + assert my_newest_instance.address.cars[0].wheels == 4 def test_can_set_new_schema_data(self): my_mapping = mapping.Mapping.build(