From 0b8f56de9240486c96027018f49449c99552e14f Mon Sep 17 00:00:00 2001 From: Curtis Mason <31265687+cumason123@users.noreply.github.com> Date: Tue, 23 Jun 2020 15:31:37 -0400 Subject: [PATCH 01/10] Created CloudEvent class (#36) CloudEvents is a more pythonic interface for using cloud events. It is powered by internal marshallers and cloud event base classes. It performs basic validation on fields, and cloud event type checking. Signed-off-by: Curtis Mason Signed-off-by: Dustin Ingram --- cloudevents/sdk/event/base.py | 17 +- cloudevents/sdk/http_events.py | 136 ++++++++++++++++ cloudevents/tests/data.py | 4 +- cloudevents/tests/test_http_events.py | 146 ++++++++++++++++++ requirements/test.txt | 2 +- .../cloudevent_to_request.py | 44 ++++++ .../python-event-requests/sample-server.py | 34 ++++ 7 files changed, 379 insertions(+), 4 deletions(-) create mode 100644 cloudevents/sdk/http_events.py create mode 100644 cloudevents/tests/test_http_events.py create mode 100644 samples/python-event-requests/cloudevent_to_request.py create mode 100644 samples/python-event-requests/sample-server.py diff --git a/cloudevents/sdk/event/base.py b/cloudevents/sdk/event/base.py index d392ae8b..a8bb099e 100644 --- a/cloudevents/sdk/event/base.py +++ b/cloudevents/sdk/event/base.py @@ -16,6 +16,21 @@ import json import typing +_ce_required_fields = { + 'id', + 'source', + 'type', + 'specversion' +} + + +_ce_optional_fields = { + 'datacontenttype', + 'schema', + 'subject', + 'time' +} + # TODO(slinkydeveloper) is this really needed? class EventGetterSetter(object): @@ -117,6 +132,7 @@ def MarshalJSON(self, data_marshaller: typing.Callable) -> typing.IO: def UnmarshalJSON(self, b: typing.IO, data_unmarshaller: typing.Callable): raw_ce = json.load(b) + for name, value in raw_ce.items(): if name == "data": value = data_unmarshaller(value) @@ -134,7 +150,6 @@ def UnmarshalBinary( self.SetContentType(value) elif header.startswith("ce-"): self.Set(header[3:], value) - self.Set("data", data_unmarshaller(body)) def MarshalBinary( diff --git a/cloudevents/sdk/http_events.py b/cloudevents/sdk/http_events.py new file mode 100644 index 00000000..4c5de1c2 --- /dev/null +++ b/cloudevents/sdk/http_events.py @@ -0,0 +1,136 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import copy + +import json +import typing + +from cloudevents.sdk import marshaller + +from cloudevents.sdk.event import base +from cloudevents.sdk.event import v03, v1 + + +class CloudEvent(base.BaseEvent): + """ + Python-friendly cloudevent class supporting v1 events + Currently only supports binary content mode CloudEvents + """ + + def __init__( + self, + headers: dict, + data: dict, + data_unmarshaller: typing.Callable = lambda x: x + ): + """ + Event HTTP Constructor + :param headers: a dict with HTTP headers + e.g. { + "content-type": "application/cloudevents+json", + "ce-id": "16fb5f0b-211e-1102-3dfe-ea6e2806f124", + "ce-source": "", + "ce-type": "cloudevent.event.type", + "ce-specversion": "0.2" + } + :type headers: dict + :param data: a dict to be stored inside Event + :type data: dict + :param binary: a bool indicating binary events + :type binary: bool + :param data_unmarshaller: callable function for reading/extracting data + :type data_unmarshaller: typing.Callable + """ + headers = {key.lower(): value for key, value in headers.items()} + data = {key.lower(): value for key, value in data.items()} + event_version = CloudEvent.detect_event_version(headers, data) + if CloudEvent.is_binary_cloud_event(headers): + + # Headers validation for binary events + for field in base._ce_required_fields: + ce_prefixed_field = f"ce-{field}" + + # Verify field exists else throw TypeError + if ce_prefixed_field not in headers: + raise TypeError( + "parameter headers has no required attribute {0}" + .format( + ce_prefixed_field + )) + + if not isinstance(headers[ce_prefixed_field], str): + raise TypeError( + "in parameter headers attribute " + "{0} expected type str but found type {1}".format( + ce_prefixed_field, type(headers[ce_prefixed_field]) + )) + + for field in base._ce_optional_fields: + ce_prefixed_field = f"ce-{field}" + if ce_prefixed_field in headers and not \ + isinstance(headers[ce_prefixed_field], str): + raise TypeError( + "in parameter headers attribute " + "{0} expected type str but found type {1}".format( + ce_prefixed_field, type(headers[ce_prefixed_field]) + )) + + else: + # TODO: Support structured CloudEvents + raise NotImplementedError + + self.headers = copy.deepcopy(headers) + self.data = copy.deepcopy(data) + self.marshall = marshaller.NewDefaultHTTPMarshaller() + self.event_handler = event_version() + self.marshall.FromRequest( + self.event_handler, + self.headers, + self.data, + data_unmarshaller + ) + + @staticmethod + def is_binary_cloud_event(headers): + for field in base._ce_required_fields: + if f"ce-{field}" not in headers: + return False + return True + + @staticmethod + def detect_event_version(headers, data): + """ + Returns event handler depending on specversion within + headers for binary cloudevents or within data for structured + cloud events + """ + specversion = headers.get('ce-specversion', data.get('specversion')) + if specversion == '1.0': + return v1.Event + elif specversion == '0.3': + return v03.Event + else: + raise TypeError(f"specversion {specversion} " + "currently unsupported") + + def __repr__(self): + return json.dumps( + { + 'Event': { + 'headers': self.headers, + 'data': self.data + } + }, + indent=4 + ) diff --git a/cloudevents/tests/data.py b/cloudevents/tests/data.py index 6605c7f5..ffe63aee 100644 --- a/cloudevents/tests/data.py +++ b/cloudevents/tests/data.py @@ -23,7 +23,7 @@ headers = { v03.Event: { - "ce-specversion": "0.3", + "ce-specversion": "1.0", "ce-type": ce_type, "ce-id": ce_id, "ce-time": eventTime, @@ -42,7 +42,7 @@ json_ce = { v03.Event: { - "specversion": "0.3", + "specversion": "1.0", "type": ce_type, "id": ce_id, "time": eventTime, diff --git a/cloudevents/tests/test_http_events.py b/cloudevents/tests/test_http_events.py new file mode 100644 index 00000000..943e219e --- /dev/null +++ b/cloudevents/tests/test_http_events.py @@ -0,0 +1,146 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import json + +import copy + +from cloudevents.sdk.http_events import CloudEvent + +from sanic import response +from sanic import Sanic + +import pytest + + +invalid_test_headers = [ + { + "ce-source": "", + "ce-type": "cloudevent.event.type", + "ce-specversion": "1.0" + }, { + "ce-id": "my-id", + "ce-type": "cloudevent.event.type", + "ce-specversion": "1.0" + }, { + "ce-id": "my-id", + "ce-source": "", + "ce-specversion": "1.0" + }, { + "ce-id": "my-id", + "ce-source": "", + "ce-type": "cloudevent.event.type", + } +] + +test_data = { + "payload-content": "Hello World!" +} + +app = Sanic(__name__) + + +def post(url, headers, json): + return app.test_client.post(url, headers=headers, data=json) + + +@app.route("/event", ["POST"]) +async def echo(request): + assert isinstance(request.json, dict) + event = CloudEvent(dict(request.headers), request.json) + return response.text(json.dumps(event.data), headers=event.headers) + + +@pytest.mark.parametrize("headers", invalid_test_headers) +def test_invalid_binary_headers(headers): + with pytest.raises((TypeError, NotImplementedError)): + # CloudEvent constructor throws TypeError if missing required field + # and NotImplementedError because structured calls aren't + # implemented. In this instance one of the required keys should have + # prefix e-id instead of ce-id therefore it should throw + _ = CloudEvent(headers, test_data) + + +@pytest.mark.parametrize("specversion", ['1.0', '0.3']) +def test_emit_binary_event(specversion): + headers = { + "ce-id": "my-id", + "ce-source": "", + "ce-type": "cloudevent.event.type", + "ce-specversion": specversion, + "Content-Type": "application/json" + } + event = CloudEvent(headers, test_data) + _, r = app.test_client.post( + "/event", + headers=event.headers, + data=json.dumps(event.data) + ) + + # Convert byte array to dict + # e.g. r.body = b'{"payload-content": "Hello World!"}' + body = json.loads(r.body.decode('utf-8')) + + # Check response fields + for key in test_data: + assert body[key] == test_data[key] + for key in headers: + assert r.headers[key] == headers[key] + assert r.status_code == 200 + + +@pytest.mark.parametrize("specversion", ['1.0', '0.3']) +def test_missing_ce_prefix_binary_event(specversion): + headers = { + "ce-id": "my-id", + "ce-source": "", + "ce-type": "cloudevent.event.type", + "ce-specversion": specversion + } + for key in headers: + val = headers.pop(key) + + # breaking prefix e.g. e-id instead of ce-id + headers[key[1:]] = val + with pytest.raises((TypeError, NotImplementedError)): + # CloudEvent constructor throws TypeError if missing required field + # and NotImplementedError because structured calls aren't + # implemented. In this instance one of the required keys should have + # prefix e-id instead of ce-id therefore it should throw + _ = CloudEvent(headers, test_data) + + +@pytest.mark.parametrize("specversion", ['1.0', '0.3']) +def test_valid_cloud_events(specversion): + # Test creating multiple cloud events + events_queue = [] + headers = {} + num_cloudevents = 30 + for i in range(num_cloudevents): + headers = { + "ce-id": f"id{i}", + "ce-source": f"source{i}.com.test", + "ce-type": f"cloudevent.test.type", + "ce-specversion": specversion + } + data = {'payload': f"payload-{i}"} + events_queue.append(CloudEvent(headers, data)) + + for i, event in enumerate(events_queue): + headers = event.headers + data = event.data + + assert headers['ce-id'] == f"id{i}" + assert headers['ce-source'] == f"source{i}.com.test" + assert headers['ce-specversion'] == specversion + assert data['payload'] == f"payload-{i}" diff --git a/requirements/test.txt b/requirements/test.txt index e9df186e..12894086 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -7,4 +7,4 @@ pytest==4.0.0 pytest-cov==2.4.0 # web app tests sanic -aiohttp \ No newline at end of file +aiohttp diff --git a/samples/python-event-requests/cloudevent_to_request.py b/samples/python-event-requests/cloudevent_to_request.py new file mode 100644 index 00000000..4b9b5678 --- /dev/null +++ b/samples/python-event-requests/cloudevent_to_request.py @@ -0,0 +1,44 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import sys +import io +from cloudevents.sdk.http_events import CloudEvent + +import requests + +if __name__ == "__main__": + # expects a url from command line. e.g. + # python3 sample-server.py http://localhost:3000/event + if len(sys.argv) < 2: + sys.exit("Usage: python with_requests.py " + "") + + url = sys.argv[1] + + # CloudEvent headers and data + headers = { + "ce-id": "my-id", + "ce-source": "", + "ce-type": "cloudevent.event.type", + "ce-specversion": "1.0" + } + data = {"payload-content": "Hello World!"} + + # Create a CloudEvent + event = CloudEvent(headers=headers, data=data) + + # Print the created CloudEvent then send it to some url we got from + # command line + print(f"Sent {event}") + requests.post(url, headers=event.headers, json=event.data) diff --git a/samples/python-event-requests/sample-server.py b/samples/python-event-requests/sample-server.py new file mode 100644 index 00000000..fd9f1870 --- /dev/null +++ b/samples/python-event-requests/sample-server.py @@ -0,0 +1,34 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from cloudevents.sdk.http_events import CloudEvent +from flask import Flask, request +app = Flask(__name__) + + +# Create an endpoint at http://localhost:/3000/event +@app.route('/event', methods=['POST']) +def hello(): + # Convert headers to dict + headers = dict(request.headers) + + # Create a CloudEvent + event = CloudEvent(headers=headers, data=request.json) + + # Print the received CloudEvent + print(f"Received {event}") + return '', 204 + + +if __name__ == '__main__': + app.run(port=3000) From 1ad120b1a97433e7c5d86f0e5d2014bbf3f2a0d8 Mon Sep 17 00:00:00 2001 From: Curtis Mason <31265687+cumason123@users.noreply.github.com> Date: Wed, 24 Jun 2020 14:02:44 -0400 Subject: [PATCH 02/10] Implemented python properties in base.py (#41) * Added SetCloudEventVersion Signed-off-by: Curtis Mason Signed-off-by: Dustin Ingram * began adding python properties Signed-off-by: Curtis Mason Signed-off-by: Dustin Ingram * added pythonic properties to base class Signed-off-by: Curtis Mason Signed-off-by: Dustin Ingram * began testing for getters/setters Signed-off-by: Curtis Mason Signed-off-by: Dustin Ingram * added general setter tests Signed-off-by: Curtis Mason Signed-off-by: Dustin Ingram * fixed spacing in base.py Signed-off-by: Curtis Mason Signed-off-by: Dustin Ingram * added __eq__ to option and datacontentencoding property to v03 Signed-off-by: Curtis Mason Signed-off-by: Dustin Ingram * lint fixes Signed-off-by: Curtis Mason Signed-off-by: Dustin Ingram * testing extensions and old getters Signed-off-by: Curtis Mason Signed-off-by: Dustin Ingram * removed versions v01 and v02 from test_data_encaps_refs.py Signed-off-by: Curtis Mason Signed-off-by: Dustin Ingram * fixed inheritance issue in CloudEvent Signed-off-by: Curtis Mason * added prefixed_headers dict to test Signed-off-by: Curtis Mason --- cloudevents/sdk/event/base.py | 117 +++++++++++++++++---- cloudevents/sdk/event/opt.py | 6 ++ cloudevents/sdk/event/v03.py | 8 ++ cloudevents/sdk/http_events.py | 2 +- cloudevents/tests/test_data_encaps_refs.py | 114 ++++++++++++++++++++ cloudevents/tests/test_http_events.py | 9 +- 6 files changed, 232 insertions(+), 24 deletions(-) create mode 100644 cloudevents/tests/test_data_encaps_refs.py diff --git a/cloudevents/sdk/event/base.py b/cloudevents/sdk/event/base.py index a8bb099e..4ad32a5d 100644 --- a/cloudevents/sdk/event/base.py +++ b/cloudevents/sdk/event/base.py @@ -35,61 +35,141 @@ # TODO(slinkydeveloper) is this really needed? class EventGetterSetter(object): + # ce-specversion def CloudEventVersion(self) -> str: raise Exception("not implemented") - # CloudEvent attribute getters - def EventType(self) -> str: - raise Exception("not implemented") + @property + def specversion(self): + return self.CloudEventVersion() - def Source(self) -> str: + def SetCloudEventVersion(self, specversion: str) -> object: raise Exception("not implemented") - def EventID(self) -> str: - raise Exception("not implemented") + @specversion.setter + def specversion(self, value: str): + self.SetCloudEventVersion(value) - def EventTime(self) -> str: + # ce-type + def EventType(self) -> str: raise Exception("not implemented") - def SchemaURL(self) -> str: - raise Exception("not implemented") + @property + def type(self): + return self.EventType() - def Data(self) -> object: + def SetEventType(self, eventType: str) -> object: raise Exception("not implemented") - def Extensions(self) -> dict: - raise Exception("not implemented") + @type.setter + def type(self, value: str): + self.SetEventType(value) - def ContentType(self) -> str: + # ce-source + def Source(self) -> str: raise Exception("not implemented") - # CloudEvent attribute constructors - # Each setter return an instance of its class - # in order to build a pipeline of setter - def SetEventType(self, eventType: str) -> object: - raise Exception("not implemented") + @property + def source(self): + return self.Source() def SetSource(self, source: str) -> object: raise Exception("not implemented") + @source.setter + def source(self, value: str): + self.SetSource(value) + + # ce-id + def EventID(self) -> str: + raise Exception("not implemented") + + @property + def id(self): + return self.EventID() + def SetEventID(self, eventID: str) -> object: raise Exception("not implemented") + @id.setter + def id(self, value: str): + self.SetEventID(value) + + # ce-time + def EventTime(self) -> str: + raise Exception("not implemented") + + @property + def time(self): + return self.EventTime() + def SetEventTime(self, eventTime: str) -> object: raise Exception("not implemented") + @time.setter + def time(self, value: str): + self.SetEventTime(value) + + # ce-schema + def SchemaURL(self) -> str: + raise Exception("not implemented") + + @property + def schema(self) -> str: + return self.SchemaURL() + def SetSchemaURL(self, schemaURL: str) -> object: raise Exception("not implemented") + @schema.setter + def schema(self, value: str): + self.SetSchemaURL(value) + + # data + def Data(self) -> object: + raise Exception("not implemented") + + @property + def data(self) -> object: + return self.Data() + def SetData(self, data: object) -> object: raise Exception("not implemented") + @data.setter + def data(self, value: object): + self.SetData(value) + + # ce-extensions + def Extensions(self) -> dict: + raise Exception("not implemented") + + @property + def extensions(self) -> dict: + return self.Extensions() + def SetExtensions(self, extensions: dict) -> object: raise Exception("not implemented") + @extensions.setter + def extensions(self, value: dict): + self.SetExtensions(value) + + # Content-Type + def ContentType(self) -> str: + raise Exception("not implemented") + + @property + def content_type(self) -> str: + return self.ContentType() + def SetContentType(self, contentType: str) -> object: raise Exception("not implemented") + @content_type.setter + def content_type(self, value: str): + self.SetContentType(value) + class BaseEvent(EventGetterSetter): def Properties(self, with_nullable=False) -> dict: @@ -120,7 +200,6 @@ def Set(self, key: str, value: object): attr.set(value) setattr(self, formatted_key, attr) return - exts = self.Extensions() exts.update({key: value}) self.Set("extensions", exts) diff --git a/cloudevents/sdk/event/opt.py b/cloudevents/sdk/event/opt.py index 2a18a52a..bf630c32 100644 --- a/cloudevents/sdk/event/opt.py +++ b/cloudevents/sdk/event/opt.py @@ -35,3 +35,9 @@ def get(self): def required(self): return self.is_required + + def __eq__(self, obj): + return isinstance(obj, Option) and \ + obj.name == self.name and \ + obj.value == self.value and \ + obj.is_required == self.is_required diff --git a/cloudevents/sdk/event/v03.py b/cloudevents/sdk/event/v03.py index 4207e400..00ff67f8 100644 --- a/cloudevents/sdk/event/v03.py +++ b/cloudevents/sdk/event/v03.py @@ -68,6 +68,10 @@ def ContentType(self) -> str: def ContentEncoding(self) -> str: return self.ce__datacontentencoding.get() + @property + def datacontentencoding(self): + return self.ContentEncoding() + def SetEventType(self, eventType: str) -> base.BaseEvent: self.Set("type", eventType) return self @@ -107,3 +111,7 @@ def SetContentType(self, contentType: str) -> base.BaseEvent: def SetContentEncoding(self, contentEncoding: str) -> base.BaseEvent: self.Set("datacontentencoding", contentEncoding) return self + + @datacontentencoding.setter + def datacontentencoding(self, value: str): + self.SetContentEncoding(value) diff --git a/cloudevents/sdk/http_events.py b/cloudevents/sdk/http_events.py index 4c5de1c2..8d8e7fa1 100644 --- a/cloudevents/sdk/http_events.py +++ b/cloudevents/sdk/http_events.py @@ -22,7 +22,7 @@ from cloudevents.sdk.event import v03, v1 -class CloudEvent(base.BaseEvent): +class CloudEvent(): """ Python-friendly cloudevent class supporting v1 events Currently only supports binary content mode CloudEvents diff --git a/cloudevents/tests/test_data_encaps_refs.py b/cloudevents/tests/test_data_encaps_refs.py new file mode 100644 index 00000000..84bc91ef --- /dev/null +++ b/cloudevents/tests/test_data_encaps_refs.py @@ -0,0 +1,114 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import io +import json +import copy +import pytest + +from uuid import uuid4 + +from cloudevents.sdk import converters +from cloudevents.sdk import marshaller + +from cloudevents.sdk.converters import structured +from cloudevents.sdk.event import v03, v1 + + +from cloudevents.tests import data + + +@pytest.mark.parametrize("event_class", [ v03.Event, v1.Event]) +def test_general_binary_properties(event_class): + m = marshaller.NewDefaultHTTPMarshaller() + event = m.FromRequest( + event_class(), + {"Content-Type": "application/cloudevents+json"}, + io.StringIO(json.dumps(data.json_ce[event_class])), + lambda x: x.read(), + ) + + new_headers, _ = m.ToRequest(event, converters.TypeBinary, lambda x: x) + assert new_headers is not None + assert "ce-specversion" in new_headers + + # Test properties + assert event is not None + assert event.type == data.ce_type + assert event.id == data.ce_id + assert event.content_type == data.contentType + assert event.source == data.source + + # Test setters + new_type = str(uuid4()) + new_id = str(uuid4()) + new_content_type = str(uuid4()) + new_source = str(uuid4()) + + event.extensions = {'test': str(uuid4)} + event.type = new_type + event.id = new_id + event.content_type = new_content_type + event.source = new_source + + assert event is not None + assert (event.type == new_type) and (event.type == event.EventType()) + assert (event.id == new_id) and (event.id == event.EventID()) + assert (event.content_type == new_content_type) and (event.content_type == event.ContentType()) + assert (event.source == new_source) and (event.source == event.Source()) + assert event.extensions['test'] == event.Extensions()['test'] + assert (event.specversion == event.CloudEventVersion()) + + +@pytest.mark.parametrize("event_class", [v03.Event, v1.Event]) +def test_general_structured_properties(event_class): + copy_of_ce = copy.deepcopy(data.json_ce[event_class]) + m = marshaller.NewDefaultHTTPMarshaller() + http_headers = {"content-type": "application/cloudevents+json"} + event = m.FromRequest( + event_class(), http_headers, io.StringIO(json.dumps(data.json_ce[event_class])), lambda x: x.read() + ) + # Test python properties + assert event is not None + assert event.type == data.ce_type + assert event.id == data.ce_id + assert event.content_type == data.contentType + assert event.source == data.source + + new_headers, _ = m.ToRequest(event, converters.TypeStructured, lambda x: x) + for key in new_headers: + if key == "content-type": + assert new_headers[key] == http_headers[key] + continue + assert key in copy_of_ce + + # Test setters + new_type = str(uuid4()) + new_id = str(uuid4()) + new_content_type = str(uuid4()) + new_source = str(uuid4()) + + event.extensions = {'test': str(uuid4)} + event.type = new_type + event.id = new_id + event.content_type = new_content_type + event.source = new_source + + assert event is not None + assert (event.type == new_type) and (event.type == event.EventType()) + assert (event.id == new_id) and (event.id == event.EventID()) + assert (event.content_type == new_content_type) and (event.content_type == event.ContentType()) + assert (event.source == new_source) and (event.source == event.Source()) + assert event.extensions['test'] == event.Extensions()['test'] + assert (event.specversion == event.CloudEventVersion()) diff --git a/cloudevents/tests/test_http_events.py b/cloudevents/tests/test_http_events.py index 943e219e..843eb75f 100644 --- a/cloudevents/tests/test_http_events.py +++ b/cloudevents/tests/test_http_events.py @@ -101,6 +101,7 @@ def test_emit_binary_event(specversion): @pytest.mark.parametrize("specversion", ['1.0', '0.3']) def test_missing_ce_prefix_binary_event(specversion): + prefixed_headers = {} headers = { "ce-id": "my-id", "ce-source": "", @@ -108,16 +109,16 @@ def test_missing_ce_prefix_binary_event(specversion): "ce-specversion": specversion } for key in headers: - val = headers.pop(key) - + # breaking prefix e.g. e-id instead of ce-id - headers[key[1:]] = val + prefixed_headers[key[1:]] = headers[key] + with pytest.raises((TypeError, NotImplementedError)): # CloudEvent constructor throws TypeError if missing required field # and NotImplementedError because structured calls aren't # implemented. In this instance one of the required keys should have # prefix e-id instead of ce-id therefore it should throw - _ = CloudEvent(headers, test_data) + _ = CloudEvent(prefixed_headers, test_data) @pytest.mark.parametrize("specversion", ['1.0', '0.3']) From 34ed2f83170d640f98003a89f99c29401fed5834 Mon Sep 17 00:00:00 2001 From: Curtis Mason <31265687+cumason123@users.noreply.github.com> Date: Fri, 26 Jun 2020 16:52:37 -0400 Subject: [PATCH 03/10] Http structured cloudevents (#47) * Moved fields out of base & structured support base._ce_required_fields and base._ce_optional_fields were moved into event classes v03 and v1. http_events.CloudEvent class now looks for fieldnames in either headers or data, and can automatically determine whether this is a binary or structured event. Signed-off-by: Curtis Mason * testing structured Signed-off-by: Curtis Mason * added tests for structured events Signed-off-by: Curtis Mason * Added test valid structured cloudevents Signed-off-by: Curtis Mason * Created default headers arg in CloudEvent Signed-off-by: Curtis Mason * Added http_events.py sample code Signed-off-by: Curtis Mason * removed ../python-event-requests Signed-off-by: Curtis Mason * README.md nit Signed-off-by: Curtis Mason * client.py nit Signed-off-by: Curtis Mason * comment nits Signed-off-by: Curtis Mason * created __getitem__ in CloudEvent Signed-off-by: Curtis Mason * sample nits Signed-off-by: Curtis Mason * fixed structured empty data issue Signed-off-by: Curtis Mason * Added CloudEvent to README Signed-off-by: Curtis Mason * added http_msg to CloudEvent Signed-off-by: Curtis Mason * implemented ToRequest in CloudEvent Signed-off-by: Curtis Mason * testing more specversions Signed-off-by: Curtis Mason * Added sample code to README.md Signed-off-by: Curtis Mason * modified sample code Signed-off-by: Curtis Mason * added datavalidation to changelog Signed-off-by: Curtis Mason * updated README Signed-off-by: Curtis Mason * README adjustment Signed-off-by: Curtis Mason * ruler 80 adjustment on http_events Signed-off-by: Curtis Mason * style and renamed ToRequest to to_request Signed-off-by: Curtis Mason * lint fix Signed-off-by: Curtis Mason * fixed self.binary typo Signed-off-by: Curtis Mason * CHANGELOG adjustment Signed-off-by: Curtis Mason * rollback CHANGELOG Signed-off-by: Curtis Mason * Added documentation to to_request Signed-off-by: Curtis Mason * README.md adjustment Signed-off-by: Curtis Mason * renamed event_handler to event_version Signed-off-by: Curtis Mason * inlined field_name_modifier Signed-off-by: Curtis Mason * renamed test body data Signed-off-by: Curtis Mason * removed unnecessary headers from test Signed-off-by: Curtis Mason * removed field_name_modifier and fixed e.g. in client.py Signed-off-by: Curtis Mason * pylint fix Signed-off-by: Curtis Mason --- CHANGELOG.md | 2 +- README.md | 67 +++++-- cloudevents/sdk/event/base.py | 15 -- cloudevents/sdk/event/v03.py | 15 ++ cloudevents/sdk/event/v1.py | 14 ++ cloudevents/sdk/http_events.py | 151 ++++++++++----- cloudevents/tests/test_http_events.py | 180 ++++++++++++++++-- samples/http-cloudevents/README.md | 20 ++ samples/http-cloudevents/client.py | 80 ++++++++ samples/http-cloudevents/requirements.txt | 2 + .../server.py} | 17 +- .../cloudevent_to_request.py | 44 ----- 12 files changed, 464 insertions(+), 143 deletions(-) create mode 100644 samples/http-cloudevents/README.md create mode 100644 samples/http-cloudevents/client.py create mode 100644 samples/http-cloudevents/requirements.txt rename samples/{python-event-requests/sample-server.py => http-cloudevents/server.py} (76%) delete mode 100644 samples/python-event-requests/cloudevent_to_request.py diff --git a/CHANGELOG.md b/CHANGELOG.md index dac430f8..89de93f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,4 +65,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [#22]: https://github.com/cloudevents/sdk-python/pull/22 [#23]: https://github.com/cloudevents/sdk-python/pull/23 [#25]: https://github.com/cloudevents/sdk-python/pull/25 -[#27]: https://github.com/cloudevents/sdk-python/pull/27 +[#27]: https://github.com/cloudevents/sdk-python/pull/27 \ No newline at end of file diff --git a/README.md b/README.md index 5e392270..5e60476a 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,57 @@ This SDK current supports the following versions of CloudEvents: Package **cloudevents** provides primitives to work with CloudEvents specification: https://github.com/cloudevents/spec. +Sending CloudEvents: + +### Binary HTTP CloudEvent + +```python +from cloudevents.sdk.http_events import CloudEvent +import requests + + +# This data defines a binary cloudevent +headers = { + "Content-Type": "application/json", + "ce-specversion": "1.0", + "ce-type": "README.sample.binary", + "ce-id": "binary-event", + "ce-time": "2018-10-23T12:28:22.4579346Z", + "ce-source": "README", +} +data = {"message": "Hello World!"} + +event = CloudEvent(data, headers=headers) +headers, body = event.to_request() + +# POST +requests.post("", json=body, headers=headers) +``` + +### Structured HTTP CloudEvent + +```python +from cloudevents.sdk.http_events import CloudEvent +import requests + + +# This data defines a structured cloudevent +data = { + "specversion": "1.0", + "type": "README.sample.structured", + "id": "structured-event", + "source": "README", + "data": {"message": "Hello World!"} +} +event = CloudEvent(data) +headers, body = event.to_request() + +# POST +requests.post("", json=body, headers=headers) +``` + +### Event base classes usage + Parsing upstream structured Event from HTTP request: ```python @@ -68,22 +119,6 @@ event = m.FromRequest( ) ``` -Creating a minimal CloudEvent in version 0.1: - -```python -from cloudevents.sdk.event import v1 - -event = ( - v1.Event() - .SetContentType("application/json") - .SetData('{"name":"john"}') - .SetEventID("my-id") - .SetSource("from-galaxy-far-far-away") - .SetEventTime("tomorrow") - .SetEventType("cloudevent.greet.you") -) -``` - Creating HTTP request from CloudEvent: ```python diff --git a/cloudevents/sdk/event/base.py b/cloudevents/sdk/event/base.py index 4ad32a5d..1f5d8358 100644 --- a/cloudevents/sdk/event/base.py +++ b/cloudevents/sdk/event/base.py @@ -16,21 +16,6 @@ import json import typing -_ce_required_fields = { - 'id', - 'source', - 'type', - 'specversion' -} - - -_ce_optional_fields = { - 'datacontenttype', - 'schema', - 'subject', - 'time' -} - # TODO(slinkydeveloper) is this really needed? class EventGetterSetter(object): diff --git a/cloudevents/sdk/event/v03.py b/cloudevents/sdk/event/v03.py index 00ff67f8..3b6a3222 100644 --- a/cloudevents/sdk/event/v03.py +++ b/cloudevents/sdk/event/v03.py @@ -17,6 +17,21 @@ class Event(base.BaseEvent): + _ce_required_fields = { + 'id', + 'source', + 'type', + 'specversion' + } + + _ce_optional_fields = { + 'datacontentencoding', + 'datacontenttype', + 'schemaurl', + 'subject', + 'time' + } + def __init__(self): self.ce__specversion = opt.Option("specversion", "0.3", True) self.ce__id = opt.Option("id", None, True) diff --git a/cloudevents/sdk/event/v1.py b/cloudevents/sdk/event/v1.py index 655111ae..6034a9b4 100644 --- a/cloudevents/sdk/event/v1.py +++ b/cloudevents/sdk/event/v1.py @@ -17,6 +17,20 @@ class Event(base.BaseEvent): + _ce_required_fields = { + 'id', + 'source', + 'type', + 'specversion' + } + + _ce_optional_fields = { + 'datacontenttype', + 'dataschema', + 'subject', + 'time' + } + def __init__(self): self.ce__specversion = opt.Option("specversion", "1.0", True) self.ce__id = opt.Option("id", None, True) diff --git a/cloudevents/sdk/http_events.py b/cloudevents/sdk/http_events.py index 8d8e7fa1..d8db759c 100644 --- a/cloudevents/sdk/http_events.py +++ b/cloudevents/sdk/http_events.py @@ -13,12 +13,14 @@ # under the License. import copy +import io + import json import typing +from cloudevents.sdk import converters from cloudevents.sdk import marshaller -from cloudevents.sdk.event import base from cloudevents.sdk.event import v03, v1 @@ -30,12 +32,14 @@ class CloudEvent(): def __init__( self, - headers: dict, - data: dict, - data_unmarshaller: typing.Callable = lambda x: x + data: typing.Union[dict, None], + headers: dict = {}, + data_unmarshaller: typing.Callable = lambda x: x, ): """ Event HTTP Constructor + :param data: a nullable dict to be stored inside Event. + :type data: dict or None :param headers: a dict with HTTP headers e.g. { "content-type": "application/cloudevents+json", @@ -45,65 +49,118 @@ def __init__( "ce-specversion": "0.2" } :type headers: dict - :param data: a dict to be stored inside Event - :type data: dict :param binary: a bool indicating binary events :type binary: bool :param data_unmarshaller: callable function for reading/extracting data :type data_unmarshaller: typing.Callable """ + self.required_attribute_values = {} + self.optional_attribute_values = {} + if data is None: + data = {} + headers = {key.lower(): value for key, value in headers.items()} data = {key.lower(): value for key, value in data.items()} - event_version = CloudEvent.detect_event_version(headers, data) - if CloudEvent.is_binary_cloud_event(headers): - - # Headers validation for binary events - for field in base._ce_required_fields: - ce_prefixed_field = f"ce-{field}" - - # Verify field exists else throw TypeError - if ce_prefixed_field not in headers: - raise TypeError( - "parameter headers has no required attribute {0}" - .format( - ce_prefixed_field - )) - - if not isinstance(headers[ce_prefixed_field], str): - raise TypeError( - "in parameter headers attribute " - "{0} expected type str but found type {1}".format( - ce_prefixed_field, type(headers[ce_prefixed_field]) - )) - - for field in base._ce_optional_fields: - ce_prefixed_field = f"ce-{field}" - if ce_prefixed_field in headers and not \ - isinstance(headers[ce_prefixed_field], str): - raise TypeError( - "in parameter headers attribute " - "{0} expected type str but found type {1}".format( - ce_prefixed_field, type(headers[ce_prefixed_field]) - )) - else: - # TODO: Support structured CloudEvents - raise NotImplementedError + # returns an event class depending on proper version + event_version = CloudEvent.detect_event_version(headers, data) + self.isbinary = CloudEvent.is_binary_cloud_event( + event_version, + headers + ) - self.headers = copy.deepcopy(headers) - self.data = copy.deepcopy(data) self.marshall = marshaller.NewDefaultHTTPMarshaller() self.event_handler = event_version() - self.marshall.FromRequest( + + self.__event = self.marshall.FromRequest( self.event_handler, - self.headers, - self.data, + headers, + io.BytesIO(json.dumps(data).encode()), data_unmarshaller ) + # headers validation for binary events + for field in event_version._ce_required_fields: + + # prefixes with ce- if this is a binary event + fieldname = f"ce-{field}" if self.isbinary else field + + # fields_refs holds a reference to where fields should be + fields_refs = headers if self.isbinary else data + + fields_refs_name = 'headers' if self.isbinary else 'data' + + # verify field exists else throw TypeError + if fieldname not in fields_refs: + raise TypeError( + f"parameter {fields_refs_name} has no required " + f"attribute {fieldname}." + ) + + elif not isinstance(fields_refs[fieldname], str): + raise TypeError( + f"in parameter {fields_refs_name}, {fieldname} " + f"expected type str but found type " + f"{type(fields_refs[fieldname])}." + ) + + else: + self.required_attribute_values[f"ce-{field}"] = \ + fields_refs[fieldname] + + for field in event_version._ce_optional_fields: + fieldname = f"ce-{field}" if self.isbinary else field + if (fieldname in fields_refs) and not \ + isinstance(fields_refs[fieldname], str): + raise TypeError( + f"in parameter {fields_refs_name}, {fieldname} " + f"expected type str but found type " + f"{type(fields_refs[fieldname])}." + ) + else: + self.optional_attribute_values[f"ce-{field}"] = field + + # structured data is inside json resp['data'] + self.data = copy.deepcopy(data) if self.isbinary else \ + copy.deepcopy(data.get('data', {})) + + self.headers = { + **self.required_attribute_values, + **self.optional_attribute_values + } + + def to_request( + self, + data_unmarshaller: typing.Callable = lambda x: json.loads( + x.read() + .decode('utf-8') + ) + ) -> (dict, dict): + """ + Returns a tuple of HTTP headers/body dicts representing this cloudevent + + :param data_unmarshaller: callable function used to read the data io + object + :type data_unmarshaller: typing.Callable + :returns: (http_headers: dict, http_body: dict) + """ + converter_type = converters.TypeBinary if self.isbinary else \ + converters.TypeStructured + + headers, data = self.marshall.ToRequest( + self.__event, + converter_type, + data_unmarshaller + ) + data = data if self.isbinary else data_unmarshaller(data)['data'] + return headers, data + + def __getitem__(self, key): + return self.data if key == 'data' else self.headers[key] + @staticmethod - def is_binary_cloud_event(headers): - for field in base._ce_required_fields: + def is_binary_cloud_event(event_version, headers): + for field in event_version._ce_required_fields: if f"ce-{field}" not in headers: return False return True diff --git a/cloudevents/tests/test_http_events.py b/cloudevents/tests/test_http_events.py index 843eb75f..8d703949 100644 --- a/cloudevents/tests/test_http_events.py +++ b/cloudevents/tests/test_http_events.py @@ -11,6 +11,8 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. +import io + import json import copy @@ -43,6 +45,26 @@ } ] +invalid_cloudevent_request_bodie = [ + { + "source": "", + "type": "cloudevent.event.type", + "specversion": "1.0" + }, { + "id": "my-id", + "type": "cloudevent.event.type", + "specversion": "1.0" + }, { + "id": "my-id", + "source": "", + "specversion": "1.0" + }, { + "id": "my-id", + "source": "", + "type": "cloudevent.event.type", + } +] + test_data = { "payload-content": "Hello World!" } @@ -57,18 +79,28 @@ def post(url, headers, json): @app.route("/event", ["POST"]) async def echo(request): assert isinstance(request.json, dict) - event = CloudEvent(dict(request.headers), request.json) + event = CloudEvent(request.json, headers=dict(request.headers)) return response.text(json.dumps(event.data), headers=event.headers) +@pytest.mark.parametrize("body", invalid_cloudevent_request_bodie) +def test_missing_required_fields_structured(body): + with pytest.raises((TypeError, NotImplementedError)): + # CloudEvent constructor throws TypeError if missing required field + # and NotImplementedError because structured calls aren't + # implemented. In this instance one of the required keys should have + # prefix e-id instead of ce-id therefore it should throw + _ = CloudEvent(body, headers={'Content-Type': 'application/json'}) + + @pytest.mark.parametrize("headers", invalid_test_headers) -def test_invalid_binary_headers(headers): +def test_missing_required_fields_binary(headers): with pytest.raises((TypeError, NotImplementedError)): # CloudEvent constructor throws TypeError if missing required field # and NotImplementedError because structured calls aren't # implemented. In this instance one of the required keys should have # prefix e-id instead of ce-id therefore it should throw - _ = CloudEvent(headers, test_data) + _ = CloudEvent(test_data, headers=headers) @pytest.mark.parametrize("specversion", ['1.0', '0.3']) @@ -78,15 +110,15 @@ def test_emit_binary_event(specversion): "ce-source": "", "ce-type": "cloudevent.event.type", "ce-specversion": specversion, - "Content-Type": "application/json" + "Content-Type": "application/cloudevents+json" } - event = CloudEvent(headers, test_data) + event = CloudEvent(test_data, headers=headers) _, r = app.test_client.post( "/event", headers=event.headers, data=json.dumps(event.data) ) - + # Convert byte array to dict # e.g. r.body = b'{"payload-content": "Hello World!"}' body = json.loads(r.body.decode('utf-8')) @@ -95,7 +127,37 @@ def test_emit_binary_event(specversion): for key in test_data: assert body[key] == test_data[key] for key in headers: - assert r.headers[key] == headers[key] + if key != 'Content-Type': + assert r.headers[key] == headers[key] + assert r.status_code == 200 + + +@pytest.mark.parametrize("specversion", ['1.0', '0.3']) +def test_emit_structured_event(specversion): + headers = { + "Content-Type": "application/json" + } + body = { + "id": "my-id", + "source": "", + "type": "cloudevent.event.type", + "specversion": specversion, + "data": test_data + } + event = CloudEvent(body, headers=headers) + _, r = app.test_client.post( + "/event", + headers=event.headers, + data=json.dumps(event.data) + ) + + # Convert byte array to dict + # e.g. r.body = b'{"payload-content": "Hello World!"}' + body = json.loads(r.body.decode('utf-8')) + + # Check response fields + for key in test_data: + assert body[key] == test_data[key] assert r.status_code == 200 @@ -109,20 +171,20 @@ def test_missing_ce_prefix_binary_event(specversion): "ce-specversion": specversion } for key in headers: - + # breaking prefix e.g. e-id instead of ce-id prefixed_headers[key[1:]] = headers[key] - + with pytest.raises((TypeError, NotImplementedError)): # CloudEvent constructor throws TypeError if missing required field # and NotImplementedError because structured calls aren't # implemented. In this instance one of the required keys should have # prefix e-id instead of ce-id therefore it should throw - _ = CloudEvent(prefixed_headers, test_data) + _ = CloudEvent(test_data, headers=prefixed_headers) @pytest.mark.parametrize("specversion", ['1.0', '0.3']) -def test_valid_cloud_events(specversion): +def test_valid_binary_events(specversion): # Test creating multiple cloud events events_queue = [] headers = {} @@ -135,12 +197,106 @@ def test_valid_cloud_events(specversion): "ce-specversion": specversion } data = {'payload': f"payload-{i}"} - events_queue.append(CloudEvent(headers, data)) + events_queue.append(CloudEvent(data, headers=headers)) for i, event in enumerate(events_queue): headers = event.headers data = event.data + assert headers['ce-id'] == f"id{i}" + assert headers['ce-source'] == f"source{i}.com.test" + assert headers['ce-specversion'] == specversion + assert data['payload'] == f"payload-{i}" + + +@pytest.mark.parametrize("specversion", ['1.0', '0.3']) +def test_structured_to_request(specversion): + data = { + "specversion": specversion, + "type": "word.found.name", + "id": "96fb5f0b-001e-0108-6dfe-da6e2806f124", + "source": "pytest", + "data": {"message": "Hello World!"} + } + event = CloudEvent(data) + headers, body = event.to_request() + assert isinstance(body, dict) + + assert headers['content-type'] == 'application/cloudevents+json' + for key in data: + assert body[key] == data[key] + +@pytest.mark.parametrize("specversion", ['1.0', '0.3']) +def test_binary_to_request(specversion): + test_headers = { + "ce-specversion": specversion, + "ce-type": "word.found.name", + "ce-id": "96fb5f0b-001e-0108-6dfe-da6e2806f124", + "ce-source": "pytest" + } + data = { + "message": "Hello World!" + } + event = CloudEvent(data, headers=test_headers) + headers, body = event.to_request() + assert isinstance(body, dict) + + for key in data: + assert body[key] == data[key] + for key in test_headers: + assert test_headers[key] == headers[key] + + +@pytest.mark.parametrize("specversion", ['1.0', '0.3']) +def test_empty_data_structured_event(specversion): + # Testing if cloudevent breaks when no structured data field present + data = { + "specversion": specversion, + "datacontenttype": "application/json", + "type": "word.found.name", + "id": "96fb5f0b-001e-0108-6dfe-da6e2806f124", + "time": "2018-10-23T12:28:22.4579346Z", + "source": "", + "data": {"message": "Hello World!"} + } + _ = CloudEvent(data) + + +@pytest.mark.parametrize("specversion", ['1.0', '0.3']) +def test_empty_data_binary_event(specversion): + # Testing if cloudevent breaks when no structured data field present + headers = { + "Content-Type": "application/cloudevents+json", + "ce-specversion": specversion, + "ce-type": "word.found.name", + "ce-id": "96fb5f0b-001e-0108-6dfe-da6e2806f124", + "ce-time": "2018-10-23T12:28:22.4579346Z", + "ce-source": "", + } + _ = CloudEvent(None, headers=headers) + + +@pytest.mark.parametrize("specversion", ['1.0', '0.3']) +def test_valid_structured_events(specversion): + # Test creating multiple cloud events + events_queue = [] + headers = {} + num_cloudevents = 30 + for i in range(num_cloudevents): + data = { + "id": f"id{i}", + "source": f"source{i}.com.test", + "type": f"cloudevent.test.type", + "specversion": specversion, + "data": { + 'payload': f"payload-{i}" + } + } + events_queue.append(CloudEvent(data)) + + for i, event in enumerate(events_queue): + headers = event.headers + data = event.data assert headers['ce-id'] == f"id{i}" assert headers['ce-source'] == f"source{i}.com.test" assert headers['ce-specversion'] == specversion diff --git a/samples/http-cloudevents/README.md b/samples/http-cloudevents/README.md new file mode 100644 index 00000000..a72244dc --- /dev/null +++ b/samples/http-cloudevents/README.md @@ -0,0 +1,20 @@ +## Quickstart + +Install dependencies: + +```sh +pip3 install -r requirements.txt +``` + +Start server: + +```sh +python3 server.py +``` + +In a new shell, run the client code which sends a structured and binary +cloudevent to your local server: + +```sh +python3 client.py http://localhost:3000/ +``` diff --git a/samples/http-cloudevents/client.py b/samples/http-cloudevents/client.py new file mode 100644 index 00000000..89d03680 --- /dev/null +++ b/samples/http-cloudevents/client.py @@ -0,0 +1,80 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import sys +import io +from cloudevents.sdk.http_events import CloudEvent + +import requests + + +def send_binary_cloud_event(url): + # define cloudevents data + headers = { + "ce-id": "binary-event-id", + "ce-source": "localhost", + "ce-type": "template.http.binary", + "ce-specversion": "1.0", + "Content-Type": "application/json", + "ce-time": "2018-10-23T12:28:23.3464579Z" + } + data = {"payload-content": "Hello World!"} + + # create a CloudEvent + event = CloudEvent(data, headers=headers) + headers, body = event.to_request() + + # send and print event + requests.post(url, headers=headers, json=body) + print( + f"Sent {event['ce-id']} from {event['ce-source']} with " + f"{event['data']}" + ) + + +def send_structured_cloud_event(url): + # define cloudevents data + data = { + "id": "structured-event-id", + "source": "localhost", + "type": "template.http.structured", + "specversion": "1.0", + "Content-Type": "application/json", + "time": "2018-10-23T12:28:23.3464579Z", + "data": { + "payload-content": "Hello World!" + } + } + + # create a CloudEvent + event = CloudEvent(data) + headers, body = event.to_request() + + # send and print event + requests.post(url, headers=headers, json=body) + print( + f"Sent {event['ce-id']} from {event['ce-source']} with " + f"{event['data']}" + ) + + +if __name__ == "__main__": + # expects a url from command line. + # e.g. python3 client.py http://localhost:3000/ + if len(sys.argv) < 2: + sys.exit("Usage: python with_requests.py " + "") + + url = sys.argv[1] + send_binary_cloud_event(url) + send_structured_cloud_event(url) diff --git a/samples/http-cloudevents/requirements.txt b/samples/http-cloudevents/requirements.txt new file mode 100644 index 00000000..5eaf725f --- /dev/null +++ b/samples/http-cloudevents/requirements.txt @@ -0,0 +1,2 @@ +flask +requests \ No newline at end of file diff --git a/samples/python-event-requests/sample-server.py b/samples/http-cloudevents/server.py similarity index 76% rename from samples/python-event-requests/sample-server.py rename to samples/http-cloudevents/server.py index fd9f1870..11e48419 100644 --- a/samples/python-event-requests/sample-server.py +++ b/samples/http-cloudevents/server.py @@ -16,17 +16,18 @@ app = Flask(__name__) -# Create an endpoint at http://localhost:/3000/event -@app.route('/event', methods=['POST']) -def hello(): - # Convert headers to dict +# create an endpoint at http://localhost:/3000/ +@app.route('/', methods=['POST']) +def home(): + # convert headers to dict headers = dict(request.headers) - - # Create a CloudEvent + print(request.json) + print(headers) + # create a CloudEvent event = CloudEvent(headers=headers, data=request.json) - # Print the received CloudEvent - print(f"Received {event}") + # print the received CloudEvent + print(f"Received CloudEvent {event}") return '', 204 diff --git a/samples/python-event-requests/cloudevent_to_request.py b/samples/python-event-requests/cloudevent_to_request.py deleted file mode 100644 index 4b9b5678..00000000 --- a/samples/python-event-requests/cloudevent_to_request.py +++ /dev/null @@ -1,44 +0,0 @@ -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -import sys -import io -from cloudevents.sdk.http_events import CloudEvent - -import requests - -if __name__ == "__main__": - # expects a url from command line. e.g. - # python3 sample-server.py http://localhost:3000/event - if len(sys.argv) < 2: - sys.exit("Usage: python with_requests.py " - "") - - url = sys.argv[1] - - # CloudEvent headers and data - headers = { - "ce-id": "my-id", - "ce-source": "", - "ce-type": "cloudevent.event.type", - "ce-specversion": "1.0" - } - data = {"payload-content": "Hello World!"} - - # Create a CloudEvent - event = CloudEvent(headers=headers, data=data) - - # Print the created CloudEvent then send it to some url we got from - # command line - print(f"Sent {event}") - requests.post(url, headers=event.headers, json=event.data) From 7e054fa07aa022614de3d038e0309f928d239963 Mon Sep 17 00:00:00 2001 From: Evan Anderson Date: Wed, 8 Jul 2020 16:16:56 -0700 Subject: [PATCH 04/10] Update types and handle data_base64 structured. (#34) * Update types and handle data_base64 structured. - Add sane defaults for encoding - Unfortunately, defaults for structured and binary need to be *different* - Push types through interfaces - Make it easy to call 'ToRequest' using Marshaller defaults - Add tests for above Signed-off-by: Evan Anderson * Fix lint warnings due to changes to W503/W504 See https://gitlab.com/pycqa/flake8/-/issues/466 for details. Signed-off-by: Evan Anderson * Adopt di's suggestions. Signed-off-by: Evan Anderson * Fix lint. Signed-off-by: Evan Anderson * Move types to another package. Signed-off-by: Evan Anderson * Adjust CloudEvent class in http_events.py to support binary data as well as JSON. Signed-off-by: Evan Anderson * Apply suggested changes by MacrBoissonneault Signed-off-by: Evan Anderson * Fix samples as well. Signed-off-by: Evan Anderson * Fix lint. Apparently, we can complain about formating issues, but a human has to fix them. Signed-off-by: Evan Anderson * Add test for binary encoding of messages. Fix usability of binary detection in MarshalJSON to support memoryview. Signed-off-by: Evan Anderson * Fix errors noticed by cumason123 Signed-off-by: Evan Anderson --- README.md | 99 +++---- cloudevents/sdk/converters/binary.py | 10 +- cloudevents/sdk/converters/structured.py | 12 +- cloudevents/sdk/event/base.py | 56 +++- cloudevents/sdk/http_events.py | 268 ++++++++---------- cloudevents/sdk/marshaller.py | 23 +- cloudevents/sdk/types.py | 24 ++ cloudevents/tests/test_data_encaps_refs.py | 19 +- .../test_event_from_request_converter.py | 8 +- cloudevents/tests/test_event_pipeline.py | 43 ++- .../tests/test_event_to_request_converter.py | 8 +- cloudevents/tests/test_http_events.py | 150 ++++++---- samples/http-cloudevents/client.py | 39 ++- samples/http-cloudevents/server.py | 7 +- .../http-cloudevents/test_verify_sample.py | 0 tox.ini | 3 +- 16 files changed, 433 insertions(+), 336 deletions(-) create mode 100644 cloudevents/sdk/types.py create mode 100644 samples/http-cloudevents/test_verify_sample.py diff --git a/README.md b/README.md index 5e60476a..a52bdcb2 100644 --- a/README.md +++ b/README.md @@ -19,26 +19,25 @@ Sending CloudEvents: ### Binary HTTP CloudEvent ```python +from cloudevents.sdk import converters from cloudevents.sdk.http_events import CloudEvent import requests # This data defines a binary cloudevent -headers = { +attributes = { "Content-Type": "application/json", - "ce-specversion": "1.0", - "ce-type": "README.sample.binary", - "ce-id": "binary-event", - "ce-time": "2018-10-23T12:28:22.4579346Z", - "ce-source": "README", + "type": "README.sample.binary", + "id": "binary-event", + "source": "README", } data = {"message": "Hello World!"} -event = CloudEvent(data, headers=headers) -headers, body = event.to_request() +event = CloudEvent(attributes, data) +headers, body = event.to_http(converters.TypeBinary) # POST -requests.post("", json=body, headers=headers) +requests.post("", data=body, headers=headers) ``` ### Structured HTTP CloudEvent @@ -49,18 +48,17 @@ import requests # This data defines a structured cloudevent -data = { - "specversion": "1.0", +attributes = { "type": "README.sample.structured", "id": "structured-event", "source": "README", - "data": {"message": "Hello World!"} } -event = CloudEvent(data) -headers, body = event.to_request() +data = {"message": "Hello World!"} +event = CloudEvent(attributes, data) +headers, body = event.to_http() # POST -requests.post("", json=body, headers=headers) +requests.post("", data=body, headers=headers) ``` ### Event base classes usage @@ -78,19 +76,16 @@ m = marshaller.NewDefaultHTTPMarshaller() event = m.FromRequest( v1.Event(), {"content-type": "application/cloudevents+json"}, - io.StringIO( - """ - { - "specversion": "1.0", - "datacontenttype": "application/json", - "type": "word.found.name", - "id": "96fb5f0b-001e-0108-6dfe-da6e2806f124", - "time": "2018-10-23T12:28:22.4579346Z", - "source": "" - } - """ - ), - lambda x: x.read(), + """ + { + "specversion": "1.0", + "datacontenttype": "application/json", + "type": "word.found.name", + "id": "96fb5f0b-001e-0108-6dfe-da6e2806f124", + "time": "2018-10-23T12:28:22.4579346Z", + "source": "" + } + """, ) ``` @@ -108,14 +103,14 @@ event = m.FromRequest( v1.Event(), { "ce-specversion": "1.0", - "content-type": "application/json", + "content-type": "application/octet-stream", "ce-type": "word.found.name", "ce-id": "96fb5f0b-001e-0108-6dfe-da6e2806f124", "ce-time": "2018-10-23T12:28:22.4579346Z", "ce-source": "", }, - io.BytesIO(b"this is where your CloudEvent data"), - lambda x: x.read(), + b"this is where your CloudEvent data is, including image/jpeg", + lambda x: x, # Do not decode body as JSON ) ``` @@ -137,7 +132,7 @@ event = ( .SetEventType("cloudevent.greet.you") ) -m = marshaller.NewHTTPMarshaller([structured.NewJSONHTTPCloudEventConverter()]) +m = marshaller.NewDefaultHTTPMarshaller() headers, body = m.ToRequest(event, converters.TypeStructured, lambda x: x) ``` @@ -155,30 +150,33 @@ One of popular framework is [`requests`](http://docs.python-requests.org/en/mast The code below shows how integrate both libraries in order to convert a CloudEvent into an HTTP request: ```python -def run_binary(event, url): - binary_headers, binary_data = http_marshaller.ToRequest( - event, converters.TypeBinary, json.dumps) +from cloudevents.sdk.http_events import CloudEvent +from cloudevents.sdk.types import converters + +def run_binary(event: CloudEvent, url): + # If event.data is a custom type, set data_marshaller as well + headers, data = event.to_http(converters.TypeBinary) print("binary CloudEvent") - for k, v in binary_headers.items(): + for k, v in headers.items(): print("{0}: {1}\r\n".format(k, v)) - print(binary_data.getvalue()) + print(data) response = requests.post(url, - headers=binary_headers, - data=binary_data.getvalue()) + headers=headers, + data=data) response.raise_for_status() -def run_structured(event, url): - structured_headers, structured_data = http_marshaller.ToRequest( - event, converters.TypeStructured, json.dumps - ) +def run_structured(event: CloudEvent, url): + # If event.data is a custom type, set data_marshaller as well + headers, data = event.to_http(converters.TypeStructured) + print("structured CloudEvent") - print(structured_data.getvalue()) + print(data) response = requests.post(url, - headers=structured_headers, - data=structured_data.getvalue()) + headers=headers, + data=data) response.raise_for_status() ``` @@ -187,18 +185,13 @@ Complete example of turning a CloudEvent into a request you can find [here](samp #### Request to CloudEvent -The code below shows how integrate both libraries in order to create a CloudEvent from an HTTP request: +The code below shows how integrate both libraries in order to create a CloudEvent from an HTTP response: ```python response = requests.get(url) response.raise_for_status() - headers = response.headers - data = io.BytesIO(response.content) - event = v1.Event() - http_marshaller = marshaller.NewDefaultHTTPMarshaller() - event = http_marshaller.FromRequest( - event, headers, data, json.load) + event = CloudEvent.from_http(response.content, response.headers) ``` Complete example of turning a CloudEvent into a request you can find [here](samples/python-requests/request_to_cloudevent.py). diff --git a/cloudevents/sdk/converters/binary.py b/cloudevents/sdk/converters/binary.py index 7bc0025e..dfb08fe6 100644 --- a/cloudevents/sdk/converters/binary.py +++ b/cloudevents/sdk/converters/binary.py @@ -14,7 +14,7 @@ import typing -from cloudevents.sdk import exceptions +from cloudevents.sdk import exceptions, types from cloudevents.sdk.converters import base from cloudevents.sdk.event import base as event_base from cloudevents.sdk.event import v03, v1 @@ -36,7 +36,7 @@ def read( event: event_base.BaseEvent, headers: dict, body: typing.IO, - data_unmarshaller: typing.Callable, + data_unmarshaller: types.UnmarshallerType, ) -> event_base.BaseEvent: if type(event) not in self.SUPPORTED_VERSIONS: raise exceptions.UnsupportedEvent(type(event)) @@ -44,8 +44,10 @@ def read( return event def write( - self, event: event_base.BaseEvent, data_marshaller: typing.Callable - ) -> (dict, typing.IO): + self, + event: event_base.BaseEvent, + data_marshaller: types.MarshallerType + ) -> (dict, bytes): return event.MarshalBinary(data_marshaller) diff --git a/cloudevents/sdk/converters/structured.py b/cloudevents/sdk/converters/structured.py index 589a977a..435e004f 100644 --- a/cloudevents/sdk/converters/structured.py +++ b/cloudevents/sdk/converters/structured.py @@ -14,6 +14,8 @@ import typing +from cloudevents.sdk import types + from cloudevents.sdk.converters import base from cloudevents.sdk.event import base as event_base @@ -35,16 +37,18 @@ def read( event: event_base.BaseEvent, headers: dict, body: typing.IO, - data_unmarshaller: typing.Callable, + data_unmarshaller: types.UnmarshallerType, ) -> event_base.BaseEvent: event.UnmarshalJSON(body, data_unmarshaller) return event def write( - self, event: event_base.BaseEvent, data_marshaller: typing.Callable - ) -> (dict, typing.IO): + self, + event: event_base.BaseEvent, + data_marshaller: types.MarshallerType + ) -> (dict, bytes): http_headers = {"content-type": self.MIME_TYPE} - return http_headers, event.MarshalJSON(data_marshaller) + return http_headers, event.MarshalJSON(data_marshaller).encode("utf-8") def NewJSONHTTPCloudEventConverter() -> JSONHTTPCloudEventConverter: diff --git a/cloudevents/sdk/event/base.py b/cloudevents/sdk/event/base.py index 1f5d8358..25ba6bd7 100644 --- a/cloudevents/sdk/event/base.py +++ b/cloudevents/sdk/event/base.py @@ -12,10 +12,12 @@ # License for the specific language governing permissions and limitations # under the License. -import io +import base64 import json import typing +from cloudevents.sdk import types + # TODO(slinkydeveloper) is this really needed? class EventGetterSetter(object): @@ -189,25 +191,47 @@ def Set(self, key: str, value: object): exts.update({key: value}) self.Set("extensions", exts) - def MarshalJSON(self, data_marshaller: typing.Callable) -> typing.IO: + def MarshalJSON(self, data_marshaller: types.MarshallerType) -> str: + if data_marshaller is None: + data_marshaller = lambda x: x # noqa: E731 props = self.Properties() - props["data"] = data_marshaller(props.get("data")) - return io.BytesIO(json.dumps(props).encode("utf-8")) + if "data" in props: + data = data_marshaller(props.pop("data")) + if isinstance(data, (bytes, bytes, memoryview)): + props["data_base64"] = base64.b64encode(data).decode("ascii") + else: + props["data"] = data + return json.dumps(props) + + def UnmarshalJSON( + self, + b: typing.Union[str, bytes], + data_unmarshaller: types.UnmarshallerType + ): + raw_ce = json.loads(b) - def UnmarshalJSON(self, b: typing.IO, data_unmarshaller: typing.Callable): - raw_ce = json.load(b) + missing_fields = self._ce_required_fields - raw_ce.keys() + if len(missing_fields) > 0: + raise ValueError(f"Missing required attributes: {missing_fields}") for name, value in raw_ce.items(): if name == "data": - value = data_unmarshaller(value) + # Use the user-provided serializer, which may have customized + # JSON decoding + value = data_unmarshaller(json.dumps(value)) + if name == "data_base64": + value = data_unmarshaller(base64.b64decode(value)) + name = "data" self.Set(name, value) def UnmarshalBinary( self, headers: dict, - body: typing.IO, - data_unmarshaller: typing.Callable + body: typing.Union[bytes, str], + data_unmarshaller: types.UnmarshallerType ): + if 'ce-specversion' not in headers: + raise ValueError("Missing required attribute: 'specversion'") for header, value in headers.items(): header = header.lower() if header == "content-type": @@ -215,11 +239,16 @@ def UnmarshalBinary( elif header.startswith("ce-"): self.Set(header[3:], value) self.Set("data", data_unmarshaller(body)) + missing_attrs = self._ce_required_fields - self.Properties().keys() + if len(missing_attrs) > 0: + raise ValueError(f"Missing required attributes: {missing_attrs}") def MarshalBinary( self, - data_marshaller: typing.Callable - ) -> (dict, object): + data_marshaller: types.MarshallerType + ) -> (dict, bytes): + if data_marshaller is None: + data_marshaller = json.dumps headers = {} if self.ContentType(): headers["content-type"] = self.ContentType() @@ -233,4 +262,7 @@ def MarshalBinary( headers["ce-{0}".format(key)] = value data, _ = self.Get("data") - return headers, data_marshaller(data) + data = data_marshaller(data) + if isinstance(data, str): # Convenience method for json.dumps + data = data.encode("utf-8") + return headers, data diff --git a/cloudevents/sdk/http_events.py b/cloudevents/sdk/http_events.py index d8db759c..b27c1a26 100644 --- a/cloudevents/sdk/http_events.py +++ b/cloudevents/sdk/http_events.py @@ -11,183 +11,157 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -import copy - -import io +import datetime import json import typing +import uuid from cloudevents.sdk import converters from cloudevents.sdk import marshaller +from cloudevents.sdk import types from cloudevents.sdk.event import v03, v1 +_marshaller_by_format = { + converters.TypeStructured: lambda x: x, + converters.TypeBinary: json.dumps, +} +_required_by_version = { + "1.0": v1.Event._ce_required_fields, + "0.3": v03.Event._ce_required_fields, +} +_obj_by_version = {"1.0": v1.Event, "0.3": v03.Event} + + +def _json_or_string(content: typing.Union[str, bytes]): + if len(content) == 0: + return None + try: + return json.loads(content) + except json.JSONDecodeError: + return content + class CloudEvent(): """ Python-friendly cloudevent class supporting v1 events - Currently only supports binary content mode CloudEvents + Supports both binary and structured mode CloudEvents """ + @classmethod + def from_http( + cls, + data: typing.Union[str, bytes], + headers: typing.Dict[str, str], + data_unmarshaller: types.UnmarshallerType = None + ): + """Unwrap a CloudEvent (binary or structured) from an HTTP request. + :param data: the HTTP request body + :type data: typing.IO + :param headers: the HTTP headers + :type headers: typing.Dict[str, str] + :param data_unmarshaller: Function to decode data into a python object. + :type data_unmarshaller: types.UnmarshallerType + """ + if data_unmarshaller is None: + data_unmarshaller = _json_or_string + + event = marshaller.NewDefaultHTTPMarshaller().FromRequest( + v1.Event(), headers, data, data_unmarshaller) + attrs = event.Properties() + attrs.pop('data', None) + attrs.pop('extensions', None) + attrs.update(**event.extensions) + + return cls(attrs, event.data) + def __init__( - self, - data: typing.Union[dict, None], - headers: dict = {}, - data_unmarshaller: typing.Callable = lambda x: x, + self, + attributes: typing.Dict[str, str], + data: typing.Any = None ): """ - Event HTTP Constructor - :param data: a nullable dict to be stored inside Event. - :type data: dict or None - :param headers: a dict with HTTP headers + Event Constructor + :param attributes: a dict with HTTP headers e.g. { "content-type": "application/cloudevents+json", - "ce-id": "16fb5f0b-211e-1102-3dfe-ea6e2806f124", - "ce-source": "", - "ce-type": "cloudevent.event.type", - "ce-specversion": "0.2" + "id": "16fb5f0b-211e-1102-3dfe-ea6e2806f124", + "source": "", + "type": "cloudevent.event.type", + "specversion": "0.2" } - :type headers: dict - :param binary: a bool indicating binary events - :type binary: bool - :param data_unmarshaller: callable function for reading/extracting data - :type data_unmarshaller: typing.Callable + :type attributes: typing.Dict[str, str] + :param data: The payload of the event, as a python object + :type data: typing.Any """ - self.required_attribute_values = {} - self.optional_attribute_values = {} - if data is None: - data = {} - - headers = {key.lower(): value for key, value in headers.items()} - data = {key.lower(): value for key, value in data.items()} - - # returns an event class depending on proper version - event_version = CloudEvent.detect_event_version(headers, data) - self.isbinary = CloudEvent.is_binary_cloud_event( - event_version, - headers - ) - - self.marshall = marshaller.NewDefaultHTTPMarshaller() - self.event_handler = event_version() - - self.__event = self.marshall.FromRequest( - self.event_handler, - headers, - io.BytesIO(json.dumps(data).encode()), - data_unmarshaller - ) - - # headers validation for binary events - for field in event_version._ce_required_fields: - - # prefixes with ce- if this is a binary event - fieldname = f"ce-{field}" if self.isbinary else field - - # fields_refs holds a reference to where fields should be - fields_refs = headers if self.isbinary else data - - fields_refs_name = 'headers' if self.isbinary else 'data' - - # verify field exists else throw TypeError - if fieldname not in fields_refs: - raise TypeError( - f"parameter {fields_refs_name} has no required " - f"attribute {fieldname}." - ) - - elif not isinstance(fields_refs[fieldname], str): - raise TypeError( - f"in parameter {fields_refs_name}, {fieldname} " - f"expected type str but found type " - f"{type(fields_refs[fieldname])}." - ) - - else: - self.required_attribute_values[f"ce-{field}"] = \ - fields_refs[fieldname] - - for field in event_version._ce_optional_fields: - fieldname = f"ce-{field}" if self.isbinary else field - if (fieldname in fields_refs) and not \ - isinstance(fields_refs[fieldname], str): - raise TypeError( - f"in parameter {fields_refs_name}, {fieldname} " - f"expected type str but found type " - f"{type(fields_refs[fieldname])}." - ) - else: - self.optional_attribute_values[f"ce-{field}"] = field - - # structured data is inside json resp['data'] - self.data = copy.deepcopy(data) if self.isbinary else \ - copy.deepcopy(data.get('data', {})) - - self.headers = { - **self.required_attribute_values, - **self.optional_attribute_values - } - - def to_request( + self._attributes = {k.lower(): v for k, v in attributes.items()} + self.data = data + if 'specversion' not in self._attributes: + self._attributes['specversion'] = "1.0" + if 'id' not in self._attributes: + self._attributes['id'] = str(uuid.uuid4()) + if 'time' not in self._attributes: + self._attributes['time'] = datetime.datetime.now( + datetime.timezone.utc).isoformat() + + if self._attributes['specversion'] not in _required_by_version: + raise ValueError( + f"Invalid specversion: {self._attributes['specversion']}") + # There is no good way to default 'source' and 'type', so this + # checks for those (or any new required attributes). + required_set = _required_by_version[self._attributes['specversion']] + if not required_set <= self._attributes.keys(): + raise ValueError( + f"Missing required keys: {required_set - attributes.keys()}") + + def to_http( self, - data_unmarshaller: typing.Callable = lambda x: json.loads( - x.read() - .decode('utf-8') - ) - ) -> (dict, dict): + format: str = converters.TypeStructured, + data_marshaller: types.MarshallerType = None + ) -> (dict, typing.Union[bytes, str]): """ Returns a tuple of HTTP headers/body dicts representing this cloudevent - :param data_unmarshaller: callable function used to read the data io - object - :type data_unmarshaller: typing.Callable - :returns: (http_headers: dict, http_body: dict) + :param format: constant specifying an encoding format + :type format: str + :param data_unmarshaller: Function used to read the data to string. + :type data_unmarshaller: types.UnmarshallerType + :returns: (http_headers: dict, http_body: bytes or str) """ - converter_type = converters.TypeBinary if self.isbinary else \ - converters.TypeStructured + if data_marshaller is None: + data_marshaller = _marshaller_by_format[format] + if self._attributes["specversion"] not in _obj_by_version: + raise ValueError( + f"Unsupported specversion: {self._attributes['specversion']}") + + event = _obj_by_version[self._attributes["specversion"]]() + for k, v in self._attributes.items(): + event.Set(k, v) + event.data = self.data + + return marshaller.NewDefaultHTTPMarshaller().ToRequest( + event, format, data_marshaller) + + # Data access is handled via `.data` member + # Attribute access is managed via Mapping type + def __getitem__(self, key): + return self._attributes[key] - headers, data = self.marshall.ToRequest( - self.__event, - converter_type, - data_unmarshaller - ) - data = data if self.isbinary else data_unmarshaller(data)['data'] - return headers, data + def __setitem__(self, key, value): + self._attributes[key] = value - def __getitem__(self, key): - return self.data if key == 'data' else self.headers[key] + def __delitem__(self, key): + del self._attributes[key] - @staticmethod - def is_binary_cloud_event(event_version, headers): - for field in event_version._ce_required_fields: - if f"ce-{field}" not in headers: - return False - return True + def __iter__(self): + return iter(self._attributes) - @staticmethod - def detect_event_version(headers, data): - """ - Returns event handler depending on specversion within - headers for binary cloudevents or within data for structured - cloud events - """ - specversion = headers.get('ce-specversion', data.get('specversion')) - if specversion == '1.0': - return v1.Event - elif specversion == '0.3': - return v03.Event - else: - raise TypeError(f"specversion {specversion} " - "currently unsupported") + def __len__(self): + return len(self._attributes) + + def __contains__(self, key): + return key in self._attributes def __repr__(self): - return json.dumps( - { - 'Event': { - 'headers': self.headers, - 'data': self.data - } - }, - indent=4 - ) + return self.to_http()[1].decode() diff --git a/cloudevents/sdk/marshaller.py b/cloudevents/sdk/marshaller.py index a54a1359..1276dca3 100644 --- a/cloudevents/sdk/marshaller.py +++ b/cloudevents/sdk/marshaller.py @@ -12,9 +12,10 @@ # License for the specific language governing permissions and limitations # under the License. +import json import typing -from cloudevents.sdk import exceptions +from cloudevents.sdk import exceptions, types from cloudevents.sdk.converters import base from cloudevents.sdk.converters import binary @@ -42,8 +43,8 @@ def FromRequest( self, event: event_base.BaseEvent, headers: dict, - body: typing.IO, - data_unmarshaller: typing.Callable, + body: typing.Union[str, bytes], + data_unmarshaller: types.UnmarshallerType = json.loads, ) -> event_base.BaseEvent: """ Reads a CloudEvent from an HTTP headers and request body @@ -51,8 +52,8 @@ def FromRequest( :type event: cloudevents.sdk.event.base.BaseEvent :param headers: a dict-like HTTP headers :type headers: dict - :param body: a stream-like HTTP request body - :type body: typing.IO + :param body: an HTTP request body as a string or bytes + :type body: typing.Union[str, bytes] :param data_unmarshaller: a callable-like unmarshaller the CloudEvent data :return: a CloudEvent @@ -78,9 +79,9 @@ def FromRequest( def ToRequest( self, event: event_base.BaseEvent, - converter_type: str, - data_marshaller: typing.Callable, - ) -> (dict, typing.IO): + converter_type: str = None, + data_marshaller: types.MarshallerType = None, + ) -> (dict, bytes): """ Writes a CloudEvent into a HTTP-ready form of headers and request body :param event: CloudEvent @@ -92,9 +93,13 @@ def ToRequest( :return: dict of HTTP headers and stream of HTTP request body :rtype: tuple """ - if not isinstance(data_marshaller, typing.Callable): + if (data_marshaller is not None + and not isinstance(data_marshaller, typing.Callable)): raise exceptions.InvalidDataMarshaller() + if converter_type is None: + converter_type = self.__converters[0].TYPE + if converter_type in self.__converters_by_type: cnvrtr = self.__converters_by_type[converter_type] return cnvrtr.write(event, data_marshaller) diff --git a/cloudevents/sdk/types.py b/cloudevents/sdk/types.py new file mode 100644 index 00000000..82885842 --- /dev/null +++ b/cloudevents/sdk/types.py @@ -0,0 +1,24 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import typing + +# Use consistent types for marshal and unmarshal functions across +# both JSON and Binary format. + +MarshallerType = typing.Optional[ + typing.Callable[[typing.Any], typing.Union[bytes, str]]] +UnmarshallerType = typing.Optional[ + typing.Callable[ + [typing.Union[bytes, str]], typing.Any]] diff --git a/cloudevents/tests/test_data_encaps_refs.py b/cloudevents/tests/test_data_encaps_refs.py index 84bc91ef..3afb40e8 100644 --- a/cloudevents/tests/test_data_encaps_refs.py +++ b/cloudevents/tests/test_data_encaps_refs.py @@ -29,16 +29,16 @@ from cloudevents.tests import data -@pytest.mark.parametrize("event_class", [ v03.Event, v1.Event]) +@pytest.mark.parametrize("event_class", [v03.Event, v1.Event]) def test_general_binary_properties(event_class): m = marshaller.NewDefaultHTTPMarshaller() event = m.FromRequest( event_class(), {"Content-Type": "application/cloudevents+json"}, - io.StringIO(json.dumps(data.json_ce[event_class])), + json.dumps(data.json_ce[event_class]), lambda x: x.read(), ) - + new_headers, _ = m.ToRequest(event, converters.TypeBinary, lambda x: x) assert new_headers is not None assert "ce-specversion" in new_headers @@ -55,7 +55,7 @@ def test_general_binary_properties(event_class): new_id = str(uuid4()) new_content_type = str(uuid4()) new_source = str(uuid4()) - + event.extensions = {'test': str(uuid4)} event.type = new_type event.id = new_id @@ -65,7 +65,8 @@ def test_general_binary_properties(event_class): assert event is not None assert (event.type == new_type) and (event.type == event.EventType()) assert (event.id == new_id) and (event.id == event.EventID()) - assert (event.content_type == new_content_type) and (event.content_type == event.ContentType()) + assert (event.content_type == new_content_type) and ( + event.content_type == event.ContentType()) assert (event.source == new_source) and (event.source == event.Source()) assert event.extensions['test'] == event.Extensions()['test'] assert (event.specversion == event.CloudEventVersion()) @@ -77,7 +78,8 @@ def test_general_structured_properties(event_class): m = marshaller.NewDefaultHTTPMarshaller() http_headers = {"content-type": "application/cloudevents+json"} event = m.FromRequest( - event_class(), http_headers, io.StringIO(json.dumps(data.json_ce[event_class])), lambda x: x.read() + event_class(), http_headers, json.dumps( + data.json_ce[event_class]), lambda x: x ) # Test python properties assert event is not None @@ -104,11 +106,12 @@ def test_general_structured_properties(event_class): event.id = new_id event.content_type = new_content_type event.source = new_source - + assert event is not None assert (event.type == new_type) and (event.type == event.EventType()) assert (event.id == new_id) and (event.id == event.EventID()) - assert (event.content_type == new_content_type) and (event.content_type == event.ContentType()) + assert (event.content_type == new_content_type) and ( + event.content_type == event.ContentType()) assert (event.source == new_source) and (event.source == event.Source()) assert event.extensions['test'] == event.Extensions()['test'] assert (event.specversion == event.CloudEventVersion()) diff --git a/cloudevents/tests/test_event_from_request_converter.py b/cloudevents/tests/test_event_from_request_converter.py index 65a89703..7163533c 100644 --- a/cloudevents/tests/test_event_from_request_converter.py +++ b/cloudevents/tests/test_event_from_request_converter.py @@ -51,7 +51,7 @@ def test_structured_converter_upstream(event_class): event = m.FromRequest( event_class(), {"Content-Type": "application/cloudevents+json"}, - io.StringIO(json.dumps(data.json_ce[event_class])), + json.dumps(data.json_ce[event_class]), lambda x: x.read(), ) @@ -68,7 +68,7 @@ def test_default_http_marshaller_with_structured(event_class): event = m.FromRequest( event_class(), {"Content-Type": "application/cloudevents+json"}, - io.StringIO(json.dumps(data.json_ce[event_class])), + json.dumps(data.json_ce[event_class]), lambda x: x.read(), ) assert event is not None @@ -83,8 +83,8 @@ def test_default_http_marshaller_with_binary(event_class): event = m.FromRequest( event_class(), data.headers[event_class], - io.StringIO(json.dumps(data.body)), - json.load + json.dumps(data.body), + json.loads ) assert event is not None assert event.EventType() == data.ce_type diff --git a/cloudevents/tests/test_event_pipeline.py b/cloudevents/tests/test_event_pipeline.py index 09f029b2..43fc3f7e 100644 --- a/cloudevents/tests/test_event_pipeline.py +++ b/cloudevents/tests/test_event_pipeline.py @@ -46,8 +46,8 @@ def test_event_pipeline_upstream(event_class): assert "ce-id" in new_headers assert "ce-time" in new_headers assert "content-type" in new_headers - assert isinstance(body, str) - assert data.body == body + assert isinstance(body, bytes) + assert data.body == body.decode("utf-8") def test_extensions_are_set_upstream(): @@ -62,3 +62,42 @@ def test_extensions_are_set_upstream(): assert event.Extensions() == extensions assert "ce-extension-key" in new_headers + + +def test_binary_event_v1(): + event = ( + v1.Event() + .SetContentType("application/octet-stream") + .SetData(b'\x00\x01') + ) + m = marshaller.NewHTTPMarshaller([structured.NewJSONHTTPCloudEventConverter()]) + + _, body = m.ToRequest(event, converters.TypeStructured, lambda x: x) + assert isinstance(body, bytes) + content = json.loads(body) + assert "data" not in content + assert content["data_base64"] == "AAE=", f"Content is: {content}" + +def test_object_event_v1(): + event = ( + v1.Event() + .SetContentType("application/json") + .SetData({"name": "john"}) + ) + + m = marshaller.NewDefaultHTTPMarshaller() + + _, structuredBody = m.ToRequest(event) + assert isinstance(structuredBody, bytes) + structuredObj = json.loads(structuredBody) + errorMsg = f"Body was {structuredBody}, obj is {structuredObj}" + assert isinstance(structuredObj, dict), errorMsg + assert isinstance(structuredObj["data"], dict), errorMsg + assert len(structuredObj["data"]) == 1, errorMsg + assert structuredObj["data"]["name"] == "john", errorMsg + + headers, binaryBody = m.ToRequest(event, converters.TypeBinary) + assert isinstance(headers, dict) + assert isinstance(binaryBody, bytes) + assert headers["content-type"] == "application/json" + assert binaryBody == b'{"name": "john"}', f"Binary is {binaryBody!r}" diff --git a/cloudevents/tests/test_event_to_request_converter.py b/cloudevents/tests/test_event_to_request_converter.py index 06f2e679..f46746b7 100644 --- a/cloudevents/tests/test_event_to_request_converter.py +++ b/cloudevents/tests/test_event_to_request_converter.py @@ -32,8 +32,7 @@ def test_binary_event_to_request_upstream(event_class): event = m.FromRequest( event_class(), {"Content-Type": "application/cloudevents+json"}, - io.StringIO(json.dumps(data.json_ce[event_class])), - lambda x: x.read(), + json.dumps(data.json_ce[event_class]), ) assert event is not None @@ -54,10 +53,7 @@ def test_structured_event_to_request_upstream(event_class): event = m.FromRequest( event_class(), http_headers, - io.StringIO( - json.dumps(data.json_ce[event_class]) - ), - lambda x: x.read() + json.dumps(data.json_ce[event_class]) ) assert event is not None assert event.EventType() == data.ce_type diff --git a/cloudevents/tests/test_http_events.py b/cloudevents/tests/test_http_events.py index 8d703949..57822a54 100644 --- a/cloudevents/tests/test_http_events.py +++ b/cloudevents/tests/test_http_events.py @@ -11,13 +11,14 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -import io - -import json +import bz2 import copy +import io +import json from cloudevents.sdk.http_events import CloudEvent +from cloudevents.sdk import converters from sanic import response from sanic import Sanic @@ -72,15 +73,19 @@ app = Sanic(__name__) -def post(url, headers, json): - return app.test_client.post(url, headers=headers, data=json) +def post(url, headers, data): + return app.test_client.post(url, headers=headers, data=data) @app.route("/event", ["POST"]) async def echo(request): - assert isinstance(request.json, dict) - event = CloudEvent(request.json, headers=dict(request.headers)) - return response.text(json.dumps(event.data), headers=event.headers) + decoder = None + if "binary-payload" in request.headers: + decoder = lambda x: x + event = CloudEvent.from_http(request.body, headers=dict(request.headers), data_unmarshaller=decoder) + data = event.data if isinstance( + event.data, (bytes, bytearray, memoryview)) else json.dumps(event.data).encode() + return response.raw(data, headers={k: event[k] for k in event}) @pytest.mark.parametrize("body", invalid_cloudevent_request_bodie) @@ -90,17 +95,18 @@ def test_missing_required_fields_structured(body): # and NotImplementedError because structured calls aren't # implemented. In this instance one of the required keys should have # prefix e-id instead of ce-id therefore it should throw - _ = CloudEvent(body, headers={'Content-Type': 'application/json'}) + _ = CloudEvent.from_http(json.dumps(body), attributes={ + 'Content-Type': 'application/json'}) @pytest.mark.parametrize("headers", invalid_test_headers) def test_missing_required_fields_binary(headers): - with pytest.raises((TypeError, NotImplementedError)): + with pytest.raises((ValueError)): # CloudEvent constructor throws TypeError if missing required field # and NotImplementedError because structured calls aren't # implemented. In this instance one of the required keys should have # prefix e-id instead of ce-id therefore it should throw - _ = CloudEvent(test_data, headers=headers) + _ = CloudEvent.from_http(json.dumps(test_data), headers=headers) @pytest.mark.parametrize("specversion", ['1.0', '0.3']) @@ -110,13 +116,13 @@ def test_emit_binary_event(specversion): "ce-source": "", "ce-type": "cloudevent.event.type", "ce-specversion": specversion, - "Content-Type": "application/cloudevents+json" + "Content-Type": "text/plain" } - event = CloudEvent(test_data, headers=headers) + data = json.dumps(test_data) _, r = app.test_client.post( "/event", - headers=event.headers, - data=json.dumps(event.data) + headers=headers, + data=data ) # Convert byte array to dict @@ -125,17 +131,18 @@ def test_emit_binary_event(specversion): # Check response fields for key in test_data: - assert body[key] == test_data[key] + assert body[key] == test_data[key], body for key in headers: if key != 'Content-Type': - assert r.headers[key] == headers[key] + attribute_key = key[3:] + assert r.headers[attribute_key] == headers[key] assert r.status_code == 200 @pytest.mark.parametrize("specversion", ['1.0', '0.3']) def test_emit_structured_event(specversion): headers = { - "Content-Type": "application/json" + "Content-Type": "application/cloudevents+json" } body = { "id": "my-id", @@ -144,11 +151,10 @@ def test_emit_structured_event(specversion): "specversion": specversion, "data": test_data } - event = CloudEvent(body, headers=headers) _, r = app.test_client.post( "/event", - headers=event.headers, - data=json.dumps(event.data) + headers=headers, + data=json.dumps(body) ) # Convert byte array to dict @@ -160,6 +166,25 @@ def test_emit_structured_event(specversion): assert body[key] == test_data[key] assert r.status_code == 200 +@pytest.mark.parametrize("converter", [converters.TypeStructured, converters.TypeStructured]) +@pytest.mark.parametrize("specversion", ["1.0", "0.3"]) +def test_roundtrip_non_json_event(converter, specversion): + input_data = io.BytesIO() + for i in range(100): + for j in range(20): + assert 1 == input_data.write(j.to_bytes(1, byteorder='big')) + compressed_data = bz2.compress(input_data.getvalue()) + attrs = {"source": "test", "type": "t"} + + event = CloudEvent(attrs, compressed_data) + headers, data = event.to_http(converter, data_marshaller=lambda x: x) + headers["binary-payload"] = "true" # Decoding hint for server + _, r = app.test_client.post("/event", headers=headers, data=data) + + assert r.status_code == 200 + for key in attrs: + assert r.headers[key] == attrs[key] + assert compressed_data == r.body, r.body @pytest.mark.parametrize("specversion", ['1.0', '0.3']) def test_missing_ce_prefix_binary_event(specversion): @@ -175,12 +200,12 @@ def test_missing_ce_prefix_binary_event(specversion): # breaking prefix e.g. e-id instead of ce-id prefixed_headers[key[1:]] = headers[key] - with pytest.raises((TypeError, NotImplementedError)): + with pytest.raises(ValueError): # CloudEvent constructor throws TypeError if missing required field # and NotImplementedError because structured calls aren't # implemented. In this instance one of the required keys should have # prefix e-id instead of ce-id therefore it should throw - _ = CloudEvent(test_data, headers=prefixed_headers) + _ = CloudEvent.from_http(test_data, headers=prefixed_headers) @pytest.mark.parametrize("specversion", ['1.0', '0.3']) @@ -197,83 +222,87 @@ def test_valid_binary_events(specversion): "ce-specversion": specversion } data = {'payload': f"payload-{i}"} - events_queue.append(CloudEvent(data, headers=headers)) + events_queue.append(CloudEvent.from_http( + json.dumps(data), headers=headers)) for i, event in enumerate(events_queue): - headers = event.headers data = event.data - assert headers['ce-id'] == f"id{i}" - assert headers['ce-source'] == f"source{i}.com.test" - assert headers['ce-specversion'] == specversion - assert data['payload'] == f"payload-{i}" + assert event['id'] == f"id{i}" + assert event['source'] == f"source{i}.com.test" + assert event['specversion'] == specversion + assert event.data['payload'] == f"payload-{i}" @pytest.mark.parametrize("specversion", ['1.0', '0.3']) def test_structured_to_request(specversion): - data = { + attributes = { "specversion": specversion, "type": "word.found.name", "id": "96fb5f0b-001e-0108-6dfe-da6e2806f124", "source": "pytest", - "data": {"message": "Hello World!"} } - event = CloudEvent(data) - headers, body = event.to_request() - assert isinstance(body, dict) + data = {"message": "Hello World!"} + + event = CloudEvent(attributes, data) + headers, body_bytes = event.to_http() + assert isinstance(body_bytes, bytes) + body = json.loads(body_bytes) assert headers['content-type'] == 'application/cloudevents+json' - for key in data: - assert body[key] == data[key] + for key in attributes: + assert body[key] == attributes[key] + assert body["data"] == data, f"|{body_bytes}|| {body}" @pytest.mark.parametrize("specversion", ['1.0', '0.3']) def test_binary_to_request(specversion): - test_headers = { - "ce-specversion": specversion, - "ce-type": "word.found.name", - "ce-id": "96fb5f0b-001e-0108-6dfe-da6e2806f124", - "ce-source": "pytest" + attributes = { + "specversion": specversion, + "type": "word.found.name", + "id": "96fb5f0b-001e-0108-6dfe-da6e2806f124", + "source": "pytest" } data = { "message": "Hello World!" } - event = CloudEvent(data, headers=test_headers) - headers, body = event.to_request() - assert isinstance(body, dict) + event = CloudEvent(attributes, data) + headers, body_bytes = event.to_http(converters.TypeBinary) + body = json.loads(body_bytes) for key in data: assert body[key] == data[key] - for key in test_headers: - assert test_headers[key] == headers[key] + for key in attributes: + assert attributes[key] == headers['ce-' + key] @pytest.mark.parametrize("specversion", ['1.0', '0.3']) def test_empty_data_structured_event(specversion): # Testing if cloudevent breaks when no structured data field present - data = { + attributes = { "specversion": specversion, "datacontenttype": "application/json", "type": "word.found.name", "id": "96fb5f0b-001e-0108-6dfe-da6e2806f124", "time": "2018-10-23T12:28:22.4579346Z", "source": "", - "data": {"message": "Hello World!"} } - _ = CloudEvent(data) + + _ = CloudEvent.from_http(json.dumps(attributes), { + "content-type": "application/cloudevents+json"}) @pytest.mark.parametrize("specversion", ['1.0', '0.3']) def test_empty_data_binary_event(specversion): # Testing if cloudevent breaks when no structured data field present headers = { - "Content-Type": "application/cloudevents+json", + "Content-Type": "application/octet-stream", "ce-specversion": specversion, "ce-type": "word.found.name", "ce-id": "96fb5f0b-001e-0108-6dfe-da6e2806f124", "ce-time": "2018-10-23T12:28:22.4579346Z", "ce-source": "", } - _ = CloudEvent(None, headers=headers) + _ = CloudEvent.from_http('', headers) @pytest.mark.parametrize("specversion", ['1.0', '0.3']) @@ -283,21 +312,18 @@ def test_valid_structured_events(specversion): headers = {} num_cloudevents = 30 for i in range(num_cloudevents): - data = { + event = { "id": f"id{i}", "source": f"source{i}.com.test", "type": f"cloudevent.test.type", "specversion": specversion, - "data": { - 'payload': f"payload-{i}" - } + "data": {'payload': f"payload-{i}"} } - events_queue.append(CloudEvent(data)) + events_queue.append(CloudEvent.from_http(json.dumps(event), + {"content-type": "application/cloudevents+json"})) for i, event in enumerate(events_queue): - headers = event.headers - data = event.data - assert headers['ce-id'] == f"id{i}" - assert headers['ce-source'] == f"source{i}.com.test" - assert headers['ce-specversion'] == specversion - assert data['payload'] == f"payload-{i}" + assert event['id'] == f"id{i}" + assert event['source'] == f"source{i}.com.test" + assert event['specversion'] == specversion + assert event.data['payload'] == f"payload-{i}" diff --git a/samples/http-cloudevents/client.py b/samples/http-cloudevents/client.py index 89d03680..5e6d7a15 100644 --- a/samples/http-cloudevents/client.py +++ b/samples/http-cloudevents/client.py @@ -14,57 +14,56 @@ import sys import io from cloudevents.sdk.http_events import CloudEvent +from cloudevents.sdk import converters import requests def send_binary_cloud_event(url): # define cloudevents data - headers = { - "ce-id": "binary-event-id", - "ce-source": "localhost", - "ce-type": "template.http.binary", - "ce-specversion": "1.0", - "Content-Type": "application/json", - "ce-time": "2018-10-23T12:28:23.3464579Z" + attributes = { + "id": "binary-event-id", + "source": "localhost", + "type": "template.http.binary", + "dataontenttype": "application/json", + # Time will be filled in automatically if not set + "time": "2018-10-23T12:28:23.3464579Z" } data = {"payload-content": "Hello World!"} # create a CloudEvent - event = CloudEvent(data, headers=headers) - headers, body = event.to_request() + event = CloudEvent(attributes, data) + headers, body = event.to_http(converters.TypeBinary) # send and print event requests.post(url, headers=headers, json=body) print( - f"Sent {event['ce-id']} from {event['ce-source']} with " - f"{event['data']}" + f"Sent {event['id']} from {event['source']} with " + f"{event.data}" ) def send_structured_cloud_event(url): # define cloudevents data - data = { + attributes = { "id": "structured-event-id", "source": "localhost", "type": "template.http.structured", - "specversion": "1.0", - "Content-Type": "application/json", + "datacontenttype": "application/json", + # Time will be filled in automatically if not set "time": "2018-10-23T12:28:23.3464579Z", - "data": { - "payload-content": "Hello World!" - } } + data = {"payload-content": "Hello World!"} # create a CloudEvent - event = CloudEvent(data) + event = CloudEvent(attributes, data) headers, body = event.to_request() # send and print event requests.post(url, headers=headers, json=body) print( - f"Sent {event['ce-id']} from {event['ce-source']} with " - f"{event['data']}" + f"Sent {event['id']} from {event['source']} with " + f"{event.data}" ) diff --git a/samples/http-cloudevents/server.py b/samples/http-cloudevents/server.py index 11e48419..3797239e 100644 --- a/samples/http-cloudevents/server.py +++ b/samples/http-cloudevents/server.py @@ -20,11 +20,10 @@ @app.route('/', methods=['POST']) def home(): # convert headers to dict - headers = dict(request.headers) - print(request.json) - print(headers) + print(request.get_data()) + print(request.headers) # create a CloudEvent - event = CloudEvent(headers=headers, data=request.json) + event = CloudEvent.from_http(request.get_data(), request.headers) # print the received CloudEvent print(f"Received CloudEvent {event}") diff --git a/samples/http-cloudevents/test_verify_sample.py b/samples/http-cloudevents/test_verify_sample.py new file mode 100644 index 00000000..e69de29b diff --git a/tox.ini b/tox.ini index 370f7ffd..b3a89f9c 100644 --- a/tox.ini +++ b/tox.ini @@ -17,6 +17,7 @@ commands = flake8 [flake8] -ignore = H405,H404,H403,H401,H306,S101,N802,N803,N806,I202,I201 +# See https://gitlab.com/pycqa/flake8/-/issues/466 for extend-ignore vs ignore +extend-ignore = H405,H404,H403,H401,H306,S101,N802,N803,N806,I202,I201 show-source = True exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build,docs,venv,.venv,docs,etc,samples,tests From 616304beef8e7a833540c9cf2a7860d71ade7cf4 Mon Sep 17 00:00:00 2001 From: Curtis Mason <31265687+cumason123@users.noreply.github.com> Date: Thu, 9 Jul 2020 13:18:04 -0400 Subject: [PATCH 05/10] Changelog version deprecation (#48) * added changelog Signed-off-by: Curtis Mason * Created CloudEvent class (#36) CloudEvents is a more pythonic interface for using cloud events. It is powered by internal marshallers and cloud event base classes. It performs basic validation on fields, and cloud event type checking. Signed-off-by: Curtis Mason Signed-off-by: Dustin Ingram Signed-off-by: Curtis Mason * Fix tox configuration for CI (#46) Signed-off-by: Dustin Ingram Signed-off-by: Curtis Mason * Implemented python properties in base.py (#41) * Added SetCloudEventVersion Signed-off-by: Curtis Mason Signed-off-by: Dustin Ingram * began adding python properties Signed-off-by: Curtis Mason Signed-off-by: Dustin Ingram * added pythonic properties to base class Signed-off-by: Curtis Mason Signed-off-by: Dustin Ingram * began testing for getters/setters Signed-off-by: Curtis Mason Signed-off-by: Dustin Ingram * added general setter tests Signed-off-by: Curtis Mason Signed-off-by: Dustin Ingram * fixed spacing in base.py Signed-off-by: Curtis Mason Signed-off-by: Dustin Ingram * added __eq__ to option and datacontentencoding property to v03 Signed-off-by: Curtis Mason Signed-off-by: Dustin Ingram * lint fixes Signed-off-by: Curtis Mason Signed-off-by: Dustin Ingram * testing extensions and old getters Signed-off-by: Curtis Mason Signed-off-by: Dustin Ingram * removed versions v01 and v02 from test_data_encaps_refs.py Signed-off-by: Curtis Mason Signed-off-by: Dustin Ingram * fixed inheritance issue in CloudEvent Signed-off-by: Curtis Mason * added prefixed_headers dict to test Signed-off-by: Curtis Mason * CHANGELOG adjustment Signed-off-by: Curtis Mason * Update CHANGELOG.md Co-authored-by: Dustin Ingram Signed-off-by: Curtis Mason * Update CHANGELOG.md Co-authored-by: Dustin Ingram Signed-off-by: Curtis Mason * Update CHANGELOG.md Co-authored-by: Dustin Ingram Signed-off-by: Curtis Mason * Update CHANGELOG.md Co-authored-by: Dustin Ingram Signed-off-by: Curtis Mason * Removed irrelevant files from commit diff Signed-off-by: Curtis Mason Co-authored-by: Dustin Ingram --- CHANGELOG.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 89de93f2..aa8f3a11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.0.0] +### Added +- Added a user friendly CloudEvent class with data validation ([#36]) +- CloudEvent structured cloudevent support ([#47]) + +### Removed +- Removed support for Cloudevents V0.2 and V0.1 ([#43]) + ## [0.3.0] ### Added - Added Cloudevents V0.3 and V1 implementations ([#22]) @@ -65,4 +73,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [#22]: https://github.com/cloudevents/sdk-python/pull/22 [#23]: https://github.com/cloudevents/sdk-python/pull/23 [#25]: https://github.com/cloudevents/sdk-python/pull/25 -[#27]: https://github.com/cloudevents/sdk-python/pull/27 \ No newline at end of file +[#27]: https://github.com/cloudevents/sdk-python/pull/27 +[#36]: https://github.com/cloudevents/sdk-python/pull/36 +[#43]: https://github.com/cloudevents/sdk-python/pull/43 +[#47]: https://github.com/cloudevents/sdk-python/pull/47 From 50d0e05c6996b2c8579fddcc59609862186c179a Mon Sep 17 00:00:00 2001 From: Curtis Mason <31265687+cumason123@users.noreply.github.com> Date: Thu, 9 Jul 2020 14:52:24 -0400 Subject: [PATCH 06/10] Black formatter (#51) * black and isort added to precommit Signed-off-by: Curtis Mason * main renaming Signed-off-by: Curtis Mason * fixed tox Signed-off-by: Curtis Mason * linting in tox rename Signed-off-by: Curtis Mason * fixed tox trailing space Signed-off-by: Curtis Mason * added reformat tox env Signed-off-by: Curtis Mason * Reformatting files Signed-off-by: Curtis Mason * reformatted more files Signed-off-by: Curtis Mason * documented tox in README Signed-off-by: Curtis Mason * removed -rc flag Signed-off-by: Curtis Mason --- .isort.cfg | 4 + .pre-commit-config.yaml | 10 ++ README.md | 11 ++ cloudevents/sdk/converters/__init__.py | 3 +- cloudevents/sdk/converters/base.py | 6 +- cloudevents/sdk/converters/binary.py | 6 +- cloudevents/sdk/converters/structured.py | 5 +- cloudevents/sdk/event/base.py | 9 +- cloudevents/sdk/event/opt.py | 14 +- cloudevents/sdk/event/v03.py | 24 +-- cloudevents/sdk/event/v1.py | 19 +- cloudevents/sdk/exceptions.py | 12 +- cloudevents/sdk/http_events.py | 57 +++--- cloudevents/sdk/marshaller.py | 13 +- cloudevents/sdk/types.py | 7 +- cloudevents/tests/data.py | 2 +- cloudevents/tests/test_data_encaps_refs.py | 36 ++-- .../test_event_from_request_converter.py | 31 ++-- cloudevents/tests/test_event_pipeline.py | 26 ++- .../tests/test_event_to_request_converter.py | 14 +- cloudevents/tests/test_http_events.py | 167 +++++++++--------- cloudevents/tests/test_with_sanic.py | 25 +-- etc/docs_conf/conf.py | 63 ++++--- pyproject.toml | 16 ++ samples/http-cloudevents/client.py | 28 ++- samples/http-cloudevents/server.py | 10 +- .../python-requests/cloudevent_to_request.py | 37 ++-- .../python-requests/request_to_cloudevent.py | 13 +- setup.py | 4 +- tox.ini | 20 ++- 30 files changed, 343 insertions(+), 349 deletions(-) create mode 100644 .isort.cfg create mode 100644 .pre-commit-config.yaml create mode 100644 pyproject.toml diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 00000000..22880d42 --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,4 @@ +[settings] +line_length = 80 +multi_line_output = 3 +include_trailing_comma = True diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..ed9f8e11 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,10 @@ +repos: +- repo: https://github.com/timothycrosley/isort/ + rev: 5.0.4 + hooks: + - id: isort +- repo: https://github.com/psf/black + rev: 19.10b0 + hooks: + - id: black + language_version: python3.8 diff --git a/README.md b/README.md index a52bdcb2..f911f0d2 100644 --- a/README.md +++ b/README.md @@ -217,3 +217,14 @@ the same API. It will use semantic versioning with following rules: [CNCF's Slack workspace](https://slack.cncf.io/). - Email: https://lists.cncf.io/g/cncf-cloudevents-sdk - Contact for additional information: Denis Makogon (`@denysmakogon` on slack). + +## Maintenance + +We use black and isort for autoformatting. We setup a tox environment to reformat +the codebase. + +e.g. +```python +pip install tox +tox -e reformat +``` diff --git a/cloudevents/sdk/converters/__init__.py b/cloudevents/sdk/converters/__init__.py index ee2fc412..1e786455 100644 --- a/cloudevents/sdk/converters/__init__.py +++ b/cloudevents/sdk/converters/__init__.py @@ -12,8 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. -from cloudevents.sdk.converters import binary -from cloudevents.sdk.converters import structured +from cloudevents.sdk.converters import binary, structured TypeBinary = binary.BinaryHTTPCloudEventConverter.TYPE TypeStructured = structured.JSONHTTPCloudEventConverter.TYPE diff --git a/cloudevents/sdk/converters/base.py b/cloudevents/sdk/converters/base.py index 69bf8cb0..aa75f7c7 100644 --- a/cloudevents/sdk/converters/base.py +++ b/cloudevents/sdk/converters/base.py @@ -26,7 +26,7 @@ def read( event, headers: dict, body: typing.IO, - data_unmarshaller: typing.Callable + data_unmarshaller: typing.Callable, ) -> base.BaseEvent: raise Exception("not implemented") @@ -37,8 +37,6 @@ def can_read(self, content_type: str) -> bool: raise Exception("not implemented") def write( - self, - event: base.BaseEvent, - data_marshaller: typing.Callable + self, event: base.BaseEvent, data_marshaller: typing.Callable ) -> (dict, object): raise Exception("not implemented") diff --git a/cloudevents/sdk/converters/binary.py b/cloudevents/sdk/converters/binary.py index dfb08fe6..abdc9d99 100644 --- a/cloudevents/sdk/converters/binary.py +++ b/cloudevents/sdk/converters/binary.py @@ -17,7 +17,7 @@ from cloudevents.sdk import exceptions, types from cloudevents.sdk.converters import base from cloudevents.sdk.event import base as event_base -from cloudevents.sdk.event import v03, v1 +from cloudevents.sdk.event import v1, v03 class BinaryHTTPCloudEventConverter(base.Converter): @@ -44,9 +44,7 @@ def read( return event def write( - self, - event: event_base.BaseEvent, - data_marshaller: types.MarshallerType + self, event: event_base.BaseEvent, data_marshaller: types.MarshallerType ) -> (dict, bytes): return event.MarshalBinary(data_marshaller) diff --git a/cloudevents/sdk/converters/structured.py b/cloudevents/sdk/converters/structured.py index 435e004f..d29a9284 100644 --- a/cloudevents/sdk/converters/structured.py +++ b/cloudevents/sdk/converters/structured.py @@ -15,7 +15,6 @@ import typing from cloudevents.sdk import types - from cloudevents.sdk.converters import base from cloudevents.sdk.event import base as event_base @@ -43,9 +42,7 @@ def read( return event def write( - self, - event: event_base.BaseEvent, - data_marshaller: types.MarshallerType + self, event: event_base.BaseEvent, data_marshaller: types.MarshallerType ) -> (dict, bytes): http_headers = {"content-type": self.MIME_TYPE} return http_headers, event.MarshalJSON(data_marshaller).encode("utf-8") diff --git a/cloudevents/sdk/event/base.py b/cloudevents/sdk/event/base.py index 25ba6bd7..791eb679 100644 --- a/cloudevents/sdk/event/base.py +++ b/cloudevents/sdk/event/base.py @@ -206,7 +206,7 @@ def MarshalJSON(self, data_marshaller: types.MarshallerType) -> str: def UnmarshalJSON( self, b: typing.Union[str, bytes], - data_unmarshaller: types.UnmarshallerType + data_unmarshaller: types.UnmarshallerType, ): raw_ce = json.loads(b) @@ -228,9 +228,9 @@ def UnmarshalBinary( self, headers: dict, body: typing.Union[bytes, str], - data_unmarshaller: types.UnmarshallerType + data_unmarshaller: types.UnmarshallerType, ): - if 'ce-specversion' not in headers: + if "ce-specversion" not in headers: raise ValueError("Missing required attribute: 'specversion'") for header, value in headers.items(): header = header.lower() @@ -244,8 +244,7 @@ def UnmarshalBinary( raise ValueError(f"Missing required attributes: {missing_attrs}") def MarshalBinary( - self, - data_marshaller: types.MarshallerType + self, data_marshaller: types.MarshallerType ) -> (dict, bytes): if data_marshaller is None: data_marshaller = json.dumps diff --git a/cloudevents/sdk/event/opt.py b/cloudevents/sdk/event/opt.py index bf630c32..e28d84f3 100644 --- a/cloudevents/sdk/event/opt.py +++ b/cloudevents/sdk/event/opt.py @@ -24,8 +24,8 @@ def set(self, new_value): if self.is_required and is_none: raise ValueError( "Attribute value error: '{0}', " - "" "invalid new value." - .format(self.name) + "" + "invalid new value.".format(self.name) ) self.value = new_value @@ -37,7 +37,9 @@ def required(self): return self.is_required def __eq__(self, obj): - return isinstance(obj, Option) and \ - obj.name == self.name and \ - obj.value == self.value and \ - obj.is_required == self.is_required + return ( + isinstance(obj, Option) + and obj.name == self.name + and obj.value == self.value + and obj.is_required == self.is_required + ) diff --git a/cloudevents/sdk/event/v03.py b/cloudevents/sdk/event/v03.py index 3b6a3222..03d1c1f4 100644 --- a/cloudevents/sdk/event/v03.py +++ b/cloudevents/sdk/event/v03.py @@ -12,24 +12,18 @@ # License for the specific language governing permissions and limitations # under the License. -from cloudevents.sdk.event import base -from cloudevents.sdk.event import opt +from cloudevents.sdk.event import base, opt class Event(base.BaseEvent): - _ce_required_fields = { - 'id', - 'source', - 'type', - 'specversion' - } + _ce_required_fields = {"id", "source", "type", "specversion"} _ce_optional_fields = { - 'datacontentencoding', - 'datacontenttype', - 'schemaurl', - 'subject', - 'time' + "datacontentencoding", + "datacontenttype", + "schemaurl", + "subject", + "time", } def __init__(self): @@ -40,9 +34,7 @@ def __init__(self): self.ce__datacontenttype = opt.Option("datacontenttype", None, False) self.ce__datacontentencoding = opt.Option( - "datacontentencoding", - None, - False + "datacontentencoding", None, False ) self.ce__subject = opt.Option("subject", None, False) self.ce__time = opt.Option("time", None, False) diff --git a/cloudevents/sdk/event/v1.py b/cloudevents/sdk/event/v1.py index 6034a9b4..782fd7ac 100644 --- a/cloudevents/sdk/event/v1.py +++ b/cloudevents/sdk/event/v1.py @@ -12,24 +12,13 @@ # License for the specific language governing permissions and limitations # under the License. -from cloudevents.sdk.event import base -from cloudevents.sdk.event import opt +from cloudevents.sdk.event import base, opt class Event(base.BaseEvent): - _ce_required_fields = { - 'id', - 'source', - 'type', - 'specversion' - } - - _ce_optional_fields = { - 'datacontenttype', - 'dataschema', - 'subject', - 'time' - } + _ce_required_fields = {"id", "source", "type", "specversion"} + + _ce_optional_fields = {"datacontenttype", "dataschema", "subject", "time"} def __init__(self): self.ce__specversion = opt.Option("specversion", "1.0", True) diff --git a/cloudevents/sdk/exceptions.py b/cloudevents/sdk/exceptions.py index 2f30db04..3195f90e 100644 --- a/cloudevents/sdk/exceptions.py +++ b/cloudevents/sdk/exceptions.py @@ -15,9 +15,7 @@ class UnsupportedEvent(Exception): def __init__(self, event_class): - super().__init__( - "Invalid CloudEvent class: '{0}'".format(event_class) - ) + super().__init__("Invalid CloudEvent class: '{0}'".format(event_class)) class InvalidDataUnmarshaller(Exception): @@ -27,16 +25,12 @@ def __init__(self): class InvalidDataMarshaller(Exception): def __init__(self): - super().__init__( - "Invalid data marshaller, is not a callable" - ) + super().__init__("Invalid data marshaller, is not a callable") class NoSuchConverter(Exception): def __init__(self, converter_type): - super().__init__( - "No such converter {0}".format(converter_type) - ) + super().__init__("No such converter {0}".format(converter_type)) class UnsupportedEventConverter(Exception): diff --git a/cloudevents/sdk/http_events.py b/cloudevents/sdk/http_events.py index b27c1a26..9ca3abc6 100644 --- a/cloudevents/sdk/http_events.py +++ b/cloudevents/sdk/http_events.py @@ -17,11 +17,8 @@ import typing import uuid -from cloudevents.sdk import converters -from cloudevents.sdk import marshaller -from cloudevents.sdk import types - -from cloudevents.sdk.event import v03, v1 +from cloudevents.sdk import converters, marshaller, types +from cloudevents.sdk.event import v1, v03 _marshaller_by_format = { converters.TypeStructured: lambda x: x, @@ -43,7 +40,7 @@ def _json_or_string(content: typing.Union[str, bytes]): return content -class CloudEvent(): +class CloudEvent: """ Python-friendly cloudevent class supporting v1 events Supports both binary and structured mode CloudEvents @@ -54,7 +51,7 @@ def from_http( cls, data: typing.Union[str, bytes], headers: typing.Dict[str, str], - data_unmarshaller: types.UnmarshallerType = None + data_unmarshaller: types.UnmarshallerType = None, ): """Unwrap a CloudEvent (binary or structured) from an HTTP request. :param data: the HTTP request body @@ -68,18 +65,17 @@ def from_http( data_unmarshaller = _json_or_string event = marshaller.NewDefaultHTTPMarshaller().FromRequest( - v1.Event(), headers, data, data_unmarshaller) + v1.Event(), headers, data, data_unmarshaller + ) attrs = event.Properties() - attrs.pop('data', None) - attrs.pop('extensions', None) + attrs.pop("data", None) + attrs.pop("extensions", None) attrs.update(**event.extensions) return cls(attrs, event.data) def __init__( - self, - attributes: typing.Dict[str, str], - data: typing.Any = None + self, attributes: typing.Dict[str, str], data: typing.Any = None ): """ Event Constructor @@ -97,28 +93,31 @@ def __init__( """ self._attributes = {k.lower(): v for k, v in attributes.items()} self.data = data - if 'specversion' not in self._attributes: - self._attributes['specversion'] = "1.0" - if 'id' not in self._attributes: - self._attributes['id'] = str(uuid.uuid4()) - if 'time' not in self._attributes: - self._attributes['time'] = datetime.datetime.now( - datetime.timezone.utc).isoformat() - - if self._attributes['specversion'] not in _required_by_version: + if "specversion" not in self._attributes: + self._attributes["specversion"] = "1.0" + if "id" not in self._attributes: + self._attributes["id"] = str(uuid.uuid4()) + if "time" not in self._attributes: + self._attributes["time"] = datetime.datetime.now( + datetime.timezone.utc + ).isoformat() + + if self._attributes["specversion"] not in _required_by_version: raise ValueError( - f"Invalid specversion: {self._attributes['specversion']}") + f"Invalid specversion: {self._attributes['specversion']}" + ) # There is no good way to default 'source' and 'type', so this # checks for those (or any new required attributes). - required_set = _required_by_version[self._attributes['specversion']] + required_set = _required_by_version[self._attributes["specversion"]] if not required_set <= self._attributes.keys(): raise ValueError( - f"Missing required keys: {required_set - attributes.keys()}") + f"Missing required keys: {required_set - attributes.keys()}" + ) def to_http( self, format: str = converters.TypeStructured, - data_marshaller: types.MarshallerType = None + data_marshaller: types.MarshallerType = None, ) -> (dict, typing.Union[bytes, str]): """ Returns a tuple of HTTP headers/body dicts representing this cloudevent @@ -133,7 +132,8 @@ def to_http( data_marshaller = _marshaller_by_format[format] if self._attributes["specversion"] not in _obj_by_version: raise ValueError( - f"Unsupported specversion: {self._attributes['specversion']}") + f"Unsupported specversion: {self._attributes['specversion']}" + ) event = _obj_by_version[self._attributes["specversion"]]() for k, v in self._attributes.items(): @@ -141,7 +141,8 @@ def to_http( event.data = self.data return marshaller.NewDefaultHTTPMarshaller().ToRequest( - event, format, data_marshaller) + event, format, data_marshaller + ) # Data access is handled via `.data` member # Attribute access is managed via Mapping type diff --git a/cloudevents/sdk/marshaller.py b/cloudevents/sdk/marshaller.py index 1276dca3..d9fe4920 100644 --- a/cloudevents/sdk/marshaller.py +++ b/cloudevents/sdk/marshaller.py @@ -16,11 +16,7 @@ import typing from cloudevents.sdk import exceptions, types - -from cloudevents.sdk.converters import base -from cloudevents.sdk.converters import binary -from cloudevents.sdk.converters import structured - +from cloudevents.sdk.converters import base, binary, structured from cloudevents.sdk.event import base as event_base @@ -93,8 +89,9 @@ def ToRequest( :return: dict of HTTP headers and stream of HTTP request body :rtype: tuple """ - if (data_marshaller is not None - and not isinstance(data_marshaller, typing.Callable)): + if data_marshaller is not None and not isinstance( + data_marshaller, typing.Callable + ): raise exceptions.InvalidDataMarshaller() if converter_type is None: @@ -123,7 +120,7 @@ def NewDefaultHTTPMarshaller() -> HTTPMarshaller: def NewHTTPMarshaller( - converters: typing.List[base.Converter] + converters: typing.List[base.Converter], ) -> HTTPMarshaller: """ Creates the default HTTP marshaller with both diff --git a/cloudevents/sdk/types.py b/cloudevents/sdk/types.py index 82885842..1a302ea2 100644 --- a/cloudevents/sdk/types.py +++ b/cloudevents/sdk/types.py @@ -18,7 +18,8 @@ # both JSON and Binary format. MarshallerType = typing.Optional[ - typing.Callable[[typing.Any], typing.Union[bytes, str]]] + typing.Callable[[typing.Any], typing.Union[bytes, str]] +] UnmarshallerType = typing.Optional[ - typing.Callable[ - [typing.Union[bytes, str]], typing.Any]] + typing.Callable[[typing.Union[bytes, str]], typing.Any] +] diff --git a/cloudevents/tests/data.py b/cloudevents/tests/data.py index ffe63aee..353aac50 100644 --- a/cloudevents/tests/data.py +++ b/cloudevents/tests/data.py @@ -12,7 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. -from cloudevents.sdk.event import v03, v1 +from cloudevents.sdk.event import v1, v03 contentType = "application/json" ce_type = "word.found.exclamation" diff --git a/cloudevents/tests/test_data_encaps_refs.py b/cloudevents/tests/test_data_encaps_refs.py index 3afb40e8..497334f3 100644 --- a/cloudevents/tests/test_data_encaps_refs.py +++ b/cloudevents/tests/test_data_encaps_refs.py @@ -12,20 +12,16 @@ # License for the specific language governing permissions and limitations # under the License. +import copy import io import json -import copy -import pytest - from uuid import uuid4 -from cloudevents.sdk import converters -from cloudevents.sdk import marshaller +import pytest +from cloudevents.sdk import converters, marshaller from cloudevents.sdk.converters import structured -from cloudevents.sdk.event import v03, v1 - - +from cloudevents.sdk.event import v1, v03 from cloudevents.tests import data @@ -56,7 +52,7 @@ def test_general_binary_properties(event_class): new_content_type = str(uuid4()) new_source = str(uuid4()) - event.extensions = {'test': str(uuid4)} + event.extensions = {"test": str(uuid4)} event.type = new_type event.id = new_id event.content_type = new_content_type @@ -66,10 +62,11 @@ def test_general_binary_properties(event_class): assert (event.type == new_type) and (event.type == event.EventType()) assert (event.id == new_id) and (event.id == event.EventID()) assert (event.content_type == new_content_type) and ( - event.content_type == event.ContentType()) + event.content_type == event.ContentType() + ) assert (event.source == new_source) and (event.source == event.Source()) - assert event.extensions['test'] == event.Extensions()['test'] - assert (event.specversion == event.CloudEventVersion()) + assert event.extensions["test"] == event.Extensions()["test"] + assert event.specversion == event.CloudEventVersion() @pytest.mark.parametrize("event_class", [v03.Event, v1.Event]) @@ -78,8 +75,10 @@ def test_general_structured_properties(event_class): m = marshaller.NewDefaultHTTPMarshaller() http_headers = {"content-type": "application/cloudevents+json"} event = m.FromRequest( - event_class(), http_headers, json.dumps( - data.json_ce[event_class]), lambda x: x + event_class(), + http_headers, + json.dumps(data.json_ce[event_class]), + lambda x: x, ) # Test python properties assert event is not None @@ -101,7 +100,7 @@ def test_general_structured_properties(event_class): new_content_type = str(uuid4()) new_source = str(uuid4()) - event.extensions = {'test': str(uuid4)} + event.extensions = {"test": str(uuid4)} event.type = new_type event.id = new_id event.content_type = new_content_type @@ -111,7 +110,8 @@ def test_general_structured_properties(event_class): assert (event.type == new_type) and (event.type == event.EventType()) assert (event.id == new_id) and (event.id == event.EventID()) assert (event.content_type == new_content_type) and ( - event.content_type == event.ContentType()) + event.content_type == event.ContentType() + ) assert (event.source == new_source) and (event.source == event.Source()) - assert event.extensions['test'] == event.Extensions()['test'] - assert (event.specversion == event.CloudEventVersion()) + assert event.extensions["test"] == event.Extensions()["test"] + assert event.specversion == event.CloudEventVersion() diff --git a/cloudevents/tests/test_event_from_request_converter.py b/cloudevents/tests/test_event_from_request_converter.py index 7163533c..b291b01e 100644 --- a/cloudevents/tests/test_event_from_request_converter.py +++ b/cloudevents/tests/test_event_from_request_converter.py @@ -12,31 +12,24 @@ # License for the specific language governing permissions and limitations # under the License. -import json -import pytest import io +import json -from cloudevents.sdk import exceptions -from cloudevents.sdk import marshaller - -from cloudevents.sdk.event import v03 -from cloudevents.sdk.event import v1 - -from cloudevents.sdk.converters import binary -from cloudevents.sdk.converters import structured +import pytest +from cloudevents.sdk import exceptions, marshaller +from cloudevents.sdk.converters import binary, structured +from cloudevents.sdk.event import v1, v03 from cloudevents.tests import data @pytest.mark.parametrize("event_class", [v03.Event, v1.Event]) def test_binary_converter_upstream(event_class): m = marshaller.NewHTTPMarshaller( - [binary.NewBinaryHTTPCloudEventConverter()]) + [binary.NewBinaryHTTPCloudEventConverter()] + ) event = m.FromRequest( - event_class(), - data.headers[event_class], - None, - lambda x: x + event_class(), data.headers[event_class], None, lambda x: x ) assert event is not None assert event.EventType() == data.ce_type @@ -47,7 +40,8 @@ def test_binary_converter_upstream(event_class): @pytest.mark.parametrize("event_class", [v03.Event, v1.Event]) def test_structured_converter_upstream(event_class): m = marshaller.NewHTTPMarshaller( - [structured.NewJSONHTTPCloudEventConverter()]) + [structured.NewJSONHTTPCloudEventConverter()] + ) event = m.FromRequest( event_class(), {"Content-Type": "application/cloudevents+json"}, @@ -82,9 +76,10 @@ def test_default_http_marshaller_with_binary(event_class): m = marshaller.NewDefaultHTTPMarshaller() event = m.FromRequest( - event_class(), data.headers[event_class], + event_class(), + data.headers[event_class], json.dumps(data.body), - json.loads + json.loads, ) assert event is not None assert event.EventType() == data.ce_type diff --git a/cloudevents/tests/test_event_pipeline.py b/cloudevents/tests/test_event_pipeline.py index 43fc3f7e..60da6e45 100644 --- a/cloudevents/tests/test_event_pipeline.py +++ b/cloudevents/tests/test_event_pipeline.py @@ -14,14 +14,12 @@ import io import json -import pytest -from cloudevents.sdk.event import v03, v1 +import pytest -from cloudevents.sdk import converters -from cloudevents.sdk import marshaller +from cloudevents.sdk import converters, marshaller from cloudevents.sdk.converters import structured - +from cloudevents.sdk.event import v1, v03 from cloudevents.tests import data @@ -51,11 +49,8 @@ def test_event_pipeline_upstream(event_class): def test_extensions_are_set_upstream(): - extensions = {'extension-key': 'extension-value'} - event = ( - v1.Event() - .SetExtensions(extensions) - ) + extensions = {"extension-key": "extension-value"} + event = v1.Event().SetExtensions(extensions) m = marshaller.NewDefaultHTTPMarshaller() new_headers, _ = m.ToRequest(event, converters.TypeBinary, lambda x: x) @@ -68,9 +63,11 @@ def test_binary_event_v1(): event = ( v1.Event() .SetContentType("application/octet-stream") - .SetData(b'\x00\x01') + .SetData(b"\x00\x01") + ) + m = marshaller.NewHTTPMarshaller( + [structured.NewJSONHTTPCloudEventConverter()] ) - m = marshaller.NewHTTPMarshaller([structured.NewJSONHTTPCloudEventConverter()]) _, body = m.ToRequest(event, converters.TypeStructured, lambda x: x) assert isinstance(body, bytes) @@ -78,11 +75,10 @@ def test_binary_event_v1(): assert "data" not in content assert content["data_base64"] == "AAE=", f"Content is: {content}" + def test_object_event_v1(): event = ( - v1.Event() - .SetContentType("application/json") - .SetData({"name": "john"}) + v1.Event().SetContentType("application/json").SetData({"name": "john"}) ) m = marshaller.NewDefaultHTTPMarshaller() diff --git a/cloudevents/tests/test_event_to_request_converter.py b/cloudevents/tests/test_event_to_request_converter.py index f46746b7..e54264f3 100644 --- a/cloudevents/tests/test_event_to_request_converter.py +++ b/cloudevents/tests/test_event_to_request_converter.py @@ -12,17 +12,15 @@ # License for the specific language governing permissions and limitations # under the License. +import copy import io import json -import copy -import pytest -from cloudevents.sdk import converters -from cloudevents.sdk import marshaller +import pytest +from cloudevents.sdk import converters, marshaller from cloudevents.sdk.converters import structured -from cloudevents.sdk.event import v03, v1 - +from cloudevents.sdk.event import v1, v03 from cloudevents.tests import data @@ -51,9 +49,7 @@ def test_structured_event_to_request_upstream(event_class): m = marshaller.NewDefaultHTTPMarshaller() http_headers = {"content-type": "application/cloudevents+json"} event = m.FromRequest( - event_class(), - http_headers, - json.dumps(data.json_ce[event_class]) + event_class(), http_headers, json.dumps(data.json_ce[event_class]) ) assert event is not None assert event.EventType() == data.ce_type diff --git a/cloudevents/tests/test_http_events.py b/cloudevents/tests/test_http_events.py index 57822a54..6213edda 100644 --- a/cloudevents/tests/test_http_events.py +++ b/cloudevents/tests/test_http_events.py @@ -17,58 +17,47 @@ import io import json -from cloudevents.sdk.http_events import CloudEvent -from cloudevents.sdk import converters - -from sanic import response -from sanic import Sanic - import pytest +from sanic import Sanic, response +from cloudevents.sdk import converters +from cloudevents.sdk.http_events import CloudEvent invalid_test_headers = [ { "ce-source": "", "ce-type": "cloudevent.event.type", - "ce-specversion": "1.0" - }, { + "ce-specversion": "1.0", + }, + { "ce-id": "my-id", "ce-type": "cloudevent.event.type", - "ce-specversion": "1.0" - }, { - "ce-id": "my-id", - "ce-source": "", - "ce-specversion": "1.0" - }, { + "ce-specversion": "1.0", + }, + {"ce-id": "my-id", "ce-source": "", "ce-specversion": "1.0"}, + { "ce-id": "my-id", "ce-source": "", "ce-type": "cloudevent.event.type", - } + }, ] invalid_cloudevent_request_bodie = [ { "source": "", "type": "cloudevent.event.type", - "specversion": "1.0" - }, { - "id": "my-id", - "type": "cloudevent.event.type", - "specversion": "1.0" - }, { - "id": "my-id", - "source": "", - "specversion": "1.0" - }, { + "specversion": "1.0", + }, + {"id": "my-id", "type": "cloudevent.event.type", "specversion": "1.0"}, + {"id": "my-id", "source": "", "specversion": "1.0"}, + { "id": "my-id", "source": "", "type": "cloudevent.event.type", - } + }, ] -test_data = { - "payload-content": "Hello World!" -} +test_data = {"payload-content": "Hello World!"} app = Sanic(__name__) @@ -82,9 +71,14 @@ async def echo(request): decoder = None if "binary-payload" in request.headers: decoder = lambda x: x - event = CloudEvent.from_http(request.body, headers=dict(request.headers), data_unmarshaller=decoder) - data = event.data if isinstance( - event.data, (bytes, bytearray, memoryview)) else json.dumps(event.data).encode() + event = CloudEvent.from_http( + request.body, headers=dict(request.headers), data_unmarshaller=decoder + ) + data = ( + event.data + if isinstance(event.data, (bytes, bytearray, memoryview)) + else json.dumps(event.data).encode() + ) return response.raw(data, headers={k: event[k] for k in event}) @@ -95,8 +89,9 @@ def test_missing_required_fields_structured(body): # and NotImplementedError because structured calls aren't # implemented. In this instance one of the required keys should have # prefix e-id instead of ce-id therefore it should throw - _ = CloudEvent.from_http(json.dumps(body), attributes={ - 'Content-Type': 'application/json'}) + _ = CloudEvent.from_http( + json.dumps(body), attributes={"Content-Type": "application/json"} + ) @pytest.mark.parametrize("headers", invalid_test_headers) @@ -109,70 +104,65 @@ def test_missing_required_fields_binary(headers): _ = CloudEvent.from_http(json.dumps(test_data), headers=headers) -@pytest.mark.parametrize("specversion", ['1.0', '0.3']) +@pytest.mark.parametrize("specversion", ["1.0", "0.3"]) def test_emit_binary_event(specversion): headers = { "ce-id": "my-id", "ce-source": "", "ce-type": "cloudevent.event.type", "ce-specversion": specversion, - "Content-Type": "text/plain" + "Content-Type": "text/plain", } data = json.dumps(test_data) - _, r = app.test_client.post( - "/event", - headers=headers, - data=data - ) + _, r = app.test_client.post("/event", headers=headers, data=data) # Convert byte array to dict # e.g. r.body = b'{"payload-content": "Hello World!"}' - body = json.loads(r.body.decode('utf-8')) + body = json.loads(r.body.decode("utf-8")) # Check response fields for key in test_data: assert body[key] == test_data[key], body for key in headers: - if key != 'Content-Type': + if key != "Content-Type": attribute_key = key[3:] assert r.headers[attribute_key] == headers[key] assert r.status_code == 200 -@pytest.mark.parametrize("specversion", ['1.0', '0.3']) +@pytest.mark.parametrize("specversion", ["1.0", "0.3"]) def test_emit_structured_event(specversion): - headers = { - "Content-Type": "application/cloudevents+json" - } + headers = {"Content-Type": "application/cloudevents+json"} body = { "id": "my-id", "source": "", "type": "cloudevent.event.type", "specversion": specversion, - "data": test_data + "data": test_data, } _, r = app.test_client.post( - "/event", - headers=headers, - data=json.dumps(body) + "/event", headers=headers, data=json.dumps(body) ) # Convert byte array to dict # e.g. r.body = b'{"payload-content": "Hello World!"}' - body = json.loads(r.body.decode('utf-8')) + body = json.loads(r.body.decode("utf-8")) # Check response fields for key in test_data: assert body[key] == test_data[key] assert r.status_code == 200 -@pytest.mark.parametrize("converter", [converters.TypeStructured, converters.TypeStructured]) + +@pytest.mark.parametrize( + "converter", [converters.TypeStructured, converters.TypeStructured] +) @pytest.mark.parametrize("specversion", ["1.0", "0.3"]) def test_roundtrip_non_json_event(converter, specversion): input_data = io.BytesIO() for i in range(100): for j in range(20): - assert 1 == input_data.write(j.to_bytes(1, byteorder='big')) + assert 1 == input_data.write(j.to_bytes(1, byteorder="big")) compressed_data = bz2.compress(input_data.getvalue()) attrs = {"source": "test", "type": "t"} @@ -186,14 +176,15 @@ def test_roundtrip_non_json_event(converter, specversion): assert r.headers[key] == attrs[key] assert compressed_data == r.body, r.body -@pytest.mark.parametrize("specversion", ['1.0', '0.3']) + +@pytest.mark.parametrize("specversion", ["1.0", "0.3"]) def test_missing_ce_prefix_binary_event(specversion): prefixed_headers = {} headers = { "ce-id": "my-id", "ce-source": "", "ce-type": "cloudevent.event.type", - "ce-specversion": specversion + "ce-specversion": specversion, } for key in headers: @@ -208,7 +199,7 @@ def test_missing_ce_prefix_binary_event(specversion): _ = CloudEvent.from_http(test_data, headers=prefixed_headers) -@pytest.mark.parametrize("specversion", ['1.0', '0.3']) +@pytest.mark.parametrize("specversion", ["1.0", "0.3"]) def test_valid_binary_events(specversion): # Test creating multiple cloud events events_queue = [] @@ -219,21 +210,22 @@ def test_valid_binary_events(specversion): "ce-id": f"id{i}", "ce-source": f"source{i}.com.test", "ce-type": f"cloudevent.test.type", - "ce-specversion": specversion + "ce-specversion": specversion, } - data = {'payload': f"payload-{i}"} - events_queue.append(CloudEvent.from_http( - json.dumps(data), headers=headers)) + data = {"payload": f"payload-{i}"} + events_queue.append( + CloudEvent.from_http(json.dumps(data), headers=headers) + ) for i, event in enumerate(events_queue): data = event.data - assert event['id'] == f"id{i}" - assert event['source'] == f"source{i}.com.test" - assert event['specversion'] == specversion - assert event.data['payload'] == f"payload-{i}" + assert event["id"] == f"id{i}" + assert event["source"] == f"source{i}.com.test" + assert event["specversion"] == specversion + assert event.data["payload"] == f"payload-{i}" -@pytest.mark.parametrize("specversion", ['1.0', '0.3']) +@pytest.mark.parametrize("specversion", ["1.0", "0.3"]) def test_structured_to_request(specversion): attributes = { "specversion": specversion, @@ -248,23 +240,21 @@ def test_structured_to_request(specversion): assert isinstance(body_bytes, bytes) body = json.loads(body_bytes) - assert headers['content-type'] == 'application/cloudevents+json' + assert headers["content-type"] == "application/cloudevents+json" for key in attributes: assert body[key] == attributes[key] assert body["data"] == data, f"|{body_bytes}|| {body}" -@pytest.mark.parametrize("specversion", ['1.0', '0.3']) +@pytest.mark.parametrize("specversion", ["1.0", "0.3"]) def test_binary_to_request(specversion): attributes = { "specversion": specversion, "type": "word.found.name", "id": "96fb5f0b-001e-0108-6dfe-da6e2806f124", - "source": "pytest" - } - data = { - "message": "Hello World!" + "source": "pytest", } + data = {"message": "Hello World!"} event = CloudEvent(attributes, data) headers, body_bytes = event.to_http(converters.TypeBinary) body = json.loads(body_bytes) @@ -272,10 +262,10 @@ def test_binary_to_request(specversion): for key in data: assert body[key] == data[key] for key in attributes: - assert attributes[key] == headers['ce-' + key] + assert attributes[key] == headers["ce-" + key] -@pytest.mark.parametrize("specversion", ['1.0', '0.3']) +@pytest.mark.parametrize("specversion", ["1.0", "0.3"]) def test_empty_data_structured_event(specversion): # Testing if cloudevent breaks when no structured data field present attributes = { @@ -287,11 +277,12 @@ def test_empty_data_structured_event(specversion): "source": "", } - _ = CloudEvent.from_http(json.dumps(attributes), { - "content-type": "application/cloudevents+json"}) + _ = CloudEvent.from_http( + json.dumps(attributes), {"content-type": "application/cloudevents+json"} + ) -@pytest.mark.parametrize("specversion", ['1.0', '0.3']) +@pytest.mark.parametrize("specversion", ["1.0", "0.3"]) def test_empty_data_binary_event(specversion): # Testing if cloudevent breaks when no structured data field present headers = { @@ -302,10 +293,10 @@ def test_empty_data_binary_event(specversion): "ce-time": "2018-10-23T12:28:22.4579346Z", "ce-source": "", } - _ = CloudEvent.from_http('', headers) + _ = CloudEvent.from_http("", headers) -@pytest.mark.parametrize("specversion", ['1.0', '0.3']) +@pytest.mark.parametrize("specversion", ["1.0", "0.3"]) def test_valid_structured_events(specversion): # Test creating multiple cloud events events_queue = [] @@ -317,13 +308,17 @@ def test_valid_structured_events(specversion): "source": f"source{i}.com.test", "type": f"cloudevent.test.type", "specversion": specversion, - "data": {'payload': f"payload-{i}"} + "data": {"payload": f"payload-{i}"}, } - events_queue.append(CloudEvent.from_http(json.dumps(event), - {"content-type": "application/cloudevents+json"})) + events_queue.append( + CloudEvent.from_http( + json.dumps(event), + {"content-type": "application/cloudevents+json"}, + ) + ) for i, event in enumerate(events_queue): - assert event['id'] == f"id{i}" - assert event['source'] == f"source{i}.com.test" - assert event['specversion'] == specversion - assert event.data['payload'] == f"payload-{i}" + assert event["id"] == f"id{i}" + assert event["source"] == f"source{i}.com.test" + assert event["specversion"] == specversion + assert event.data["payload"] == f"payload-{i}" diff --git a/cloudevents/tests/test_with_sanic.py b/cloudevents/tests/test_with_sanic.py index 2fd99337..135bfd5c 100644 --- a/cloudevents/tests/test_with_sanic.py +++ b/cloudevents/tests/test_with_sanic.py @@ -12,38 +12,26 @@ # License for the specific language governing permissions and limitations # under the License. -from cloudevents.sdk import marshaller -from cloudevents.sdk import converters -from cloudevents.sdk.event import v1 - -from sanic import Sanic -from sanic import response +from sanic import Sanic, response +from cloudevents.sdk import converters, marshaller +from cloudevents.sdk.event import v1 from cloudevents.tests import data as test_data - m = marshaller.NewDefaultHTTPMarshaller() app = Sanic(__name__) @app.route("/is-ok", ["POST"]) async def is_ok(request): - m.FromRequest( - v1.Event(), - dict(request.headers), - request.body, - lambda x: x - ) + m.FromRequest(v1.Event(), dict(request.headers), request.body, lambda x: x) return response.text("OK") @app.route("/echo", ["POST"]) async def echo(request): event = m.FromRequest( - v1.Event(), - dict(request.headers), - request.body, - lambda x: x + v1.Event(), dict(request.headers), request.body, lambda x: x ) hs, body = m.ToRequest(event, converters.TypeBinary, lambda x: x) return response.text(body, headers=hs) @@ -66,7 +54,8 @@ def test_web_app_integration(): def test_web_app_echo(): _, r = app.test_client.post( - "/echo", headers=test_data.headers[v1.Event], data=test_data.body) + "/echo", headers=test_data.headers[v1.Event], data=test_data.body + ) assert r.status == 200 event = m.FromRequest(v1.Event(), dict(r.headers), r.body, lambda x: x) assert event is not None diff --git a/etc/docs_conf/conf.py b/etc/docs_conf/conf.py index 3f7eb417..9ccef129 100644 --- a/etc/docs_conf/conf.py +++ b/etc/docs_conf/conf.py @@ -19,14 +19,14 @@ # -- Project information ----------------------------------------------------- -project = 'CloudEvents Python SDK' -copyright = '2018, Denis Makogon' -author = 'Denis Makogon' +project = "CloudEvents Python SDK" +copyright = "2018, Denis Makogon" +author = "Denis Makogon" # The short X.Y version -version = '' +version = "" # The full version, including alpha/beta/rc tags -release = '' +release = "" # -- General configuration --------------------------------------------------- @@ -39,21 +39,21 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.mathjax', + "sphinx.ext.autodoc", + "sphinx.ext.mathjax", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['docstemplates'] +templates_path = ["docstemplates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ".rst" # The master toctree document. -master_doc = 'index' +master_doc = "index" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -76,7 +76,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'pyramid' +html_theme = "pyramid" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -87,7 +87,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['docsstatic'] +html_static_path = ["docsstatic"] # Custom sidebar templates, must be a dictionary that maps document names # to template names. @@ -103,7 +103,7 @@ # -- Options for HTMLHelp output --------------------------------------------- # Output file base name for HTML help builder. -htmlhelp_basename = 'CloudEventsPythonSDKdoc' +htmlhelp_basename = "CloudEventsPythonSDKdoc" # -- Options for LaTeX output ------------------------------------------------ @@ -112,15 +112,12 @@ # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. # # 'preamble': '', - # Latex figure (float) alignment # # 'figure_align': 'htbp', @@ -130,8 +127,13 @@ # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'CloudEventsPythonSDK.tex', 'CloudEvents Python SDK Documentation', - 'Denis Makogon', 'manual'), + ( + master_doc, + "CloudEventsPythonSDK.tex", + "CloudEvents Python SDK Documentation", + "Denis Makogon", + "manual", + ), ] @@ -140,8 +142,13 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - (master_doc, 'cloudeventspythonsdk', 'CloudEvents Python SDK Documentation', - [author], 1) + ( + master_doc, + "cloudeventspythonsdk", + "CloudEvents Python SDK Documentation", + [author], + 1, + ) ] @@ -151,9 +158,15 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'CloudEventsPythonSDK', 'CloudEvents Python SDK Documentation', - author, 'CloudEventsPythonSDK', 'One line description of project.', - 'Miscellaneous'), + ( + master_doc, + "CloudEventsPythonSDK", + "CloudEvents Python SDK Documentation", + author, + "CloudEventsPythonSDK", + "One line description of project.", + "Miscellaneous", + ), ] @@ -172,7 +185,7 @@ # epub_uid = '' # A list of files that should not be packed into the epub file. -epub_exclude_files = ['search.html'] +epub_exclude_files = ["search.html"] -# -- Extension configuration ------------------------------------------------- \ No newline at end of file +# -- Extension configuration ------------------------------------------------- diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..672bf5c9 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,16 @@ +[tool.black] +line-length = 80 +include = '\.pyi?$' +exclude = ''' +/( + \.git + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | _build + | buck-out + | build + | dist +)/ +''' diff --git a/samples/http-cloudevents/client.py b/samples/http-cloudevents/client.py index 5e6d7a15..ce81c2c3 100644 --- a/samples/http-cloudevents/client.py +++ b/samples/http-cloudevents/client.py @@ -11,13 +11,14 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -import sys import io -from cloudevents.sdk.http_events import CloudEvent -from cloudevents.sdk import converters +import sys import requests +from cloudevents.sdk import converters +from cloudevents.sdk.http_events import CloudEvent + def send_binary_cloud_event(url): # define cloudevents data @@ -27,20 +28,17 @@ def send_binary_cloud_event(url): "type": "template.http.binary", "dataontenttype": "application/json", # Time will be filled in automatically if not set - "time": "2018-10-23T12:28:23.3464579Z" + "time": "2018-10-23T12:28:23.3464579Z", } data = {"payload-content": "Hello World!"} - # create a CloudEvent + # create a CloudEvent event = CloudEvent(attributes, data) headers, body = event.to_http(converters.TypeBinary) # send and print event requests.post(url, headers=headers, json=body) - print( - f"Sent {event['id']} from {event['source']} with " - f"{event.data}" - ) + print(f"Sent {event['id']} from {event['source']} with " f"{event.data}") def send_structured_cloud_event(url): @@ -55,24 +53,22 @@ def send_structured_cloud_event(url): } data = {"payload-content": "Hello World!"} - # create a CloudEvent + # create a CloudEvent event = CloudEvent(attributes, data) headers, body = event.to_request() # send and print event requests.post(url, headers=headers, json=body) - print( - f"Sent {event['id']} from {event['source']} with " - f"{event.data}" - ) + print(f"Sent {event['id']} from {event['source']} with " f"{event.data}") if __name__ == "__main__": # expects a url from command line. # e.g. python3 client.py http://localhost:3000/ if len(sys.argv) < 2: - sys.exit("Usage: python with_requests.py " - "") + sys.exit( + "Usage: python with_requests.py " "" + ) url = sys.argv[1] send_binary_cloud_event(url) diff --git a/samples/http-cloudevents/server.py b/samples/http-cloudevents/server.py index 3797239e..e342dab8 100644 --- a/samples/http-cloudevents/server.py +++ b/samples/http-cloudevents/server.py @@ -11,13 +11,15 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -from cloudevents.sdk.http_events import CloudEvent from flask import Flask, request + +from cloudevents.sdk.http_events import CloudEvent + app = Flask(__name__) # create an endpoint at http://localhost:/3000/ -@app.route('/', methods=['POST']) +@app.route("/", methods=["POST"]) def home(): # convert headers to dict print(request.get_data()) @@ -27,8 +29,8 @@ def home(): # print the received CloudEvent print(f"Received CloudEvent {event}") - return '', 204 + return "", 204 -if __name__ == '__main__': +if __name__ == "__main__": app.run(port=3000) diff --git a/samples/python-requests/cloudevent_to_request.py b/samples/python-requests/cloudevent_to_request.py index 0ae1d113..3df3f33c 100644 --- a/samples/python-requests/cloudevent_to_request.py +++ b/samples/python-requests/cloudevent_to_request.py @@ -13,25 +13,24 @@ # under the License. import json -import requests import sys -from cloudevents.sdk import converters -from cloudevents.sdk import marshaller +import requests +from cloudevents.sdk import converters, marshaller from cloudevents.sdk.event import v1 def run_binary(event, url): binary_headers, binary_data = http_marshaller.ToRequest( - event, converters.TypeBinary, json.dumps) + event, converters.TypeBinary, json.dumps + ) print("binary CloudEvent") for k, v in binary_headers.items(): print("{0}: {1}\r\n".format(k, v)) print(binary_data) - response = requests.post( - url, headers=binary_headers, data=binary_data) + response = requests.post(url, headers=binary_headers, data=binary_data) response.raise_for_status() @@ -42,30 +41,32 @@ def run_structured(event, url): print("structured CloudEvent") print(structured_data.getvalue()) - response = requests.post(url, - headers=structured_headers, - data=structured_data.getvalue()) + response = requests.post( + url, headers=structured_headers, data=structured_data.getvalue() + ) response.raise_for_status() if __name__ == "__main__": if len(sys.argv) < 3: - sys.exit("Usage: python with_requests.py " - "[binary | structured] " - "") + sys.exit( + "Usage: python with_requests.py " + "[binary | structured] " + "" + ) fmt = sys.argv[1] url = sys.argv[2] http_marshaller = marshaller.NewDefaultHTTPMarshaller() event = ( - v1.Event(). - SetContentType("application/json"). - SetData({"name": "denis"}). - SetEventID("my-id"). - SetSource("") + sys.exit("Usage: python with_requests.py " "") url = sys.argv[1] response = requests.get(url) @@ -35,7 +33,6 @@ data = io.BytesIO(response.content) event = v1.Event() http_marshaller = marshaller.NewDefaultHTTPMarshaller() - event = http_marshaller.FromRequest( - event, headers, data, json.load) + event = http_marshaller.FromRequest(event, headers, data, json.load) print(json.dumps(event.Properties())) diff --git a/setup.py b/setup.py index b242731f..93335285 100644 --- a/setup.py +++ b/setup.py @@ -15,6 +15,4 @@ import setuptools -setuptools.setup( - setup_requires=['pbr>=2.0.0'], - pbr=True) +setuptools.setup(setup_requires=["pbr>=2.0.0"], pbr=True) diff --git a/tox.ini b/tox.ini index b3a89f9c..e975ea4c 100644 --- a/tox.ini +++ b/tox.ini @@ -11,13 +11,21 @@ setenv = PYTESTARGS = -v -s --tb=long --cov=cloudevents commands = pytest {env:PYTESTARGS} {posargs} +[testenv:reformat] +basepython=python3.8 +deps = + black + isort +commands = + black . + isort cloudevents samples + [testenv:lint] basepython = python3.8 +deps = + black + isort commands = - flake8 + black --check . + isort -c cloudevents samples -[flake8] -# See https://gitlab.com/pycqa/flake8/-/issues/466 for extend-ignore vs ignore -extend-ignore = H405,H404,H403,H401,H306,S101,N802,N803,N806,I202,I201 -show-source = True -exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build,docs,venv,.venv,docs,etc,samples,tests From 2878c11cc4ed90ce72d803e17864787e4d4d106d Mon Sep 17 00:00:00 2001 From: Curtis Mason <31265687+cumason123@users.noreply.github.com> Date: Wed, 15 Jul 2020 20:19:51 -0400 Subject: [PATCH 07/10] README and http-cloudevents sample code adjustments to reflect new CloudEvent (#56) * README and http-cloudevents CloudEvent adjustments README no longer shows how to use base event classes to create events. Removed this because users shouldn't be forced to interact with the marshaller class. Additionally, CloudEvent is a simpler interface therefore we are encouraging the CloudEvent class usage. http-cloudevents now has more example usage for the getitem overload. Similarly README shows how to use getitem overload. Signed-off-by: Curtis Mason * lint reformat Signed-off-by: Curtis Mason * resolved nits Signed-off-by: Curtis Mason * lint fix Signed-off-by: Curtis Mason * renamed /mycontext to url Signed-off-by: Curtis Mason * renamed here linlk to in the samples directory Signed-off-by: Curtis Mason --- README.md | 157 ++++++----------------------- samples/http-cloudevents/client.py | 34 +++---- samples/http-cloudevents/server.py | 17 +++- 3 files changed, 55 insertions(+), 153 deletions(-) diff --git a/README.md b/README.md index f911f0d2..2c15c413 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,10 @@ This SDK current supports the following versions of CloudEvents: Package **cloudevents** provides primitives to work with CloudEvents specification: https://github.com/cloudevents/spec. -Sending CloudEvents: +## Sending CloudEvents + +Below we will provide samples on how to send cloudevents using the popular +[`requests`](http://docs.python-requests.org) library. ### Binary HTTP CloudEvent @@ -26,10 +29,8 @@ import requests # This data defines a binary cloudevent attributes = { - "Content-Type": "application/json", - "type": "README.sample.binary", - "id": "binary-event", - "source": "README", + "type": "com.example.sampletype1", + "source": "https://example.com/event-producer", } data = {"message": "Hello World!"} @@ -49,9 +50,8 @@ import requests # This data defines a structured cloudevent attributes = { - "type": "README.sample.structured", - "id": "structured-event", - "source": "README", + "type": "com.example.sampletype2", + "source": "https://example.com/event-producer", } data = {"message": "Hello World!"} event = CloudEvent(attributes, data) @@ -61,140 +61,44 @@ headers, body = event.to_http() requests.post("", data=body, headers=headers) ``` -### Event base classes usage - -Parsing upstream structured Event from HTTP request: - -```python -import io - -from cloudevents.sdk.event import v1 -from cloudevents.sdk import marshaller - -m = marshaller.NewDefaultHTTPMarshaller() - -event = m.FromRequest( - v1.Event(), - {"content-type": "application/cloudevents+json"}, - """ - { - "specversion": "1.0", - "datacontenttype": "application/json", - "type": "word.found.name", - "id": "96fb5f0b-001e-0108-6dfe-da6e2806f124", - "time": "2018-10-23T12:28:22.4579346Z", - "source": "" - } - """, -) -``` - -Parsing upstream binary Event from HTTP request: +You can find a complete example of turning a CloudEvent into a HTTP request [in the samples directory](samples/http-cloudevents/client.py). -```python -import io - -from cloudevents.sdk.event import v1 -from cloudevents.sdk import marshaller - -m = marshaller.NewDefaultHTTPMarshaller() - -event = m.FromRequest( - v1.Event(), - { - "ce-specversion": "1.0", - "content-type": "application/octet-stream", - "ce-type": "word.found.name", - "ce-id": "96fb5f0b-001e-0108-6dfe-da6e2806f124", - "ce-time": "2018-10-23T12:28:22.4579346Z", - "ce-source": "", - }, - b"this is where your CloudEvent data is, including image/jpeg", - lambda x: x, # Do not decode body as JSON -) -``` +#### Request to CloudEvent -Creating HTTP request from CloudEvent: +The code below shows how to consume a cloudevent using the popular python web framework +[flask](https://flask.palletsprojects.com/en/1.1.x/quickstart/): ```python -from cloudevents.sdk import converters -from cloudevents.sdk import marshaller -from cloudevents.sdk.converters import structured -from cloudevents.sdk.event import v1 - -event = ( - v1.Event() - .SetContentType("application/json") - .SetData('{"name":"john"}') - .SetEventID("my-id") - .SetSource("from-galaxy-far-far-away") - .SetEventTime("tomorrow") - .SetEventType("cloudevent.greet.you") -) - -m = marshaller.NewDefaultHTTPMarshaller() - -headers, body = m.ToRequest(event, converters.TypeStructured, lambda x: x) -``` - -## HOWTOs with various Python HTTP frameworks - -In this topic you'd find various example how to integrate an SDK with various HTTP frameworks. +from flask import Flask, request -### Python requests - -One of popular framework is [`requests`](http://docs.python-requests.org/en/master/). - -#### CloudEvent to request - -The code below shows how integrate both libraries in order to convert a CloudEvent into an HTTP request: - -```python from cloudevents.sdk.http_events import CloudEvent -from cloudevents.sdk.types import converters - -def run_binary(event: CloudEvent, url): - # If event.data is a custom type, set data_marshaller as well - headers, data = event.to_http(converters.TypeBinary) - print("binary CloudEvent") - for k, v in headers.items(): - print("{0}: {1}\r\n".format(k, v)) - print(data) - response = requests.post(url, - headers=headers, - data=data) - response.raise_for_status() +app = Flask(__name__) -def run_structured(event: CloudEvent, url): - # If event.data is a custom type, set data_marshaller as well - headers, data = event.to_http(converters.TypeStructured) +# create an endpoint at http://localhost:/3000/ +@app.route("/", methods=["POST"]) +def home(): + # create a CloudEvent + event = CloudEvent.from_http(request.get_data(), request.headers) - print("structured CloudEvent") - print(data) + # you can access cloudevent fields as seen below + print(f"Found CloudEvent from {event['source']} with specversion {event['specversion']}") - response = requests.post(url, - headers=headers, - data=data) - response.raise_for_status() + if event['type'] == 'com.example.sampletype1': + print(f"CloudEvent {event['id']} is binary") -``` - -Complete example of turning a CloudEvent into a request you can find [here](samples/python-requests/cloudevent_to_request.py). - -#### Request to CloudEvent + elif event['type'] == 'com.example.sampletype2': + print(f"CloudEvent {event['id']} is structured") -The code below shows how integrate both libraries in order to create a CloudEvent from an HTTP response: + return "", 204 -```python - response = requests.get(url) - response.raise_for_status() - event = CloudEvent.from_http(response.content, response.headers) +if __name__ == "__main__": + app.run(port=3000) ``` -Complete example of turning a CloudEvent into a request you can find [here](samples/python-requests/request_to_cloudevent.py). +You can find a complete example of turning a CloudEvent into a HTTP request [in the samples directory](samples/http-cloudevents/server.py). ## SDK versioning @@ -221,9 +125,10 @@ the same API. It will use semantic versioning with following rules: ## Maintenance We use black and isort for autoformatting. We setup a tox environment to reformat -the codebase. +the codebase. e.g. + ```python pip install tox tox -e reformat diff --git a/samples/http-cloudevents/client.py b/samples/http-cloudevents/client.py index ce81c2c3..a68cd7b8 100644 --- a/samples/http-cloudevents/client.py +++ b/samples/http-cloudevents/client.py @@ -21,44 +21,34 @@ def send_binary_cloud_event(url): - # define cloudevents data + # This data defines a binary cloudevent attributes = { - "id": "binary-event-id", - "source": "localhost", - "type": "template.http.binary", - "dataontenttype": "application/json", - # Time will be filled in automatically if not set - "time": "2018-10-23T12:28:23.3464579Z", + "type": "com.example.sampletype1", + "source": "https://example.com/event-producer", } - data = {"payload-content": "Hello World!"} + data = {"message": "Hello World!"} - # create a CloudEvent event = CloudEvent(attributes, data) headers, body = event.to_http(converters.TypeBinary) # send and print event - requests.post(url, headers=headers, json=body) + requests.post(url, data=body, headers=headers) print(f"Sent {event['id']} from {event['source']} with " f"{event.data}") def send_structured_cloud_event(url): - # define cloudevents data + # This data defines a binary cloudevent attributes = { - "id": "structured-event-id", - "source": "localhost", - "type": "template.http.structured", - "datacontenttype": "application/json", - # Time will be filled in automatically if not set - "time": "2018-10-23T12:28:23.3464579Z", + "type": "com.example.sampletype2", + "source": "https://example.com/event-producer", } - data = {"payload-content": "Hello World!"} + data = {"message": "Hello World!"} - # create a CloudEvent event = CloudEvent(attributes, data) - headers, body = event.to_request() + headers, body = event.to_http(converters.TypeBinary) - # send and print event - requests.post(url, headers=headers, json=body) + # POST + requests.post(url, data=body, headers=headers) print(f"Sent {event['id']} from {event['source']} with " f"{event.data}") diff --git a/samples/http-cloudevents/server.py b/samples/http-cloudevents/server.py index e342dab8..b461b818 100644 --- a/samples/http-cloudevents/server.py +++ b/samples/http-cloudevents/server.py @@ -21,14 +21,21 @@ # create an endpoint at http://localhost:/3000/ @app.route("/", methods=["POST"]) def home(): - # convert headers to dict - print(request.get_data()) - print(request.headers) # create a CloudEvent event = CloudEvent.from_http(request.get_data(), request.headers) + print(event) + + # you can access cloudevent fields as seen below + print( + f"Found CloudEvent from {event['source']} with specversion {event['specversion']}" + ) + + if event["type"] == "com.example.sampletype1": + print(f"CloudEvent {event['id']} is binary") + + elif event["type"] == "com.example.sampletype2": + print(f"CloudEvent {event['id']} is structured") - # print the received CloudEvent - print(f"Received CloudEvent {event}") return "", 204 From 61306dbf7f106c137bdc6e4296b36eb5c1abef38 Mon Sep 17 00:00:00 2001 From: Curtis Mason <31265687+cumason123@users.noreply.github.com> Date: Mon, 20 Jul 2020 17:09:30 -0400 Subject: [PATCH 08/10] Separated http methods (#60) * instantiated http path Signed-off-by: Curtis Mason * moved from_http from CloudEvent to http Signed-off-by: Curtis Mason * Moved to_http out of CloudEvent Signed-off-by: Curtis Mason * moved http library into event.py Signed-off-by: Curtis Mason * testing printable cloudevent Signed-off-by: Curtis Mason * Adjusted README Signed-off-by: Curtis Mason * Created EventClass Signed-off-by: Curtis Mason * reformatted event.py Signed-off-by: Curtis Mason * from_json definition Signed-off-by: Curtis Mason * server print changes Signed-off-by: Curtis Mason --- README.md | 24 ++- cloudevents/sdk/http/__init__.py | 69 ++++++++ .../sdk/{http_events.py => http/event.py} | 147 +++++++++--------- cloudevents/tests/test_http_events.py | 55 +++++-- samples/http-cloudevents/client.py | 13 +- samples/http-cloudevents/server.py | 14 +- 6 files changed, 205 insertions(+), 117 deletions(-) create mode 100644 cloudevents/sdk/http/__init__.py rename cloudevents/sdk/{http_events.py => http/event.py} (58%) diff --git a/README.md b/README.md index 2c15c413..acf934eb 100644 --- a/README.md +++ b/README.md @@ -22,8 +22,7 @@ Below we will provide samples on how to send cloudevents using the popular ### Binary HTTP CloudEvent ```python -from cloudevents.sdk import converters -from cloudevents.sdk.http_events import CloudEvent +from cloudevents.sdk.http import CloudEvent, to_binary_http import requests @@ -35,7 +34,7 @@ attributes = { data = {"message": "Hello World!"} event = CloudEvent(attributes, data) -headers, body = event.to_http(converters.TypeBinary) +headers, body = to_binary_http(event) # POST requests.post("", data=body, headers=headers) @@ -44,7 +43,7 @@ requests.post("", data=body, headers=headers) ### Structured HTTP CloudEvent ```python -from cloudevents.sdk.http_events import CloudEvent +from cloudevents.sdk.http import CloudEvent, to_structured_http import requests @@ -55,7 +54,7 @@ attributes = { } data = {"message": "Hello World!"} event = CloudEvent(attributes, data) -headers, body = event.to_http() +headers, body = to_structured_http(event) # POST requests.post("", data=body, headers=headers) @@ -71,7 +70,7 @@ The code below shows how to consume a cloudevent using the popular python web fr ```python from flask import Flask, request -from cloudevents.sdk.http_events import CloudEvent +from cloudevents.sdk.http import from_http app = Flask(__name__) @@ -80,16 +79,13 @@ app = Flask(__name__) @app.route("/", methods=["POST"]) def home(): # create a CloudEvent - event = CloudEvent.from_http(request.get_data(), request.headers) + event = from_http(request.get_data(), request.headers) # you can access cloudevent fields as seen below - print(f"Found CloudEvent from {event['source']} with specversion {event['specversion']}") - - if event['type'] == 'com.example.sampletype1': - print(f"CloudEvent {event['id']} is binary") - - elif event['type'] == 'com.example.sampletype2': - print(f"CloudEvent {event['id']} is structured") + print( + f"Found {event['id']} from {event['source']} with type " + f"{event['type']} and specversion {event['specversion']}" + ) return "", 204 diff --git a/cloudevents/sdk/http/__init__.py b/cloudevents/sdk/http/__init__.py new file mode 100644 index 00000000..2f6d9d38 --- /dev/null +++ b/cloudevents/sdk/http/__init__.py @@ -0,0 +1,69 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import json +import typing + +from cloudevents.sdk import converters, marshaller, types +from cloudevents.sdk.event import v1, v03 +from cloudevents.sdk.http.event import ( + EventClass, + to_binary_http, + to_structured_http, +) + + +class CloudEvent(EventClass): + def __repr__(self): + return to_structured_http(self)[1].decode() + + +def _json_or_string(content: typing.Union[str, bytes]): + if len(content) == 0: + return None + try: + return json.loads(content) + except json.JSONDecodeError: + return content + + +def from_http( + data: typing.Union[str, bytes], + headers: typing.Dict[str, str], + data_unmarshaller: types.UnmarshallerType = None, +) -> CloudEvent: + + """Unwrap a CloudEvent (binary or structured) from an HTTP request. + :param data: the HTTP request body + :type data: typing.IO + :param headers: the HTTP headers + :type headers: typing.Dict[str, str] + :param data_unmarshaller: Function to decode data into a python object. + :type data_unmarshaller: types.UnmarshallerType + """ + if data_unmarshaller is None: + data_unmarshaller = _json_or_string + + event = marshaller.NewDefaultHTTPMarshaller().FromRequest( + v1.Event(), headers, data, data_unmarshaller + ) + attrs = event.Properties() + attrs.pop("data", None) + attrs.pop("extensions", None) + attrs.update(**event.extensions) + + return CloudEvent(attrs, event.data) + + +def from_json(): + raise NotImplementedError diff --git a/cloudevents/sdk/http_events.py b/cloudevents/sdk/http/event.py similarity index 58% rename from cloudevents/sdk/http_events.py rename to cloudevents/sdk/http/event.py index 9ca3abc6..9791ff15 100644 --- a/cloudevents/sdk/http_events.py +++ b/cloudevents/sdk/http/event.py @@ -24,56 +24,21 @@ converters.TypeStructured: lambda x: x, converters.TypeBinary: json.dumps, } + +_obj_by_version = {"1.0": v1.Event, "0.3": v03.Event} + _required_by_version = { "1.0": v1.Event._ce_required_fields, "0.3": v03.Event._ce_required_fields, } -_obj_by_version = {"1.0": v1.Event, "0.3": v03.Event} - - -def _json_or_string(content: typing.Union[str, bytes]): - if len(content) == 0: - return None - try: - return json.loads(content) - except json.JSONDecodeError: - return content -class CloudEvent: +class EventClass: """ Python-friendly cloudevent class supporting v1 events Supports both binary and structured mode CloudEvents """ - @classmethod - def from_http( - cls, - data: typing.Union[str, bytes], - headers: typing.Dict[str, str], - data_unmarshaller: types.UnmarshallerType = None, - ): - """Unwrap a CloudEvent (binary or structured) from an HTTP request. - :param data: the HTTP request body - :type data: typing.IO - :param headers: the HTTP headers - :type headers: typing.Dict[str, str] - :param data_unmarshaller: Function to decode data into a python object. - :type data_unmarshaller: types.UnmarshallerType - """ - if data_unmarshaller is None: - data_unmarshaller = _json_or_string - - event = marshaller.NewDefaultHTTPMarshaller().FromRequest( - v1.Event(), headers, data, data_unmarshaller - ) - attrs = event.Properties() - attrs.pop("data", None) - attrs.pop("extensions", None) - attrs.update(**event.extensions) - - return cls(attrs, event.data) - def __init__( self, attributes: typing.Dict[str, str], data: typing.Any = None ): @@ -114,36 +79,6 @@ def __init__( f"Missing required keys: {required_set - attributes.keys()}" ) - def to_http( - self, - format: str = converters.TypeStructured, - data_marshaller: types.MarshallerType = None, - ) -> (dict, typing.Union[bytes, str]): - """ - Returns a tuple of HTTP headers/body dicts representing this cloudevent - - :param format: constant specifying an encoding format - :type format: str - :param data_unmarshaller: Function used to read the data to string. - :type data_unmarshaller: types.UnmarshallerType - :returns: (http_headers: dict, http_body: bytes or str) - """ - if data_marshaller is None: - data_marshaller = _marshaller_by_format[format] - if self._attributes["specversion"] not in _obj_by_version: - raise ValueError( - f"Unsupported specversion: {self._attributes['specversion']}" - ) - - event = _obj_by_version[self._attributes["specversion"]]() - for k, v in self._attributes.items(): - event.Set(k, v) - event.data = self.data - - return marshaller.NewDefaultHTTPMarshaller().ToRequest( - event, format, data_marshaller - ) - # Data access is handled via `.data` member # Attribute access is managed via Mapping type def __getitem__(self, key): @@ -164,5 +99,75 @@ def __len__(self): def __contains__(self, key): return key in self._attributes - def __repr__(self): - return self.to_http()[1].decode() + +def _to_http( + event: EventClass, + format: str = converters.TypeStructured, + data_marshaller: types.MarshallerType = None, +) -> (dict, typing.Union[bytes, str]): + """ + Returns a tuple of HTTP headers/body dicts representing this cloudevent + + :param format: constant specifying an encoding format + :type format: str + :param data_unmarshaller: Function used to read the data to string. + :type data_unmarshaller: types.UnmarshallerType + :returns: (http_headers: dict, http_body: bytes or str) + """ + if data_marshaller is None: + data_marshaller = _marshaller_by_format[format] + if event._attributes["specversion"] not in _obj_by_version: + raise ValueError( + f"Unsupported specversion: {event._attributes['specversion']}" + ) + + event_handler = _obj_by_version[event._attributes["specversion"]]() + for k, v in event._attributes.items(): + event_handler.Set(k, v) + event_handler.data = event.data + + return marshaller.NewDefaultHTTPMarshaller().ToRequest( + event_handler, format, data_marshaller + ) + + +def to_structured_http( + event: EventClass, data_marshaller: types.MarshallerType = None, +) -> (dict, typing.Union[bytes, str]): + """ + Returns a tuple of HTTP headers/body dicts representing this cloudevent + + :param event: CloudEvent to cast into http data + :type event: CloudEvent + :param data_unmarshaller: Function used to read the data to string. + :type data_unmarshaller: types.UnmarshallerType + :returns: (http_headers: dict, http_body: bytes or str) + """ + return _to_http(event=event, data_marshaller=data_marshaller) + + +def to_binary_http( + event: EventClass, data_marshaller: types.MarshallerType = None, +) -> (dict, typing.Union[bytes, str]): + """ + Returns a tuple of HTTP headers/body dicts representing this cloudevent + + :param event: CloudEvent to cast into http data + :type event: CloudEvent + :param data_unmarshaller: Function used to read the data to string. + :type data_unmarshaller: types.UnmarshallerType + :returns: (http_headers: dict, http_body: bytes or str) + """ + return _to_http( + event=event, + format=converters.TypeBinary, + data_marshaller=data_marshaller, + ) + + +def to_json(): + raise NotImplementedError + + +def from_json(): + raise NotImplementedError diff --git a/cloudevents/tests/test_http_events.py b/cloudevents/tests/test_http_events.py index 6213edda..35312a30 100644 --- a/cloudevents/tests/test_http_events.py +++ b/cloudevents/tests/test_http_events.py @@ -21,7 +21,12 @@ from sanic import Sanic, response from cloudevents.sdk import converters -from cloudevents.sdk.http_events import CloudEvent +from cloudevents.sdk.http import ( + CloudEvent, + from_http, + to_binary_http, + to_structured_http, +) invalid_test_headers = [ { @@ -71,7 +76,7 @@ async def echo(request): decoder = None if "binary-payload" in request.headers: decoder = lambda x: x - event = CloudEvent.from_http( + event = from_http( request.body, headers=dict(request.headers), data_unmarshaller=decoder ) data = ( @@ -89,7 +94,7 @@ def test_missing_required_fields_structured(body): # and NotImplementedError because structured calls aren't # implemented. In this instance one of the required keys should have # prefix e-id instead of ce-id therefore it should throw - _ = CloudEvent.from_http( + _ = from_http( json.dumps(body), attributes={"Content-Type": "application/json"} ) @@ -101,7 +106,7 @@ def test_missing_required_fields_binary(headers): # and NotImplementedError because structured calls aren't # implemented. In this instance one of the required keys should have # prefix e-id instead of ce-id therefore it should throw - _ = CloudEvent.from_http(json.dumps(test_data), headers=headers) + _ = from_http(json.dumps(test_data), headers=headers) @pytest.mark.parametrize("specversion", ["1.0", "0.3"]) @@ -155,7 +160,7 @@ def test_emit_structured_event(specversion): @pytest.mark.parametrize( - "converter", [converters.TypeStructured, converters.TypeStructured] + "converter", [converters.TypeBinary, converters.TypeStructured] ) @pytest.mark.parametrize("specversion", ["1.0", "0.3"]) def test_roundtrip_non_json_event(converter, specversion): @@ -167,7 +172,12 @@ def test_roundtrip_non_json_event(converter, specversion): attrs = {"source": "test", "type": "t"} event = CloudEvent(attrs, compressed_data) - headers, data = event.to_http(converter, data_marshaller=lambda x: x) + + if converter == converters.TypeStructured: + headers, data = to_structured_http(event, data_marshaller=lambda x: x) + elif converter == converters.TypeBinary: + headers, data = to_binary_http(event, data_marshaller=lambda x: x) + headers["binary-payload"] = "true" # Decoding hint for server _, r = app.test_client.post("/event", headers=headers, data=data) @@ -196,7 +206,7 @@ def test_missing_ce_prefix_binary_event(specversion): # and NotImplementedError because structured calls aren't # implemented. In this instance one of the required keys should have # prefix e-id instead of ce-id therefore it should throw - _ = CloudEvent.from_http(test_data, headers=prefixed_headers) + _ = from_http(test_data, headers=prefixed_headers) @pytest.mark.parametrize("specversion", ["1.0", "0.3"]) @@ -213,9 +223,7 @@ def test_valid_binary_events(specversion): "ce-specversion": specversion, } data = {"payload": f"payload-{i}"} - events_queue.append( - CloudEvent.from_http(json.dumps(data), headers=headers) - ) + events_queue.append(from_http(json.dumps(data), headers=headers)) for i, event in enumerate(events_queue): data = event.data @@ -236,7 +244,7 @@ def test_structured_to_request(specversion): data = {"message": "Hello World!"} event = CloudEvent(attributes, data) - headers, body_bytes = event.to_http() + headers, body_bytes = to_structured_http(event) assert isinstance(body_bytes, bytes) body = json.loads(body_bytes) @@ -256,7 +264,7 @@ def test_binary_to_request(specversion): } data = {"message": "Hello World!"} event = CloudEvent(attributes, data) - headers, body_bytes = event.to_http(converters.TypeBinary) + headers, body_bytes = to_binary_http(event) body = json.loads(body_bytes) for key in data: @@ -277,7 +285,7 @@ def test_empty_data_structured_event(specversion): "source": "", } - _ = CloudEvent.from_http( + _ = from_http( json.dumps(attributes), {"content-type": "application/cloudevents+json"} ) @@ -293,7 +301,7 @@ def test_empty_data_binary_event(specversion): "ce-time": "2018-10-23T12:28:22.4579346Z", "ce-source": "", } - _ = CloudEvent.from_http("", headers) + _ = from_http("", headers) @pytest.mark.parametrize("specversion", ["1.0", "0.3"]) @@ -311,7 +319,7 @@ def test_valid_structured_events(specversion): "data": {"payload": f"payload-{i}"}, } events_queue.append( - CloudEvent.from_http( + from_http( json.dumps(event), {"content-type": "application/cloudevents+json"}, ) @@ -322,3 +330,20 @@ def test_valid_structured_events(specversion): assert event["source"] == f"source{i}.com.test" assert event["specversion"] == specversion assert event.data["payload"] == f"payload-{i}" + + +@pytest.mark.parametrize("specversion", ["1.0", "0.3"]) +def test_cloudevent_repr(specversion): + headers = { + "Content-Type": "application/octet-stream", + "ce-specversion": specversion, + "ce-type": "word.found.name", + "ce-id": "96fb5f0b-001e-0108-6dfe-da6e2806f124", + "ce-time": "2018-10-23T12:28:22.4579346Z", + "ce-source": "", + } + event = from_http("", headers) + # Testing to make sure event is printable. I could runevent. __repr__() but + # we had issues in the past where event.__repr__() could run but + # print(event) would fail. + print(event) diff --git a/samples/http-cloudevents/client.py b/samples/http-cloudevents/client.py index a68cd7b8..c7a507ad 100644 --- a/samples/http-cloudevents/client.py +++ b/samples/http-cloudevents/client.py @@ -16,8 +16,7 @@ import requests -from cloudevents.sdk import converters -from cloudevents.sdk.http_events import CloudEvent +from cloudevents.sdk.http import CloudEvent, to_binary_http, to_structured_http def send_binary_cloud_event(url): @@ -29,10 +28,10 @@ def send_binary_cloud_event(url): data = {"message": "Hello World!"} event = CloudEvent(attributes, data) - headers, body = event.to_http(converters.TypeBinary) + headers, body = to_binary_http(event) # send and print event - requests.post(url, data=body, headers=headers) + requests.post(url, headers=headers, data=body) print(f"Sent {event['id']} from {event['source']} with " f"{event.data}") @@ -45,10 +44,10 @@ def send_structured_cloud_event(url): data = {"message": "Hello World!"} event = CloudEvent(attributes, data) - headers, body = event.to_http(converters.TypeBinary) + headers, body = to_structured_http(event) - # POST - requests.post(url, data=body, headers=headers) + # send and print event + requests.post(url, headers=headers, data=body) print(f"Sent {event['id']} from {event['source']} with " f"{event.data}") diff --git a/samples/http-cloudevents/server.py b/samples/http-cloudevents/server.py index b461b818..e1fbfda0 100644 --- a/samples/http-cloudevents/server.py +++ b/samples/http-cloudevents/server.py @@ -13,7 +13,7 @@ # under the License. from flask import Flask, request -from cloudevents.sdk.http_events import CloudEvent +from cloudevents.sdk.http import from_http app = Flask(__name__) @@ -22,20 +22,14 @@ @app.route("/", methods=["POST"]) def home(): # create a CloudEvent - event = CloudEvent.from_http(request.get_data(), request.headers) - print(event) + event = from_http(request.get_data(), request.headers) # you can access cloudevent fields as seen below print( - f"Found CloudEvent from {event['source']} with specversion {event['specversion']}" + f"Found {event['id']} from {event['source']} with type " + f"{event['type']} and specversion {event['specversion']}" ) - if event["type"] == "com.example.sampletype1": - print(f"CloudEvent {event['id']} is binary") - - elif event["type"] == "com.example.sampletype2": - print(f"CloudEvent {event['id']} is structured") - return "", 204 From 6081d693181f57b0ada9bc580bb5949d77fc3fb5 Mon Sep 17 00:00:00 2001 From: Curtis Mason <31265687+cumason123@users.noreply.github.com> Date: Mon, 20 Jul 2020 17:57:48 -0400 Subject: [PATCH 09/10] Specversion toggling (#57) * cloudevent now switches specversion types Signed-off-by: Curtis Mason * removed duplicate marshall instance Signed-off-by: Curtis Mason * resolved grant requests Signed-off-by: Curtis Mason * converters now can check headers for fields Signed-off-by: Curtis Mason * removed print statement Signed-off-by: Curtis Mason * Fixed marshallers looking at headers for specversion Signed-off-by: Curtis Mason * lint fixes Signed-off-by: Curtis Mason * is_binary static method and structured isinstance rework Signed-off-by: Curtis Mason * testing for is_binary and is_structured Signed-off-by: Curtis Mason --- cloudevents/sdk/converters/__init__.py | 28 ++++++++++++ cloudevents/sdk/converters/binary.py | 12 ++++- cloudevents/sdk/converters/structured.py | 11 ++++- cloudevents/sdk/http/__init__.py | 38 +++++++++++----- cloudevents/sdk/http/event.py | 2 + cloudevents/sdk/marshaller.py | 18 ++++---- cloudevents/tests/test_http_events.py | 56 +++++++++++++++++++++++- 7 files changed, 141 insertions(+), 24 deletions(-) diff --git a/cloudevents/sdk/converters/__init__.py b/cloudevents/sdk/converters/__init__.py index 1e786455..289cfab4 100644 --- a/cloudevents/sdk/converters/__init__.py +++ b/cloudevents/sdk/converters/__init__.py @@ -12,7 +12,35 @@ # License for the specific language governing permissions and limitations # under the License. +import typing + from cloudevents.sdk.converters import binary, structured TypeBinary = binary.BinaryHTTPCloudEventConverter.TYPE TypeStructured = structured.JSONHTTPCloudEventConverter.TYPE + + +def is_binary(headers: typing.Dict[str, str]) -> bool: + """Uses internal marshallers to determine whether this event is binary + :param headers: the HTTP headers + :type headers: typing.Dict[str, str] + :returns bool: returns a bool indicating whether the headers indicate a binary event type + """ + headers = {key.lower(): value for key, value in headers.items()} + content_type = headers.get("content-type", "") + binary_parser = binary.BinaryHTTPCloudEventConverter() + return binary_parser.can_read(content_type=content_type, headers=headers) + + +def is_structured(headers: typing.Dict[str, str]) -> bool: + """Uses internal marshallers to determine whether this event is structured + :param headers: the HTTP headers + :type headers: typing.Dict[str, str] + :returns bool: returns a bool indicating whether the headers indicate a structured event type + """ + headers = {key.lower(): value for key, value in headers.items()} + content_type = headers.get("content-type", "") + structured_parser = structured.JSONHTTPCloudEventConverter() + return structured_parser.can_read( + content_type=content_type, headers=headers + ) diff --git a/cloudevents/sdk/converters/binary.py b/cloudevents/sdk/converters/binary.py index abdc9d99..46277727 100644 --- a/cloudevents/sdk/converters/binary.py +++ b/cloudevents/sdk/converters/binary.py @@ -16,6 +16,7 @@ from cloudevents.sdk import exceptions, types from cloudevents.sdk.converters import base +from cloudevents.sdk.converters.structured import JSONHTTPCloudEventConverter from cloudevents.sdk.event import base as event_base from cloudevents.sdk.event import v1, v03 @@ -25,8 +26,15 @@ class BinaryHTTPCloudEventConverter(base.Converter): TYPE = "binary" SUPPORTED_VERSIONS = [v03.Event, v1.Event] - def can_read(self, content_type: str) -> bool: - return True + def can_read( + self, + content_type: str, + headers: typing.Dict[str, str] = {"ce-specversion": None}, + ) -> bool: + return ("ce-specversion" in headers) and not ( + isinstance(content_type, str) + and content_type.startswith(JSONHTTPCloudEventConverter.MIME_TYPE) + ) def event_supported(self, event: object) -> bool: return type(event) in self.SUPPORTED_VERSIONS diff --git a/cloudevents/sdk/converters/structured.py b/cloudevents/sdk/converters/structured.py index d29a9284..d6ba6548 100644 --- a/cloudevents/sdk/converters/structured.py +++ b/cloudevents/sdk/converters/structured.py @@ -24,8 +24,15 @@ class JSONHTTPCloudEventConverter(base.Converter): TYPE = "structured" MIME_TYPE = "application/cloudevents+json" - def can_read(self, content_type: str) -> bool: - return content_type and content_type.startswith(self.MIME_TYPE) + def can_read( + self, + content_type: str, + headers: typing.Dict[str, str] = {"ce-specversion": None}, + ) -> bool: + return ( + isinstance(content_type, str) + and content_type.startswith(self.MIME_TYPE) + ) or ("ce-specversion" not in headers) def event_supported(self, event: object) -> bool: # structured format supported by both spec 0.1 and 0.2 diff --git a/cloudevents/sdk/http/__init__.py b/cloudevents/sdk/http/__init__.py index 2f6d9d38..a0c30e99 100644 --- a/cloudevents/sdk/http/__init__.py +++ b/cloudevents/sdk/http/__init__.py @@ -18,6 +18,7 @@ from cloudevents.sdk.event import v1, v03 from cloudevents.sdk.http.event import ( EventClass, + _obj_by_version, to_binary_http, to_structured_http, ) @@ -41,21 +42,36 @@ def from_http( data: typing.Union[str, bytes], headers: typing.Dict[str, str], data_unmarshaller: types.UnmarshallerType = None, -) -> CloudEvent: - +): """Unwrap a CloudEvent (binary or structured) from an HTTP request. - :param data: the HTTP request body - :type data: typing.IO - :param headers: the HTTP headers - :type headers: typing.Dict[str, str] - :param data_unmarshaller: Function to decode data into a python object. - :type data_unmarshaller: types.UnmarshallerType - """ + :param data: the HTTP request body + :type data: typing.IO + :param headers: the HTTP headers + :type headers: typing.Dict[str, str] + :param data_unmarshaller: Function to decode data into a python object. + :type data_unmarshaller: types.UnmarshallerType + """ if data_unmarshaller is None: data_unmarshaller = _json_or_string - event = marshaller.NewDefaultHTTPMarshaller().FromRequest( - v1.Event(), headers, data, data_unmarshaller + marshall = marshaller.NewDefaultHTTPMarshaller() + + if converters.is_binary(headers): + specversion = headers.get("ce-specversion", None) + else: + raw_ce = json.loads(data) + specversion = raw_ce.get("specversion", None) + + if specversion is None: + raise ValueError("could not find specversion in HTTP request") + + event_handler = _obj_by_version.get(specversion, None) + + if event_handler is None: + raise ValueError(f"found invalid specversion {specversion}") + + event = marshall.FromRequest( + event_handler(), headers, data, data_unmarshaller ) attrs = event.Properties() attrs.pop("data", None) diff --git a/cloudevents/sdk/http/event.py b/cloudevents/sdk/http/event.py index 9791ff15..7a02feea 100644 --- a/cloudevents/sdk/http/event.py +++ b/cloudevents/sdk/http/event.py @@ -18,7 +18,9 @@ import uuid from cloudevents.sdk import converters, marshaller, types +from cloudevents.sdk.converters import is_binary from cloudevents.sdk.event import v1, v03 +from cloudevents.sdk.marshaller import HTTPMarshaller _marshaller_by_format = { converters.TypeStructured: lambda x: x, diff --git a/cloudevents/sdk/marshaller.py b/cloudevents/sdk/marshaller.py index d9fe4920..ed9e02a3 100644 --- a/cloudevents/sdk/marshaller.py +++ b/cloudevents/sdk/marshaller.py @@ -32,8 +32,8 @@ def __init__(self, converters: typing.List[base.Converter]): :param converters: a list of HTTP-to-CloudEvent-to-HTTP constructors :type converters: typing.List[base.Converter] """ - self.__converters = [c for c in converters] - self.__converters_by_type = {c.TYPE: c for c in converters} + self.http_converters = [c for c in converters] + self.http_converters_by_type = {c.TYPE: c for c in converters} def FromRequest( self, @@ -62,13 +62,15 @@ def FromRequest( headers = {key.lower(): value for key, value in headers.items()} content_type = headers.get("content-type", None) - for cnvrtr in self.__converters: - if cnvrtr.can_read(content_type) and cnvrtr.event_supported(event): + for cnvrtr in self.http_converters: + if cnvrtr.can_read( + content_type, headers=headers + ) and cnvrtr.event_supported(event): return cnvrtr.read(event, headers, body, data_unmarshaller) raise exceptions.UnsupportedEventConverter( "No registered marshaller for {0} in {1}".format( - content_type, self.__converters + content_type, self.http_converters ) ) @@ -95,10 +97,10 @@ def ToRequest( raise exceptions.InvalidDataMarshaller() if converter_type is None: - converter_type = self.__converters[0].TYPE + converter_type = self.http_converters[0].TYPE - if converter_type in self.__converters_by_type: - cnvrtr = self.__converters_by_type[converter_type] + if converter_type in self.http_converters_by_type: + cnvrtr = self.http_converters_by_type[converter_type] return cnvrtr.write(event, data_marshaller) raise exceptions.NoSuchConverter(converter_type) diff --git a/cloudevents/tests/test_http_events.py b/cloudevents/tests/test_http_events.py index 35312a30..eba7a20f 100644 --- a/cloudevents/tests/test_http_events.py +++ b/cloudevents/tests/test_http_events.py @@ -206,7 +206,7 @@ def test_missing_ce_prefix_binary_event(specversion): # and NotImplementedError because structured calls aren't # implemented. In this instance one of the required keys should have # prefix e-id instead of ce-id therefore it should throw - _ = from_http(test_data, headers=prefixed_headers) + _ = from_http(json.dumps(test_data), headers=prefixed_headers) @pytest.mark.parametrize("specversion", ["1.0", "0.3"]) @@ -332,6 +332,60 @@ def test_valid_structured_events(specversion): assert event.data["payload"] == f"payload-{i}" +@pytest.mark.parametrize("specversion", ["1.0", "0.3"]) +def test_structured_no_content_type(specversion): + # Test creating multiple cloud events + events_queue = [] + headers = {} + num_cloudevents = 30 + data = { + "id": "id", + "source": "source.com.test", + "type": "cloudevent.test.type", + "specversion": specversion, + "data": test_data, + } + event = from_http(json.dumps(data), {},) + + assert event["id"] == "id" + assert event["source"] == "source.com.test" + assert event["specversion"] == specversion + for key, val in test_data.items(): + assert event.data[key] == val + + +def test_is_binary(): + headers = { + "ce-id": "my-id", + "ce-source": "", + "ce-type": "cloudevent.event.type", + "ce-specversion": "1.0", + "Content-Type": "text/plain", + } + assert converters.is_binary(headers) + + headers = { + "Content-Type": "application/cloudevents+json", + } + assert not converters.is_binary(headers) + + headers = {} + assert not converters.is_binary(headers) + + +def test_is_structured(): + headers = { + "Content-Type": "application/cloudevents+json", + } + assert converters.is_structured(headers) + + headers = {} + assert converters.is_structured(headers) + + headers = {"ce-specversion": "1.0"} + assert not converters.is_structured(headers) + + @pytest.mark.parametrize("specversion", ["1.0", "0.3"]) def test_cloudevent_repr(specversion): headers = { From b2a87a8af6ce900d99f80bcea91094933dfa6e07 Mon Sep 17 00:00:00 2001 From: Curtis Mason <31265687+cumason123@users.noreply.github.com> Date: Wed, 22 Jul 2020 02:53:35 -0400 Subject: [PATCH 10/10] Image sample code (#65) * added image example Signed-off-by: Curtis Mason * moved size into headers Signed-off-by: Curtis Mason * lint fix Signed-off-by: Curtis Mason * renamed sample code Signed-off-by: Curtis Mason * added test to http-image-cloudevents sample Signed-off-by: Curtis Mason * removed unnecessary function Signed-off-by: Curtis Mason * Added testing for http-image-cloudevents Signed-off-by: Curtis Mason * Data marshall arg fix and better image in sample Fixed bug where data_marshaller and data_unmarshaller wasn't being passed into positional arguments. Also used cloudevents logo for the image in http-image-cloudevents Signed-off-by: Curtis Mason * adjusted http-image-cloudevents samples Signed-off-by: Curtis Mason * reformat and README changes Signed-off-by: Curtis Mason * io bytes casting in data_unmarshaller Signed-off-by: Curtis Mason * lint fix Signed-off-by: Curtis Mason * removed unusued imports in http-image samples Signed-off-by: Curtis Mason * removed samples/http-cloudevents/tmp.png Signed-off-by: Curtis Mason * Nits Signed-off-by: Curtis Mason --- cloudevents/sdk/http/__init__.py | 7 +- cloudevents/sdk/http/event.py | 15 +++- requirements/test.txt | 1 + samples/http-cloudevents/requirements.txt | 2 - samples/http-image-cloudevents/README.md | 26 ++++++ samples/http-image-cloudevents/client.py | 72 +++++++++++++++ .../http-image-cloudevents/requirements.txt | 4 + samples/http-image-cloudevents/server.py | 43 +++++++++ .../test_image_sample.py | 87 +++++++++++++++++++ .../README.md | 0 .../client.py | 0 .../http-json-cloudevents/requirements.txt | 3 + .../server.py | 0 .../test_verify_sample.py | 0 14 files changed, 253 insertions(+), 7 deletions(-) delete mode 100644 samples/http-cloudevents/requirements.txt create mode 100644 samples/http-image-cloudevents/README.md create mode 100644 samples/http-image-cloudevents/client.py create mode 100644 samples/http-image-cloudevents/requirements.txt create mode 100644 samples/http-image-cloudevents/server.py create mode 100644 samples/http-image-cloudevents/test_image_sample.py rename samples/{http-cloudevents => http-json-cloudevents}/README.md (100%) rename samples/{http-cloudevents => http-json-cloudevents}/client.py (100%) create mode 100644 samples/http-json-cloudevents/requirements.txt rename samples/{http-cloudevents => http-json-cloudevents}/server.py (100%) rename samples/{http-cloudevents => http-json-cloudevents}/test_verify_sample.py (100%) diff --git a/cloudevents/sdk/http/__init__.py b/cloudevents/sdk/http/__init__.py index a0c30e99..40101481 100644 --- a/cloudevents/sdk/http/__init__.py +++ b/cloudevents/sdk/http/__init__.py @@ -34,7 +34,7 @@ def _json_or_string(content: typing.Union[str, bytes]): return None try: return json.loads(content) - except json.JSONDecodeError: + except (json.JSONDecodeError, TypeError) as e: return content @@ -48,7 +48,8 @@ def from_http( :type data: typing.IO :param headers: the HTTP headers :type headers: typing.Dict[str, str] - :param data_unmarshaller: Function to decode data into a python object. + :param data_unmarshaller: Callable function to map data arg to python object + e.g. lambda x: x or lambda x: json.loads(x) :type data_unmarshaller: types.UnmarshallerType """ if data_unmarshaller is None: @@ -71,7 +72,7 @@ def from_http( raise ValueError(f"found invalid specversion {specversion}") event = marshall.FromRequest( - event_handler(), headers, data, data_unmarshaller + event_handler(), headers, data, data_unmarshaller=data_unmarshaller ) attrs = event.Properties() attrs.pop("data", None) diff --git a/cloudevents/sdk/http/event.py b/cloudevents/sdk/http/event.py index 7a02feea..a991918f 100644 --- a/cloudevents/sdk/http/event.py +++ b/cloudevents/sdk/http/event.py @@ -22,9 +22,19 @@ from cloudevents.sdk.event import v1, v03 from cloudevents.sdk.marshaller import HTTPMarshaller + +def default_marshaller(content: any): + if len(content) == 0: + return None + try: + return json.dumps(content) + except TypeError: + return content + + _marshaller_by_format = { converters.TypeStructured: lambda x: x, - converters.TypeBinary: json.dumps, + converters.TypeBinary: default_marshaller, } _obj_by_version = {"1.0": v1.Event, "0.3": v03.Event} @@ -118,6 +128,7 @@ def _to_http( """ if data_marshaller is None: data_marshaller = _marshaller_by_format[format] + if event._attributes["specversion"] not in _obj_by_version: raise ValueError( f"Unsupported specversion: {event._attributes['specversion']}" @@ -129,7 +140,7 @@ def _to_http( event_handler.data = event.data return marshaller.NewDefaultHTTPMarshaller().ToRequest( - event_handler, format, data_marshaller + event_handler, format, data_marshaller=data_marshaller ) diff --git a/requirements/test.txt b/requirements/test.txt index 12894086..4c9fb756 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -8,3 +8,4 @@ pytest-cov==2.4.0 # web app tests sanic aiohttp +Pillow diff --git a/samples/http-cloudevents/requirements.txt b/samples/http-cloudevents/requirements.txt deleted file mode 100644 index 5eaf725f..00000000 --- a/samples/http-cloudevents/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -flask -requests \ No newline at end of file diff --git a/samples/http-image-cloudevents/README.md b/samples/http-image-cloudevents/README.md new file mode 100644 index 00000000..35e758c0 --- /dev/null +++ b/samples/http-image-cloudevents/README.md @@ -0,0 +1,26 @@ +## Image Payloads Quickstart + +Install dependencies: + +```sh +pip3 install -r requirements.txt +``` + +Start server: + +```sh +python3 server.py +``` + +In a new shell, run the client code which sends a structured and binary +cloudevent to your local server: + +```sh +python3 client.py http://localhost:3000/ +``` + +## Test + +```sh +pytest +``` diff --git a/samples/http-image-cloudevents/client.py b/samples/http-image-cloudevents/client.py new file mode 100644 index 00000000..8b2fc695 --- /dev/null +++ b/samples/http-image-cloudevents/client.py @@ -0,0 +1,72 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import sys + +import requests + +from cloudevents.sdk.http import CloudEvent, to_binary_http, to_structured_http + +resp = requests.get( + "https://raw.githubusercontent.com/cncf/artwork/master/projects/cloudevents/horizontal/color/cloudevents-horizontal-color.png" +) +image_bytes = resp.content + + +def send_binary_cloud_event(url: str): + # Create cloudevent + attributes = { + "type": "com.example.string", + "source": "https://example.com/event-producer", + } + + event = CloudEvent(attributes, image_bytes) + + # Create cloudevent HTTP headers and content + headers, body = to_binary_http(event) + + # Send cloudevent + requests.post(url, headers=headers, data=body) + print(f"Sent {event['id']} of type {event['type']}") + + +def send_structured_cloud_event(url: str): + # Create cloudevent + attributes = { + "type": "com.example.base64", + "source": "https://example.com/event-producer", + } + + event = CloudEvent(attributes, image_bytes) + + # Create cloudevent HTTP headers and content + # Note that to_structured_http will create a data_base64 data field in + # specversion 1.0 (default specversion) if given + # an event whose data field is of type bytes. + headers, body = to_structured_http(event) + + # Send cloudevent + requests.post(url, headers=headers, data=body) + print(f"Sent {event['id']} of type {event['type']}") + + +if __name__ == "__main__": + # Run client.py via: 'python3 client.py http://localhost:3000/' + if len(sys.argv) < 2: + sys.exit( + "Usage: python with_requests.py " "" + ) + + url = sys.argv[1] + send_binary_cloud_event(url) + send_structured_cloud_event(url) diff --git a/samples/http-image-cloudevents/requirements.txt b/samples/http-image-cloudevents/requirements.txt new file mode 100644 index 00000000..10f72867 --- /dev/null +++ b/samples/http-image-cloudevents/requirements.txt @@ -0,0 +1,4 @@ +flask +requests +Pillow +pytest diff --git a/samples/http-image-cloudevents/server.py b/samples/http-image-cloudevents/server.py new file mode 100644 index 00000000..048d0efa --- /dev/null +++ b/samples/http-image-cloudevents/server.py @@ -0,0 +1,43 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import io + +from flask import Flask, request +from PIL import Image + +from cloudevents.sdk.http import from_http + +app = Flask(__name__) + + +@app.route("/", methods=["POST"]) +def home(): + # Create a CloudEvent. + # data_unmarshaller will cast event.data into an io.BytesIO object + event = from_http( + request.get_data(), + request.headers, + data_unmarshaller=lambda x: io.BytesIO(x), + ) + + # Create image from cloudevent data + image = Image.open(event.data) + + # Print + print(f"Found event {event['id']} with image of size {image.size}") + return "", 204 + + +if __name__ == "__main__": + app.run(port=3000) diff --git a/samples/http-image-cloudevents/test_image_sample.py b/samples/http-image-cloudevents/test_image_sample.py new file mode 100644 index 00000000..83436c1e --- /dev/null +++ b/samples/http-image-cloudevents/test_image_sample.py @@ -0,0 +1,87 @@ +import base64 +import io +import json + +import requests +from PIL import Image + +from cloudevents.sdk.http import ( + CloudEvent, + from_http, + to_binary_http, + to_structured_http, +) + +resp = requests.get( + "https://raw.githubusercontent.com/cncf/artwork/master/projects/cloudevents/horizontal/color/cloudevents-horizontal-color.png" +) +image_bytes = resp.content + +image_fileobj = io.BytesIO(image_bytes) +image_expected_shape = (1880, 363) + + +def test_create_binary_image(): + # Create image and turn image into bytes + attributes = { + "type": "com.example.string", + "source": "https://example.com/event-producer", + } + + # Create CloudEvent + event = CloudEvent(attributes, image_bytes) + + # Create http headers/body content + headers, body = to_binary_http(event) + + # Unmarshall CloudEvent and re-create image + reconstruct_event = from_http( + body, headers, data_unmarshaller=lambda x: io.BytesIO(x) + ) + + # reconstruct_event.data is an io.BytesIO object due to data_unmarshaller + restore_image = Image.open(reconstruct_event.data) + assert restore_image.size == image_expected_shape + + # # Test cloudevent extension from http fields and data + assert isinstance(body, bytes) + assert body == image_bytes + + +def test_create_structured_image(): + # Create image and turn image into bytes + attributes = { + "type": "com.example.string", + "source": "https://example.com/event-producer", + } + + # Create CloudEvent + event = CloudEvent(attributes, image_bytes) + + # Create http headers/body content + headers, body = to_structured_http(event) + + # Structured has cloudevent attributes marshalled inside the body. For this + # reason we must load the byte object to create the python dict containing + # the cloudevent attributes + data = json.loads(body) + + # Test cloudevent extension from http fields and data + assert isinstance(data, dict) + assert base64.b64decode(data["data_base64"]) == image_bytes + + # Unmarshall CloudEvent and re-create image + reconstruct_event = from_http( + body, headers, data_unmarshaller=lambda x: io.BytesIO(x) + ) + + # reconstruct_event.data is an io.BytesIO object due to data_unmarshaller + restore_image = Image.open(reconstruct_event.data) + assert restore_image.size == image_expected_shape + + +def test_image_content(): + # Get image and check size + im = Image.open(image_fileobj) + # size of this image + assert im.size == (1880, 363) diff --git a/samples/http-cloudevents/README.md b/samples/http-json-cloudevents/README.md similarity index 100% rename from samples/http-cloudevents/README.md rename to samples/http-json-cloudevents/README.md diff --git a/samples/http-cloudevents/client.py b/samples/http-json-cloudevents/client.py similarity index 100% rename from samples/http-cloudevents/client.py rename to samples/http-json-cloudevents/client.py diff --git a/samples/http-json-cloudevents/requirements.txt b/samples/http-json-cloudevents/requirements.txt new file mode 100644 index 00000000..71bd9694 --- /dev/null +++ b/samples/http-json-cloudevents/requirements.txt @@ -0,0 +1,3 @@ +flask +requests +pytest diff --git a/samples/http-cloudevents/server.py b/samples/http-json-cloudevents/server.py similarity index 100% rename from samples/http-cloudevents/server.py rename to samples/http-json-cloudevents/server.py diff --git a/samples/http-cloudevents/test_verify_sample.py b/samples/http-json-cloudevents/test_verify_sample.py similarity index 100% rename from samples/http-cloudevents/test_verify_sample.py rename to samples/http-json-cloudevents/test_verify_sample.py