From 5fcdfa8e4499c2e0dd2c2bae175240a8902a67c5 Mon Sep 17 00:00:00 2001 From: Matthew Robertson Date: Sat, 8 May 2021 01:05:07 +0000 Subject: [PATCH 1/2] Implement cloudevent to legacy event conversion --- .github/workflows/conformance.yml | 2 +- src/invoker.ts | 9 +- src/middelware/ce_to_legacy_event.ts | 188 ++++++++++++++++++++++++++ src/middelware/index.ts | 1 + src/server.ts | 8 +- test/integration/legacy_event.ts | 46 ++++++- test/middleware/ce_to_legacy_event.ts | 182 +++++++++++++++++++++++++ 7 files changed, 418 insertions(+), 18 deletions(-) create mode 100644 src/middelware/ce_to_legacy_event.ts create mode 100644 src/middelware/index.ts create mode 100644 test/middleware/ce_to_legacy_event.ts diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 6940e7ca..78129542 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -45,7 +45,7 @@ jobs: with: functionType: 'legacyevent' useBuildpacks: false - validateMapping: false + validateMapping: true workingDirectory: 'test/conformance' cmd: "'npm start -- --target=writeLegacyEvent --signature-type=event'" diff --git a/src/invoker.ts b/src/invoker.ts index 2f33680b..303bd7d6 100644 --- a/src/invoker.ts +++ b/src/invoker.ts @@ -195,14 +195,9 @@ export function wrapEventFunction( } } ); - let data = event.data; + const data = event.data; let context = event.context; - if (isBinaryCloudEvent(req)) { - // Support CloudEvents in binary content mode, with data being the whole - // request body and context attributes retrieved from request headers. - data = event; - context = getBinaryCloudEventContext(req); - } else if (context === undefined) { + if (context === undefined) { // Support legacy events and CloudEvents in structured content mode, with // context properties represented as event top-level properties. // Context is everything but data. diff --git a/src/middelware/ce_to_legacy_event.ts b/src/middelware/ce_to_legacy_event.ts new file mode 100644 index 00000000..126c7482 --- /dev/null +++ b/src/middelware/ce_to_legacy_event.ts @@ -0,0 +1,188 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +import {Request, Response, NextFunction} from 'express'; +import {isBinaryCloudEvent, getBinaryCloudEventContext} from '../cloudevents'; + +const CE_TO_BACKGROUND_TYPE = new Map( + Object.entries({ + 'google.cloud.pubsub.topic.v1.messagePublished': + 'google.pubsub.topic.publish', + 'google.cloud.storage.object.v1.finalized': + 'google.storage.object.finalize', + 'google.cloud.storage.object.v1.deleted': 'google.storage.object.delete', + 'google.cloud.storage.object.v1.archived': 'google.storage.object.archive', + 'google.cloud.storage.object.v1.metadataUpdated': + 'google.storage.object.metadataUpdate', + 'google.cloud.firestore.document.v1.written': + 'providers/cloud.firestore/eventTypes/document.write', + 'google.cloud.firestore.document.v1.created': + 'providers/cloud.firestore/eventTypes/document.create', + 'google.cloud.firestore.document.v1.updated': + 'providers/cloud.firestore/eventTypes/document.update', + 'google.cloud.firestore.document.v1.deleted': + 'providers/cloud.firestore/eventTypes/document.delete', + 'google.firebase.auth.user.v1.created': + 'providers/firebase.auth/eventTypes/user.create', + 'google.firebase.auth.user.v1.deleted': + 'providers/firebase.auth/eventTypes/user.delete', + 'google.firebase.analytics.log.v1.written': + 'providers/google.firebase.analytics/eventTypes/event.log', + 'google.firebase.database.document.v1.created': + 'providers/google.firebase.database/eventTypes/ref.create', + 'google.firebase.database.document.v1.written': + 'providers/google.firebase.database/eventTypes/ref.write', + 'google.firebase.database.document.v1.updated': + 'providers/google.firebase.database/eventTypes/ref.update', + 'google.firebase.database.document.v1.deleted': + 'providers/google.firebase.database/eventTypes/ref.delete', + }) +); + +// CloudEvent service names. +const FIREBASE_AUTH_CE_SERVICE = 'firebaseauth.googleapis.com'; +const PUBSUB_CE_SERVICE = 'pubsub.googleapis.com'; +const STORAGE_CE_SERVICE = 'storage.googleapis.com'; + +const PUBSUB_MESSAGE_TYPE = + 'type.googleapis.com/google.pubsub.v1.PubsubMessage'; + +/** + * Regex to split a CE source string into service and name components. + */ +const CE_SOURCE_REGEX = /\/\/([^/]+)\/(.+)/; + +/** + * Costom exception class to represent errors durring event converion. + */ +export class EventConversionError extends Error {} + +/** + * Is the given request a known CloudEvent that can be converted to a legacy event. + * @param request express request object + * @returns true if the request can be converted + */ +const isConvertableCloudEvent = (request: Request): boolean => { + if (isBinaryCloudEvent(request)) { + const ceType = request.header('ce-type'); + return CE_TO_BACKGROUND_TYPE.has(ceType!); + } + return false; +}; + +/** + * Splits a CloudEvent source string into resource and subject components. + * @param source the cloud event source + * @returns the parsed service and name components of the CE source string + */ +export const parseSource = ( + source: string +): {service: string; name: string} => { + const match = source.match(CE_SOURCE_REGEX); + if (!match) { + throw new EventConversionError( + `Failed to convert CloudEvent with invalid source: "${source}"` + ); + } + return { + service: match![1], + name: match![2], + }; +}; + +/** + * Marshal a known GCP CloudEvent request the equivalent context/data legacy event format. + * @param req express request object + * @returns the request body of the equivalent legacy event request + */ +const marshallConvertableCloudEvent = ( + req: Request +): {context: object; data: object} => { + const ceContext = getBinaryCloudEventContext(req); + const {service, name} = parseSource(ceContext.source!); + const subject = ceContext.subject!; + let data = req.body; + + // The default resource is a string made up of the source name and subject. + let resource: string | {[key: string]: string} = `${name}/${subject}`; + + if (service === PUBSUB_CE_SERVICE) { + // PubSub resource format + resource = { + service: service, + name: name, + type: PUBSUB_MESSAGE_TYPE, + }; + // If the data payload has a "message", it needs to be flattened + if ('message' in data) { + data = data['message']; + } + } else if (service === FIREBASE_AUTH_CE_SERVICE) { + // FirebaseAuth resource format + resource = name; + if ('metadata' in data) { + // Some metadata are not consistent between cloudevents and legacy events + const metadata: object = data['metadata']; + data['metadata'] = {}; + // eslint-disable-next-line prefer-const + for (let [k, v] of Object.entries(metadata)) { + k = k === 'createTime' ? 'createdAt' : k; + k = k === 'lastSignInTime' ? 'lastSignedInAt' : k; + data['metadata'][k] = v; + } + } + } else if (service === STORAGE_CE_SERVICE) { + // CloudStorage resource format + resource = { + name: `${name}/${subject}`, + service: service, + type: data['kind'], + }; + } + + return { + context: { + eventId: ceContext.id!, + timestamp: ceContext.time!, + eventType: CE_TO_BACKGROUND_TYPE.get(ceContext.type!), + resource: resource, + }, + data: data, + }; +}; + +/** + * Express middleware to convert cloud event requests to legacy GCF events. This enables + * functions using the "EVENT" signature type to accept requests from a cloud event producer. + * @param req express request object + * @param res express response object + * @param next function used to pass control to the next middle middleware function in the stack + */ +export const ceToLegacyEvent = ( + req: Request, + res: Response, + next: NextFunction +) => { + if (isConvertableCloudEvent(req)) { + // This is a CloudEvent that can be converted a known legacy event. + req.body = marshallConvertableCloudEvent(req); + } else if (isBinaryCloudEvent(req)) { + // Support CloudEvents in binary content mode, with data being the whole + // request body and context attributes retrieved from request headers. + req.body = { + context: getBinaryCloudEventContext(req), + data: req.body, + }; + } + next(); +}; diff --git a/src/middelware/index.ts b/src/middelware/index.ts new file mode 100644 index 00000000..185de5d3 --- /dev/null +++ b/src/middelware/index.ts @@ -0,0 +1 @@ +export {ceToLegacyEvent} from './ce_to_legacy_event'; diff --git a/src/server.ts b/src/server.ts index f48536af..8d9433e1 100644 --- a/src/server.ts +++ b/src/server.ts @@ -20,6 +20,7 @@ import {SignatureType} from './types'; import {setLatestRes} from './invoker'; import {registerFunctionRoutes} from './router'; import {legacyPubSubEventMiddleware} from './pubsub_middleware'; +import {ceToLegacyEvent} from './middelware'; /** * Creates and configures an Express application and returns an HTTP server @@ -98,11 +99,12 @@ export function getServer( // http://expressjs.com/en/advanced/best-practice-security.html#at-a-minimum-disable-x-powered-by-header app.disable('x-powered-by'); - // If a Pub/Sub subscription is configured to invoke a user's function directly, the request body - // needs to be marshalled into the structure that wrapEventFunction expects. This unblocks local - // development with the Pub/Sub emulator if (functionSignatureType === SignatureType.EVENT) { + // If a Pub/Sub subscription is configured to invoke a user's function directly, the request body + // needs to be marshalled into the structure that wrapEventFunction expects. This unblocks local + // development with the Pub/Sub emulator app.use(legacyPubSubEventMiddleware); + app.use(ceToLegacyEvent); } registerFunctionRoutes(app, userFunction, functionSignatureType); diff --git a/test/integration/legacy_event.ts b/test/integration/legacy_event.ts index 48d7da77..2b940f30 100644 --- a/test/integration/legacy_event.ts +++ b/test/integration/legacy_event.ts @@ -45,7 +45,8 @@ describe('Event Function', () => { }, data: {some: 'payload'}, }, - expectedResource: { + expectedData: {some: 'payload'}, + expectedContext: { eventId: 'testEventId', eventType: 'testEventType', resource: 'testResource', @@ -62,7 +63,8 @@ describe('Event Function', () => { resource: 'testResource', data: {some: 'payload'}, }, - expectedResource: { + expectedData: {some: 'payload'}, + expectedContext: { eventId: 'testEventId', eventType: 'testEventType', resource: 'testResource', @@ -85,7 +87,8 @@ describe('Event Function', () => { }, data: {some: 'payload'}, }, - expectedResource: { + expectedData: {some: 'payload'}, + expectedContext: { eventId: 'testEventId', eventType: 'testEventType', resource: { @@ -100,7 +103,8 @@ describe('Event Function', () => { name: 'CloudEvents v1.0 structured content request', headers: {'Content-Type': 'application/cloudevents+json'}, body: TEST_CLOUD_EVENT, - expectedResource: { + expectedData: {some: 'payload'}, + expectedContext: { datacontenttype: 'application/json', id: 'test-1234-1234', source: @@ -124,7 +128,8 @@ describe('Event Function', () => { 'ce-datacontenttype': TEST_CLOUD_EVENT.datacontenttype, }, body: TEST_CLOUD_EVENT.data, - expectedResource: { + expectedData: TEST_CLOUD_EVENT.data, + expectedContext: { datacontenttype: 'application/json', id: 'test-1234-1234', source: @@ -135,6 +140,33 @@ describe('Event Function', () => { type: 'com.google.cloud.storage', }, }, + { + name: 'Firebase Database CloudEvent', + headers: { + 'ce-specversion': '1.0', + 'ce-type': 'google.firebase.database.document.v1.written', + 'ce-source': + '//firebasedatabase.googleapis.com/projects/_/instances/my-project-id', + 'ce-subject': 'refs/gcf-test/xyz', + 'ce-id': 'aaaaaa-1111-bbbb-2222-cccccccccccc', + 'ce-time': '2020-09-29T11:32:00.000Z', + 'ce-datacontenttype': 'application/json', + }, + body: { + data: null, + delta: 10, + }, + expectedData: { + data: null, + delta: 10, + }, + expectedContext: { + resource: 'projects/_/instances/my-project-id/refs/gcf-test/xyz', + timestamp: '2020-09-29T11:32:00.000Z', + eventType: 'providers/google.firebase.database/eventTypes/ref.write', + eventId: 'aaaaaa-1111-bbbb-2222-cccccccccccc', + }, + }, ]; testData.forEach(test => { it(test.name, async () => { @@ -153,8 +185,8 @@ describe('Event Function', () => { .send(test.body) .set(requestHeaders) .expect(204); - assert.deepStrictEqual(receivedData, {some: 'payload'}); - assert.deepStrictEqual(receivedContext, test.expectedResource); + assert.deepStrictEqual(receivedData, test.expectedData); + assert.deepStrictEqual(receivedContext, test.expectedContext); }); }); }); diff --git a/test/middleware/ce_to_legacy_event.ts b/test/middleware/ce_to_legacy_event.ts new file mode 100644 index 00000000..7785a079 --- /dev/null +++ b/test/middleware/ce_to_legacy_event.ts @@ -0,0 +1,182 @@ +import * as assert from 'assert'; +import * as sinon from 'sinon'; +import {Response, Request} from 'express'; + +import { + ceToLegacyEvent, + parseSource, + EventConversionError, +} from '../../src/middelware/ce_to_legacy_event'; + +const ceHeaders = (eventType: string, source: string) => ({ + 'ce-id': 'my-id', + 'ce-type': eventType, + 'ce-source': source, + 'ce-specversion': '1.0', + 'ce-subject': 'my/subject', + 'ce-time': '2020-08-16T13:58:54.471765', +}); + +describe('parseSource', () => { + const testData = [ + { + name: 'firebasedatabase CE source string', + source: + '//firebasedatabase.googleapis.com/projects/_/instances/my-project-id', + expectedService: 'firebasedatabase.googleapis.com', + expectedName: 'projects/_/instances/my-project-id', + }, + { + name: 'firebaseauth CE source string', + source: '//firebaseauth.googleapis.com/projects/my-project-id', + expectedService: 'firebaseauth.googleapis.com', + expectedName: 'projects/my-project-id', + }, + { + name: 'firestore CE source string', + source: + '//firestore.googleapis.com/projects/project-id/databases/(default)', + expectedService: 'firestore.googleapis.com', + expectedName: 'projects/project-id/databases/(default)', + }, + ]; + + testData.forEach(testCase => { + it(testCase.name, () => { + const {service, name} = parseSource(testCase.source); + assert.strictEqual(service, testCase.expectedService); + assert.strictEqual(name, testCase.expectedName); + }); + }); + + it('throws an exception on invalid input', () => { + assert.throws(() => parseSource('invalid'), EventConversionError); + }); +}); + +describe('ceToLegacyEvent', () => { + const testData = [ + { + name: 'Non-CE-Request is not altered', + headers: {foo: 'bar'}, + body: {some: 'value'}, + expectedBody: {some: 'value'}, + }, + { + name: 'Firebase database request', + headers: ceHeaders( + 'google.firebase.database.document.v1.written', + '//firebasedatabase.googleapis.com/projects/_/instances/my-project-id' + ), + body: {some: 'value'}, + expectedBody: { + context: { + eventId: 'my-id', + eventType: 'providers/google.firebase.database/eventTypes/ref.write', + resource: 'projects/_/instances/my-project-id/my/subject', + timestamp: '2020-08-16T13:58:54.471765', + }, + data: { + some: 'value', + }, + }, + }, + { + name: 'PubSub request', + headers: ceHeaders( + 'google.cloud.pubsub.topic.v1.messagePublished', + '//pubsub.googleapis.com/projects/sample-project/topics/gcf-test' + ), + body: { + message: { + data: 'value', + }, + }, + expectedBody: { + data: { + data: 'value', + }, + context: { + eventId: 'my-id', + eventType: 'google.pubsub.topic.publish', + resource: { + name: 'projects/sample-project/topics/gcf-test', + service: 'pubsub.googleapis.com', + type: 'type.googleapis.com/google.pubsub.v1.PubsubMessage', + }, + timestamp: '2020-08-16T13:58:54.471765', + }, + }, + }, + { + name: 'Cloud Storage request', + headers: ceHeaders( + 'google.cloud.storage.object.v1.finalized', + '//storage.googleapis.com/projects/_/buckets/some-bucket' + ), + body: { + some: 'value', + kind: 'storage#object', + }, + expectedBody: { + data: { + some: 'value', + kind: 'storage#object', + }, + context: { + eventId: 'my-id', + eventType: 'google.storage.object.finalize', + resource: { + name: 'projects/_/buckets/some-bucket/my/subject', + service: 'storage.googleapis.com', + type: 'storage#object', + }, + timestamp: '2020-08-16T13:58:54.471765', + }, + }, + }, + { + name: 'Firebase auth request', + headers: ceHeaders( + 'google.firebase.auth.user.v1.created', + '//firebaseauth.googleapis.com/projects/my-project-id' + ), + body: { + metadata: { + createTime: '2020-05-26T10:42:27Z', + lastSignInTime: '2020-10-24T11:00:00Z', + }, + uid: 'my-id', + }, + expectedBody: { + data: { + metadata: { + createdAt: '2020-05-26T10:42:27Z', + lastSignedInAt: '2020-10-24T11:00:00Z', + }, + uid: 'my-id', + }, + context: { + eventId: 'my-id', + eventType: 'providers/firebase.auth/eventTypes/user.create', + resource: 'projects/my-project-id', + timestamp: '2020-08-16T13:58:54.471765', + }, + }, + }, + ]; + + testData.forEach(test => { + it(test.name, () => { + const next = sinon.spy(); + const request = { + body: test.body, + headers: test.headers as object, + header: (key: string) => (test.headers as {[key: string]: string})[key], + }; + ceToLegacyEvent(request as Request, {} as Response, next); + assert.deepStrictEqual(request.body, test.expectedBody); + assert.strictEqual(next.called, true); + }); + }); +}); From ed43c5dd2b4e3a37a2ce40c47ff413b112de0a87 Mon Sep 17 00:00:00 2001 From: Matthew Robertson Date: Mon, 10 May 2021 22:21:10 +0000 Subject: [PATCH 2/2] review feedback: - no re-export - switch statement - shorhand object props - dot member expressions --- src/middelware/ce_to_legacy_event.ts | 72 ++++++++++++++------------- src/middelware/index.ts | 1 - src/server.ts | 4 +- test/middleware/ce_to_legacy_event.ts | 6 +-- 4 files changed, 43 insertions(+), 40 deletions(-) delete mode 100644 src/middelware/index.ts diff --git a/src/middelware/ce_to_legacy_event.ts b/src/middelware/ce_to_legacy_event.ts index 126c7482..d3c9b365 100644 --- a/src/middelware/ce_to_legacy_event.ts +++ b/src/middelware/ce_to_legacy_event.ts @@ -116,38 +116,42 @@ const marshallConvertableCloudEvent = ( // The default resource is a string made up of the source name and subject. let resource: string | {[key: string]: string} = `${name}/${subject}`; - if (service === PUBSUB_CE_SERVICE) { - // PubSub resource format - resource = { - service: service, - name: name, - type: PUBSUB_MESSAGE_TYPE, - }; - // If the data payload has a "message", it needs to be flattened - if ('message' in data) { - data = data['message']; - } - } else if (service === FIREBASE_AUTH_CE_SERVICE) { - // FirebaseAuth resource format - resource = name; - if ('metadata' in data) { - // Some metadata are not consistent between cloudevents and legacy events - const metadata: object = data['metadata']; - data['metadata'] = {}; - // eslint-disable-next-line prefer-const - for (let [k, v] of Object.entries(metadata)) { - k = k === 'createTime' ? 'createdAt' : k; - k = k === 'lastSignInTime' ? 'lastSignedInAt' : k; - data['metadata'][k] = v; + switch (service) { + case PUBSUB_CE_SERVICE: + // PubSub resource format + resource = { + service: service, + name: name, + type: PUBSUB_MESSAGE_TYPE, + }; + // If the data payload has a "message", it needs to be flattened + if ('message' in data) { + data = data.message; } - } - } else if (service === STORAGE_CE_SERVICE) { - // CloudStorage resource format - resource = { - name: `${name}/${subject}`, - service: service, - type: data['kind'], - }; + break; + case FIREBASE_AUTH_CE_SERVICE: + // FirebaseAuth resource format + resource = name; + if ('metadata' in data) { + // Some metadata are not consistent between cloudevents and legacy events + const metadata: object = data.metadata; + data.metadata = {}; + // eslint-disable-next-line prefer-const + for (let [k, v] of Object.entries(metadata)) { + k = k === 'createTime' ? 'createdAt' : k; + k = k === 'lastSignInTime' ? 'lastSignedInAt' : k; + data.metadata[k] = v; + } + } + break; + case STORAGE_CE_SERVICE: + // CloudStorage resource format + resource = { + name: `${name}/${subject}`, + service: service, + type: data.kind, + }; + break; } return { @@ -155,9 +159,9 @@ const marshallConvertableCloudEvent = ( eventId: ceContext.id!, timestamp: ceContext.time!, eventType: CE_TO_BACKGROUND_TYPE.get(ceContext.type!), - resource: resource, + resource, }, - data: data, + data, }; }; @@ -168,7 +172,7 @@ const marshallConvertableCloudEvent = ( * @param res express response object * @param next function used to pass control to the next middle middleware function in the stack */ -export const ceToLegacyEvent = ( +export const ceToLegacyEventMiddleware = ( req: Request, res: Response, next: NextFunction diff --git a/src/middelware/index.ts b/src/middelware/index.ts deleted file mode 100644 index 185de5d3..00000000 --- a/src/middelware/index.ts +++ /dev/null @@ -1 +0,0 @@ -export {ceToLegacyEvent} from './ce_to_legacy_event'; diff --git a/src/server.ts b/src/server.ts index 8d9433e1..0c94dd37 100644 --- a/src/server.ts +++ b/src/server.ts @@ -20,7 +20,7 @@ import {SignatureType} from './types'; import {setLatestRes} from './invoker'; import {registerFunctionRoutes} from './router'; import {legacyPubSubEventMiddleware} from './pubsub_middleware'; -import {ceToLegacyEvent} from './middelware'; +import {ceToLegacyEventMiddleware} from './middelware/ce_to_legacy_event'; /** * Creates and configures an Express application and returns an HTTP server @@ -104,7 +104,7 @@ export function getServer( // needs to be marshalled into the structure that wrapEventFunction expects. This unblocks local // development with the Pub/Sub emulator app.use(legacyPubSubEventMiddleware); - app.use(ceToLegacyEvent); + app.use(ceToLegacyEventMiddleware); } registerFunctionRoutes(app, userFunction, functionSignatureType); diff --git a/test/middleware/ce_to_legacy_event.ts b/test/middleware/ce_to_legacy_event.ts index 7785a079..7eb363bc 100644 --- a/test/middleware/ce_to_legacy_event.ts +++ b/test/middleware/ce_to_legacy_event.ts @@ -3,7 +3,7 @@ import * as sinon from 'sinon'; import {Response, Request} from 'express'; import { - ceToLegacyEvent, + ceToLegacyEventMiddleware, parseSource, EventConversionError, } from '../../src/middelware/ce_to_legacy_event'; @@ -54,7 +54,7 @@ describe('parseSource', () => { }); }); -describe('ceToLegacyEvent', () => { +describe('ceToLegacyEventMiddleware', () => { const testData = [ { name: 'Non-CE-Request is not altered', @@ -174,7 +174,7 @@ describe('ceToLegacyEvent', () => { headers: test.headers as object, header: (key: string) => (test.headers as {[key: string]: string})[key], }; - ceToLegacyEvent(request as Request, {} as Response, next); + ceToLegacyEventMiddleware(request as Request, {} as Response, next); assert.deepStrictEqual(request.body, test.expectedBody); assert.strictEqual(next.called, true); });