From a626d58f2c503ccc482a211f0bae3a2fb0ede240 Mon Sep 17 00:00:00 2001 From: liuliquan Date: Fri, 24 Nov 2023 23:41:34 +0800 Subject: [PATCH 1/6] CORE-65 Post Resource events to Harmony after CRUD --- src/common/helper.js | 32 +++++++++++++++++++++++++++++ src/services/ResourceRoleService.js | 2 ++ src/services/ResourceService.js | 9 +++++--- 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/src/common/helper.js b/src/common/helper.js index 17905bd..442e2cb 100644 --- a/src/common/helper.js +++ b/src/common/helper.js @@ -520,7 +520,39 @@ async function advanceChallengePhase (challengeId, phase, operation, numAttempts } } +const harmonyClient = new AWS.Lambda({ apiVersion: 'latest' }) +/** + * Send event to Harmony. + * @param {String} eventType The event type + * @param {String} payloadType The payload type + * @param {Object} payload The event payload + * @returns {Promise} + */ +async function sendHarmonyEvent (eventType, payloadType, payload) { + const event = { + publisher: config.KAFKA_MESSAGE_ORIGINATOR, + timestamp: new Date().getTime(), + eventType, + payloadType, + payload + } + return new Promise((resolve, reject) => { + harmonyClient.invoke({ + FunctionName: config.HARMONY_LAMBDA_FUNCTION, + InvocationType: 'Event', + Payload: JSON.stringify(event) + }, (err, data) => { + if (err) { + reject(err) + } else { + resolve(data) + } + }) + }) +} + module.exports = { + sendHarmonyEvent, wrapExpress, autoWrapExpress, getMemberInfoById, diff --git a/src/services/ResourceRoleService.js b/src/services/ResourceRoleService.js index 1024c3a..d130b06 100644 --- a/src/services/ResourceRoleService.js +++ b/src/services/ResourceRoleService.js @@ -60,6 +60,7 @@ async function createResourceRole (resourceRole) { const entity = await helper.create('ResourceRole', _.assign({ id: uuid(), nameLower }, resourceRole)) const ret = _.pick(entity, payloadFields) await helper.postEvent(config.RESOURCE_ROLE_CREATE_TOPIC, ret) + await helper.sendHarmonyEvent('CREATE', 'ResourceRole', {...ret, nameLower}); return ret } catch (err) { if (!helper.isCustomError(err)) { @@ -96,6 +97,7 @@ async function updateResourceRole (resourceRoleId, data) { const entity = await helper.update(resourceRole, data) const ret = _.pick(entity, payloadFields) await helper.postEvent(config.RESOURCE_ROLE_UPDATE_TOPIC, ret) + await helper.sendHarmonyEvent('UPDATE', 'ResourceRole', {...ret, nameLower}); return ret } catch (err) { if (!helper.isCustomError(err)) { diff --git a/src/services/ResourceService.js b/src/services/ResourceService.js index 929061c..9322080 100644 --- a/src/services/ResourceService.js +++ b/src/services/ResourceService.js @@ -373,18 +373,20 @@ async function createResource (currentUser, resource) { createdBy: currentUser.handle || currentUser.sub }, resource)) + const eventPayload = _.pick(ret, payloadFields) // Create resources in ES const esClient = await helper.getESClient() await esClient.create({ index: config.ES.ES_INDEX, type: config.ES.ES_TYPE, id: ret.id, - body: _.pick(ret, payloadFields), + body: eventPayload, refresh: 'true' // refresh ES so that it is visible for read operations instantly }) + await helper.sendHarmonyEvent('UPDATE', 'Resource', eventPayload) - logger.debug(`Created resource: ${JSON.stringify(_.pick(ret, payloadFields))}`) - await helper.postEvent(config.RESOURCE_CREATE_TOPIC, _.pick(ret, payloadFields)) + logger.debug(`Created resource: ${JSON.stringify(eventPayload)}`) + await helper.postEvent(config.RESOURCE_CREATE_TOPIC, eventPayload) if (!_.get(challenge, 'task.isTask', false) && resource.roleId === config.SUBMITTER_RESOURCE_ROLE_ID) { const forumUrl = _.get(challenge, 'discussions[0].url') let templateId = config.REGISTRATION_EMAIL.SENDGRID_TEMPLATE_ID @@ -468,6 +470,7 @@ async function deleteResource (currentUser, resource) { logger.debug(`Deleted resource, posting to Bus API: ${JSON.stringify(_.pick(ret, payloadFields))}`) await helper.postEvent(config.RESOURCE_DELETE_TOPIC, _.pick(ret, payloadFields)) + await helper.sendHarmonyEvent('DELETE', 'Resource', { id: ret.id, roleId: ret.roleId, challengeId }) return ret } catch (err) { logger.error(`Delete Resource Error ${JSON.stringify(err)}`) From 41ef499c80ff9ee2b1d347c2328252fb5e186bf0 Mon Sep 17 00:00:00 2001 From: liuliquan Date: Sat, 25 Nov 2023 02:08:20 +0800 Subject: [PATCH 2/6] Use promise when invoke lambda --- config/default.js | 2 ++ src/common/helper.js | 19 ++++++------------- src/services/ResourceRoleService.js | 4 ++-- 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/config/default.js b/config/default.js index ae5a090..6ca2fee 100644 --- a/config/default.js +++ b/config/default.js @@ -79,5 +79,7 @@ module.exports = { SUPPORT_EMAIL: process.env.SUPPORT_EMAIL || 'support@topcoder.com' }, + HARMONY_LAMBDA_FUNCTION: process.env.HARMONY_LAMBDA_FUNCTION || 'arn:aws:lambda:us-east-1:811668436784:function:harmony-api-dev-processMessage', + AUTOMATED_TESTING_NAME_PREFIX: process.env.AUTOMATED_TESTING_NAME_PREFIX || 'POSTMANE2E-' } diff --git a/src/common/helper.js b/src/common/helper.js index 442e2cb..fb153bf 100644 --- a/src/common/helper.js +++ b/src/common/helper.js @@ -536,19 +536,12 @@ async function sendHarmonyEvent (eventType, payloadType, payload) { payloadType, payload } - return new Promise((resolve, reject) => { - harmonyClient.invoke({ - FunctionName: config.HARMONY_LAMBDA_FUNCTION, - InvocationType: 'Event', - Payload: JSON.stringify(event) - }, (err, data) => { - if (err) { - reject(err) - } else { - resolve(data) - } - }) - }) + + await harmonyClient.invoke({ + FunctionName: config.HARMONY_LAMBDA_FUNCTION, + InvocationType: 'Event', + Payload: JSON.stringify(event) + }).promise() } module.exports = { diff --git a/src/services/ResourceRoleService.js b/src/services/ResourceRoleService.js index d130b06..4b30999 100644 --- a/src/services/ResourceRoleService.js +++ b/src/services/ResourceRoleService.js @@ -60,7 +60,7 @@ async function createResourceRole (resourceRole) { const entity = await helper.create('ResourceRole', _.assign({ id: uuid(), nameLower }, resourceRole)) const ret = _.pick(entity, payloadFields) await helper.postEvent(config.RESOURCE_ROLE_CREATE_TOPIC, ret) - await helper.sendHarmonyEvent('CREATE', 'ResourceRole', {...ret, nameLower}); + await helper.sendHarmonyEvent('CREATE', 'ResourceRole', { ...ret, nameLower }) return ret } catch (err) { if (!helper.isCustomError(err)) { @@ -97,7 +97,7 @@ async function updateResourceRole (resourceRoleId, data) { const entity = await helper.update(resourceRole, data) const ret = _.pick(entity, payloadFields) await helper.postEvent(config.RESOURCE_ROLE_UPDATE_TOPIC, ret) - await helper.sendHarmonyEvent('UPDATE', 'ResourceRole', {...ret, nameLower}); + await helper.sendHarmonyEvent('UPDATE', 'ResourceRole', { ...ret, nameLower: data.nameLower }) return ret } catch (err) { if (!helper.isCustomError(err)) { From cb4340728871e1c4dae0276660b07b58de441c40 Mon Sep 17 00:00:00 2001 From: liuliquan Date: Sat, 25 Nov 2023 21:30:40 +0800 Subject: [PATCH 3/6] Sync invoke Harmony and handle error --- src/common/helper.js | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/common/helper.js b/src/common/helper.js index fb153bf..a370824 100644 --- a/src/common/helper.js +++ b/src/common/helper.js @@ -520,7 +520,7 @@ async function advanceChallengePhase (challengeId, phase, operation, numAttempts } } -const harmonyClient = new AWS.Lambda({ apiVersion: 'latest' }) +const harmonyClient = new AWS.Lambda({ apiVersion: 'latest', maxRetries: 2 }) /** * Send event to Harmony. * @param {String} eventType The event type @@ -537,11 +537,21 @@ async function sendHarmonyEvent (eventType, payloadType, payload) { payload } - await harmonyClient.invoke({ + const result = await harmonyClient.invoke({ FunctionName: config.HARMONY_LAMBDA_FUNCTION, - InvocationType: 'Event', - Payload: JSON.stringify(event) + InvocationType: 'RequestResponse', + Payload: JSON.stringify(event), + LogType: 'None' }).promise() + + if (result.FunctionError) { + console.error( + 'Failed to send Harmony event', + result.FunctionError, + _.toString(result.Payload) + ) + throw new Error(result.FunctionError) + } } module.exports = { From ddd02b26a2fa12505c6cd922b522a9dbc4f4b5ff Mon Sep 17 00:00:00 2001 From: liuliquan Date: Mon, 27 Nov 2023 06:42:17 +0800 Subject: [PATCH 4/6] ci: deploy CORE-65 --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 95a3f99..245808c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -108,6 +108,7 @@ workflows: only: - develop - CORE-103 + - CORE-65 # Production builds are exectuted only on tagged commits to the testing # master branch. From 6bb893b82142417a559810a7fd8883b52f6f9ad0 Mon Sep 17 00:00:00 2001 From: liuliquan Date: Tue, 28 Nov 2023 18:36:02 +0800 Subject: [PATCH 5/6] fix: event type should be CREATE --- src/services/ResourceService.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/ResourceService.js b/src/services/ResourceService.js index be63ed3..baf23dc 100644 --- a/src/services/ResourceService.js +++ b/src/services/ResourceService.js @@ -389,7 +389,7 @@ async function createResource (currentUser, resource) { body: eventPayload, refresh: 'true' // refresh ES so that it is visible for read operations instantly }) - await helper.sendHarmonyEvent('UPDATE', 'Resource', eventPayload) + await helper.sendHarmonyEvent('CREATE', 'Resource', eventPayload) logger.debug(`Created resource: ${JSON.stringify(eventPayload)}`) await helper.postEvent(config.RESOURCE_CREATE_TOPIC, eventPayload) From 20885ec992c76f0e0c3a8e94ab7f98993d20c589 Mon Sep 17 00:00:00 2001 From: liuliquan Date: Mon, 4 Dec 2023 07:05:13 +0800 Subject: [PATCH 6/6] feat: send billingAccountId --- src/common/helper.js | 6 ++++-- src/services/ResourceService.js | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/common/helper.js b/src/common/helper.js index a370824..dcef0f6 100644 --- a/src/common/helper.js +++ b/src/common/helper.js @@ -526,15 +526,17 @@ const harmonyClient = new AWS.Lambda({ apiVersion: 'latest', maxRetries: 2 }) * @param {String} eventType The event type * @param {String} payloadType The payload type * @param {Object} payload The event payload + * @param {Number} billingAccountId The billing account id * @returns {Promise} */ -async function sendHarmonyEvent (eventType, payloadType, payload) { +async function sendHarmonyEvent (eventType, payloadType, payload, billingAccountId) { const event = { publisher: config.KAFKA_MESSAGE_ORIGINATOR, timestamp: new Date().getTime(), eventType, payloadType, - payload + payload, + billingAccountId } const result = await harmonyClient.invoke({ diff --git a/src/services/ResourceService.js b/src/services/ResourceService.js index baf23dc..706e89c 100644 --- a/src/services/ResourceService.js +++ b/src/services/ResourceService.js @@ -389,7 +389,7 @@ async function createResource (currentUser, resource) { body: eventPayload, refresh: 'true' // refresh ES so that it is visible for read operations instantly }) - await helper.sendHarmonyEvent('CREATE', 'Resource', eventPayload) + await helper.sendHarmonyEvent('CREATE', 'Resource', eventPayload, _.get(challenge, 'billing.billingAccountId')) logger.debug(`Created resource: ${JSON.stringify(eventPayload)}`) await helper.postEvent(config.RESOURCE_CREATE_TOPIC, eventPayload) @@ -453,7 +453,7 @@ async function deleteResource (currentUser, resource) { try { const challengeId = resource.challengeId - const { allResources, memberId, handle } = await init(currentUser, challengeId, resource) + const { allResources, memberId, handle, challenge } = await init(currentUser, challengeId, resource) const ret = _.reduce(allResources, (result, r) => _.toString(r.memberId) === _.toString(memberId) && r.roleId === resource.roleId ? r : result, @@ -476,7 +476,7 @@ async function deleteResource (currentUser, resource) { logger.debug(`Deleted resource, posting to Bus API: ${JSON.stringify(_.pick(ret, payloadFields))}`) await helper.postEvent(config.RESOURCE_DELETE_TOPIC, _.pick(ret, payloadFields)) - await helper.sendHarmonyEvent('DELETE', 'Resource', { id: ret.id, roleId: ret.roleId, challengeId }) + await helper.sendHarmonyEvent('DELETE', 'Resource', { id: ret.id, roleId: ret.roleId, challengeId }, _.get(challenge, 'billing.billingAccountId')) return ret } catch (err) { logger.error(`Delete Resource Error ${JSON.stringify(err)}`)