From e86fec09b15de81d463d11ea6eab8ce92525fefe Mon Sep 17 00:00:00 2001 From: liuliquan Date: Fri, 24 Nov 2023 23:12:45 +0800 Subject: [PATCH 1/5] CORE-66 Post Submission events to Harmony after CRUD --- src/common/helper.js | 33 +++++++++++++ src/services/ReviewService.js | 68 ++++++++++++++------------ src/services/ReviewSummationService.js | 62 +++++++++++++---------- src/services/ReviewTypeService.js | 15 ++++-- src/services/SubmissionService.js | 19 ++++++- 5 files changed, 135 insertions(+), 62 deletions(-) diff --git a/src/common/helper.js b/src/common/helper.js index 2a60fe38..8d67a4b8 100755 --- a/src/common/helper.js +++ b/src/common/helper.js @@ -14,6 +14,7 @@ const errors = require('common-errors') const { validate: uuidValidate } = require('uuid') const NodeCache = require('node-cache') const { axiosInstance } = require('./axiosInstance') +const { originator } = require('../../constants').busApiMeta AWS.config.region = config.get('aws.AWS_REGION') const s3 = new AWS.S3() @@ -873,7 +874,39 @@ function flushInternalCache () { internalCache.flushAll() } +const harmonyClient = new AWS.Lambda({ apiVersion: 'latest' }) +/** + * Send event to Harmony. + * @param {String} eventType The event type + * @param {String} payloadType The payload type + * @param {Object} payload The event payload + * @returns {Promise} + */ +async function sendHarmonyEvent (eventType, payloadType, payload) { + const event = { + publisher: originator, + timestamp: new Date().getTime(), + eventType, + payloadType, + payload + } + return new Promise((resolve, reject) => { + harmonyClient.invoke({ + FunctionName: config.HARMONY_LAMBDA_FUNCTION, + InvocationType: 'Event', + Payload: JSON.stringify(event) + }, (err, data) => { + if (err) { + reject(err) + } else { + resolve(data) + } + }) + }) +} + module.exports = { + sendHarmonyEvent, wrapExpress, autoWrapExpress, getEsClient, diff --git a/src/services/ReviewService.js b/src/services/ReviewService.js index f3a27b00..2beafeaa 100644 --- a/src/services/ReviewService.js +++ b/src/services/ReviewService.js @@ -183,6 +183,8 @@ async function createReview (authUser, entity) { await dbhelper.insertRecord(record) + await helper.sendHarmonyEvent('CREATE', table, item) + // Push Review created event to Bus API // Request body for Posting to Bus API const reqBody = { @@ -283,6 +285,20 @@ async function _updateReview (authUser, reviewId, entity) { } } + const item = { + id: reviewId, + submissionId: entity.submissionId || exist.submissionId, + scoreCardId, + v5ScoreCardId, + score: entity.score || exist.score, + typeId: entity.typeId || exist.typeId, + reviewerId: entity.reviewerId || exist.reviewerId, + status: entity.status || exist.status || 'completed', + reviewedDate: entity.reviewedDate || exist.reviewedDate || exist.created, + updated: currDate, + updatedBy: authUser.handle || authUser.sub + } + // Record used for updating in Database const record = { TableName: table, @@ -294,15 +310,15 @@ async function _updateReview (authUser, reviewId, entity) { updated = :ua, updatedBy = :ub, reviewedDate = :rd ${v5ScoreCardId ? ', v5ScoreCardId = :v5s' : ''}`, ExpressionAttributeValues: { - ':s': entity.score || exist.score, - ':sc': scoreCardId, - ':su': entity.submissionId || exist.submissionId, - ':t': entity.typeId || exist.typeId, - ':r': entity.reviewerId || exist.reviewerId, - ':st': entity.status || exist.status || 'completed', - ':ua': currDate, - ':ub': authUser.handle || authUser.sub, - ':rd': entity.reviewedDate || exist.reviewedDate || exist.created, + ':s': item.score, + ':sc': item.scoreCardId, + ':su': item.submissionId, + ':t': item.typeId, + ':r': item.reviewerId, + ':st': item.status, + ':ua': item.updated, + ':ub': item.updatedBy, + ':rd': item.reviewedDate, ...(v5ScoreCardId ? { ':v5s': v5ScoreCardId } : {}) }, ExpressionAttributeNames: { @@ -312,17 +328,16 @@ async function _updateReview (authUser, reviewId, entity) { // If metadata exists, add it to the update expression if (entity.metadata || exist.metadata) { + item.metadata = _.merge({}, exist.metadata, entity.metadata) record.UpdateExpression = record.UpdateExpression + ', metadata = :ma' - record.ExpressionAttributeValues[':ma'] = _.merge( - {}, - exist.metadata, - entity.metadata - ) + record.ExpressionAttributeValues[':ma'] = item.metadata } await dbhelper.updateRecord(record) + await helper.sendHarmonyEvent('UPDATE', table, item) + // Push Review updated event to Bus API // Request body for Posting to Bus API const reqBody = { @@ -332,17 +347,9 @@ async function _updateReview (authUser, reviewId, entity) { 'mime-type': mimeType, payload: _.extend( { - resource: helper.camelize(table), - id: reviewId, - updated: currDate, - updatedBy: authUser.handle || authUser.sub, - reviewedDate: entity.reviewedDate || exist.reviewedDate || exist.created + resource: helper.camelize(table) }, - entity, - { - scoreCardId, - v5ScoreCardId - } + item ) } @@ -351,13 +358,7 @@ async function _updateReview (authUser, reviewId, entity) { // Updating records in DynamoDB doesn't return any response // Hence returning the response which will be in compliance with Swagger - return _.extend(exist, entity, { - updated: currDate, - updatedBy: authUser.handle || authUser.sub, - reviewedDate: entity.reviewedDate || exist.reviewedDate || exist.created, - scoreCardId, - v5ScoreCardId - }) + return _.extend(exist, item) } /** @@ -454,6 +455,11 @@ async function deleteReview (reviewId) { await dbhelper.deleteRecord(filter) + await helper.sendHarmonyEvent('DELETE', table, { + id: reviewId, + submissionId: exist.submissionId + }) + // Push Review deleted event to Bus API // Request body for Posting to Bus API const reqBody = { diff --git a/src/services/ReviewSummationService.js b/src/services/ReviewSummationService.js index 24c86437..2b0e06d2 100644 --- a/src/services/ReviewSummationService.js +++ b/src/services/ReviewSummationService.js @@ -117,6 +117,8 @@ async function createReviewSummation (authUser, entity) { await dbhelper.insertRecord(record) + await helper.sendHarmonyEvent('CREATE', table, item) + // Push Review Summation created event to Bus API // Request body for Posting to Bus API const reqBody = { @@ -175,6 +177,17 @@ async function _updateReviewSummation (authUser, reviewSummationId, entity) { const currDate = (new Date()).toISOString() + const item = { + id: reviewSummationId, + submissionId: entity.submissionId || exist.submissionId, + scoreCardId: entity.scoreCardId || exist.scoreCardId, + aggregateScore: entity.aggregateScore || exist.aggregateScore, + reviewedDate: entity.reviewedDate || exist.reviewedDate || exist.created, + isPassing, + updated: currDate, + updatedBy: authUser.handle || authUser.sub + } + // Record used for updating in Database const record = { TableName: table, @@ -184,20 +197,21 @@ async function _updateReviewSummation (authUser, reviewSummationId, entity) { UpdateExpression: `set aggregateScore = :s, scoreCardId = :sc, submissionId = :su, isPassing = :ip, updated = :ua, updatedBy = :ub, reviewedDate = :rd`, ExpressionAttributeValues: { - ':s': entity.aggregateScore || exist.aggregateScore, - ':sc': entity.scoreCardId || exist.scoreCardId, - ':su': entity.submissionId || exist.submissionId, - ':ip': isPassing, - ':ua': currDate, - ':ub': authUser.handle || authUser.sub, - ':rd': entity.reviewedDate || exist.reviewedDate || exist.created + ':s': item.aggregateScore, + ':sc': item.scoreCardId, + ':su': item.submissionId, + ':ip': item.isPassing, + ':ua': item.updated, + ':ub': item.updatedBy, + ':rd': item.reviewedDate } } // If metadata exists, add it to the update expression if (entity.metadata || exist.metadata) { + item.metadata = _.merge({}, exist.metadata, entity.metadata) record.UpdateExpression = record.UpdateExpression + ', metadata = :ma' - record.ExpressionAttributeValues[':ma'] = _.merge({}, exist.metadata, entity.metadata) + record.ExpressionAttributeValues[':ma'] = item.metadata } // If legacy submission ID exists, add it to the update expression @@ -209,13 +223,15 @@ async function _updateReviewSummation (authUser, reviewSummationId, entity) { } else { isFinal = entity.isFinal } - + item.isFinal = isFinal record.UpdateExpression = record.UpdateExpression + ', isFinal = :ls' record.ExpressionAttributeValues[':ls'] = isFinal } await dbhelper.updateRecord(record) + await helper.sendHarmonyEvent('UPDATE', table, item) + // Push Review Summation updated event to Bus API // Request body for Posting to Bus API const reqBody = { @@ -223,13 +239,12 @@ async function _updateReviewSummation (authUser, reviewSummationId, entity) { originator, timestamp: currDate, // time when submission was updated 'mime-type': mimeType, - payload: _.extend({ - resource: helper.camelize(table), - id: reviewSummationId, - updated: currDate, - updatedBy: authUser.handle || authUser.sub, - reviewedDate: entity.reviewedDate || exist.reviewedDate || exist.created - }, entity) + payload: _.extend( + { + resource: helper.camelize(table) + }, + item + ) } // Post to Bus API using Client @@ -237,15 +252,7 @@ async function _updateReviewSummation (authUser, reviewSummationId, entity) { // Updating records in DynamoDB doesn't return any response // Hence returning the response which will be in compliance with Swagger - return _.extend( - exist, - entity, - { - updated: currDate, - updatedBy: authUser.handle || authUser.sub, - reviewedDate: entity.reviewedDate || exist.reviewedDate || exist.created - } - ) + return _.extend(exist, item) } /** @@ -319,6 +326,11 @@ async function deleteReviewSummation (reviewSummationId) { await dbhelper.deleteRecord(filter) + await helper.sendHarmonyEvent('DELETE', table, { + id: reviewSummationId, + submissionId: exist.submissionId + }) + // Push Review Summation deleted event to Bus API // Request body for Posting to Bus API const reqBody = { diff --git a/src/services/ReviewTypeService.js b/src/services/ReviewTypeService.js index 736b923c..bf3e63cf 100755 --- a/src/services/ReviewTypeService.js +++ b/src/services/ReviewTypeService.js @@ -89,6 +89,8 @@ async function createReviewType (entity) { await dbhelper.insertRecord(record) + await helper.sendHarmonyEvent('CREATE', table, item) + // Push Review Type created event to Bus API // Request body for Posting to Bus API const reqBody = { @@ -152,6 +154,12 @@ async function _updateReviewType (reviewTypeId, entity) { } await dbhelper.updateRecord(record) + const eventPayload = _.extend({ + resource: helper.camelize(table), + id: reviewTypeId + }, entity) + await helper.sendHarmonyEvent('UPDATE', table, _.omit(eventPayload, 'resource')) + // Push Review Type updated event to Bus API // Request body for Posting to Bus API const reqBody = { @@ -159,10 +167,7 @@ async function _updateReviewType (reviewTypeId, entity) { originator, timestamp: (new Date()).toISOString(), // time when submission was updated 'mime-type': mimeType, - payload: _.extend({ - resource: helper.camelize(table), - id: reviewTypeId - }, entity) + payload: eventPayload } // Post to Bus API using Client @@ -230,6 +235,8 @@ async function deleteReviewType (reviewTypeId) { await dbhelper.deleteRecord(filter) + await helper.sendHarmonyEvent('DELETE', table, { id: reviewTypeId }) + // Push Review Type deleted event to Bus API // Request body for Posting to Bus API const reqBody = { diff --git a/src/services/SubmissionService.js b/src/services/SubmissionService.js index 30a0516b..b4691d4d 100755 --- a/src/services/SubmissionService.js +++ b/src/services/SubmissionService.js @@ -403,6 +403,12 @@ async function createSubmission (authUser, files, entity) { logger.info(JSON.stringify(record)) await dbhelper.insertRecord(record) + await helper.sendHarmonyEvent('CREATE', table, { + ...item, + isFileSubmission: files && files.submission, + ...((files && files.submission) ? { filename: files.submission.name } : {}) + }) + // After save to db, adjust challengeId to busApi and response helper.adjustSubmissionChallengeId(item) @@ -540,7 +546,11 @@ async function _updateSubmission (authUser, submissionId, entity) { logger.info('Prepared submission item to update in Dynamodb. Updating...') await dbhelper.updateRecord(record) - const updatedSub = await _getSubmission(submissionId) + const updatedSub = await _getSubmission(submissionId, false) + + await helper.sendHarmonyEvent('UPDATE', table, _.pick(updatedSub, + ['id', 'challengeId', 'legacyChallengeId', 'legacySubmissionId', 'legacyUploadId', + 'memberId', 'submissionPhaseId', 'submittedDate', 'type', 'url'])) helper.adjustSubmissionChallengeId(updatedSub) // Push Submission updated event to Bus API @@ -651,7 +661,7 @@ patchSubmission.schema = joi.object({ * @return {Promise} */ async function deleteSubmission (authUser, submissionId) { - const exist = await _getSubmission(submissionId) + const exist = await _getSubmission(submissionId, false) if (!exist) { throw new errors.HttpStatusError(404, `Submission with ID = ${submissionId} is not found`) } @@ -670,6 +680,11 @@ async function deleteSubmission (authUser, submissionId) { await dbhelper.deleteRecord(filter) + await helper.sendHarmonyEvent('DELETE', table, { + id: submissionId, + challengeId: exist.challengeId + }) + // Push Submission deleted event to Bus API // Request body for Posting to Bus API const reqBody = { From 2c2e73c4916b761d0677852db0a24aec4ebf4054 Mon Sep 17 00:00:00 2001 From: liuliquan Date: Sat, 25 Nov 2023 02:07:59 +0800 Subject: [PATCH 2/5] Use promise when invoke lambda --- config/default.js | 2 ++ src/common/helper.js | 19 ++++++------------- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/config/default.js b/config/default.js index d0dfd257..43a32590 100755 --- a/config/default.js +++ b/config/default.js @@ -55,5 +55,7 @@ module.exports = { 'd6d31f34-8ee5-4589-ae65-45652fcc01a6': 30000720 }, + HARMONY_LAMBDA_FUNCTION: process.env.HARMONY_LAMBDA_FUNCTION || 'arn:aws:lambda:us-east-1:811668436784:function:harmony-api-dev-processMessage', + INTERNAL_CACHE_TTL: process.env.INTERNAL_CACHE_TTL || 1800 } diff --git a/src/common/helper.js b/src/common/helper.js index 8d67a4b8..e66e1142 100755 --- a/src/common/helper.js +++ b/src/common/helper.js @@ -890,19 +890,12 @@ async function sendHarmonyEvent (eventType, payloadType, payload) { payloadType, payload } - return new Promise((resolve, reject) => { - harmonyClient.invoke({ - FunctionName: config.HARMONY_LAMBDA_FUNCTION, - InvocationType: 'Event', - Payload: JSON.stringify(event) - }, (err, data) => { - if (err) { - reject(err) - } else { - resolve(data) - } - }) - }) + + await harmonyClient.invoke({ + FunctionName: config.HARMONY_LAMBDA_FUNCTION, + InvocationType: 'Event', + Payload: JSON.stringify(event) + }).promise() } module.exports = { From f1ac2e60f324d32cc15e8647b67542b0cbe13e93 Mon Sep 17 00:00:00 2001 From: liuliquan Date: Sat, 25 Nov 2023 21:58:16 +0800 Subject: [PATCH 3/5] Sync invoke Harmony and handle error --- src/common/helper.js | 18 ++++++++++++++---- src/services/SubmissionService.js | 4 +--- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/common/helper.js b/src/common/helper.js index e66e1142..97cfa674 100755 --- a/src/common/helper.js +++ b/src/common/helper.js @@ -874,7 +874,7 @@ function flushInternalCache () { internalCache.flushAll() } -const harmonyClient = new AWS.Lambda({ apiVersion: 'latest' }) +const harmonyClient = new AWS.Lambda({ apiVersion: 'latest', maxRetries: 2 }) /** * Send event to Harmony. * @param {String} eventType The event type @@ -891,11 +891,21 @@ async function sendHarmonyEvent (eventType, payloadType, payload) { payload } - await harmonyClient.invoke({ + const result = await harmonyClient.invoke({ FunctionName: config.HARMONY_LAMBDA_FUNCTION, - InvocationType: 'Event', - Payload: JSON.stringify(event) + InvocationType: 'RequestResponse', + Payload: JSON.stringify(event), + LogType: 'None' }).promise() + + if (result.FunctionError) { + console.error( + 'Failed to send Harmony event', + result.FunctionError, + _.toString(result.Payload) + ) + throw new Error(result.FunctionError) + } } module.exports = { diff --git a/src/services/SubmissionService.js b/src/services/SubmissionService.js index b4691d4d..77a6ddea 100755 --- a/src/services/SubmissionService.js +++ b/src/services/SubmissionService.js @@ -548,9 +548,7 @@ async function _updateSubmission (authUser, submissionId, entity) { await dbhelper.updateRecord(record) const updatedSub = await _getSubmission(submissionId, false) - await helper.sendHarmonyEvent('UPDATE', table, _.pick(updatedSub, - ['id', 'challengeId', 'legacyChallengeId', 'legacySubmissionId', 'legacyUploadId', - 'memberId', 'submissionPhaseId', 'submittedDate', 'type', 'url'])) + await helper.sendHarmonyEvent('UPDATE', table, updatedSub) helper.adjustSubmissionChallengeId(updatedSub) // Push Submission updated event to Bus API From 62a87930feb0a751731f3df67753de9c27d973d6 Mon Sep 17 00:00:00 2001 From: liuliquan Date: Mon, 27 Nov 2023 06:46:55 +0800 Subject: [PATCH 4/5] ci: deploy CORE-66 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d229451b..df929505 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -69,7 +69,7 @@ workflows: context : org-global filters: branches: - only: ['develop', 'PLAT-3383'] + only: ['develop', 'CORE-66'] - "build-prod": context : org-global filters: From 0627580e8db10f0e52460ba65c04969e3a9c6468 Mon Sep 17 00:00:00 2001 From: liuliquan Date: Mon, 4 Dec 2023 07:06:40 +0800 Subject: [PATCH 5/5] feat: send billingAccountId --- src/common/helper.js | 6 ++++-- src/services/SubmissionService.js | 10 ++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/common/helper.js b/src/common/helper.js index 97cfa674..25af9f90 100755 --- a/src/common/helper.js +++ b/src/common/helper.js @@ -880,15 +880,17 @@ const harmonyClient = new AWS.Lambda({ apiVersion: 'latest', maxRetries: 2 }) * @param {String} eventType The event type * @param {String} payloadType The payload type * @param {Object} payload The event payload + * @param {Number} billingAccountId The billing account id * @returns {Promise} */ -async function sendHarmonyEvent (eventType, payloadType, payload) { +async function sendHarmonyEvent (eventType, payloadType, payload, billingAccountId) { const event = { publisher: originator, timestamp: new Date().getTime(), eventType, payloadType, - payload + payload, + billingAccountId } const result = await harmonyClient.invoke({ diff --git a/src/services/SubmissionService.js b/src/services/SubmissionService.js index 77a6ddea..3dd73742 100755 --- a/src/services/SubmissionService.js +++ b/src/services/SubmissionService.js @@ -407,7 +407,7 @@ async function createSubmission (authUser, files, entity) { ...item, isFileSubmission: files && files.submission, ...((files && files.submission) ? { filename: files.submission.name } : {}) - }) + }, _.get(challenge, 'billing.billingAccountId')) // After save to db, adjust challengeId to busApi and response helper.adjustSubmissionChallengeId(item) @@ -479,8 +479,9 @@ async function _updateSubmission (authUser, submissionId, entity) { let legacyChallengeId = exist.legacyChallengeId let hasIterativeReview = false + let challenge if (entity.challengeId || challengeId) { - const challenge = await helper.getChallenge(entity.challengeId || challengeId) + challenge = await helper.getChallenge(entity.challengeId || challengeId) if (!challenge) { throw new errors.HttpStatusError(404, `Challenge with ID = ${entity.challengeId || challengeId} is not found`) } @@ -548,7 +549,7 @@ async function _updateSubmission (authUser, submissionId, entity) { await dbhelper.updateRecord(record) const updatedSub = await _getSubmission(submissionId, false) - await helper.sendHarmonyEvent('UPDATE', table, updatedSub) + await helper.sendHarmonyEvent('UPDATE', table, updatedSub, _.get(challenge, 'billing.billingAccountId')) helper.adjustSubmissionChallengeId(updatedSub) // Push Submission updated event to Bus API @@ -678,10 +679,11 @@ async function deleteSubmission (authUser, submissionId) { await dbhelper.deleteRecord(filter) + const challenge = await helper.getChallenge(exist.challengeId) await helper.sendHarmonyEvent('DELETE', table, { id: submissionId, challengeId: exist.challengeId - }) + }, _.get(challenge, 'billing.billingAccountId')) // Push Submission deleted event to Bus API // Request body for Posting to Bus API