From f5742f60dabd59fb92164f84b285af247346fdaf Mon Sep 17 00:00:00 2001 From: bountyCoder Date: Fri, 10 Feb 2023 10:35:32 +0000 Subject: [PATCH 01/33] moved cache to redis --- app.js | 1 - config/default.js | 5 +++- package-lock.json | 58 ++++++++++++++++++++++++++++++++++++ package.json | 1 + src/common/helper.js | 34 +++++++++++++-------- src/services/GroupService.js | 50 ++++++++++++++++++++----------- 6 files changed, 117 insertions(+), 32 deletions(-) diff --git a/app.js b/app.js index dc86a6c..95b62ca 100644 --- a/app.js +++ b/app.js @@ -70,7 +70,6 @@ app.use((err, req, res, next) => { const server = app.listen(app.get('port'), () => { logger.info(`Express server listening on port ${app.get('port')}`) - helper.initiateCache() }) server.on('close', async (error) => { diff --git a/config/default.js b/config/default.js index 6dfb3a6..c7bdfbf 100644 --- a/config/default.js +++ b/config/default.js @@ -52,5 +52,8 @@ module.exports = { MEMBERSHIP_TYPES: { Group: 'group', User: 'user' - } + }, + + REDIS_URL: process.env.REDIS_URL || 'redis://localhost:6379', + CACHE_TTL: process.env.CACHE_TTL || 3600 } diff --git a/package-lock.json b/package-lock.json index 24a98e5..4795b4e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -127,6 +127,41 @@ "fastq": "^1.6.0" } }, + "@redis/bloom": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz", + "integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==" + }, + "@redis/client": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.5.5.tgz", + "integrity": "sha512-fuMnpDYSjT5JXR9rrCW1YWA4L8N/9/uS4ImT3ZEC/hcaQRI1D/9FvwjriRj1UvepIgzZXthFVKMNRzP/LNL7BQ==", + "requires": { + "cluster-key-slot": "1.1.2", + "generic-pool": "3.9.0", + "yallist": "4.0.0" + } + }, + "@redis/graph": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.0.tgz", + "integrity": "sha512-16yZWngxyXPd+MJxeSr0dqh2AIOi8j9yXKcKCwVaKDbH3HTuETpDVPcLujhFYVPtYrngSco31BUcSa9TH31Gqg==" + }, + "@redis/json": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.4.tgz", + "integrity": "sha512-LUZE2Gdrhg0Rx7AN+cZkb1e6HjoSKaeeW8rYnt89Tly13GBI5eP4CwDVr+MY8BAYfCg4/N15OUrtLoona9uSgw==" + }, + "@redis/search": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@redis/search/-/search-1.1.1.tgz", + "integrity": "sha512-pqCXTc5e7wJJgUuJiC3hBgfoFRoPxYzwn0BEfKgejTM7M/9zP3IpUcqcjgfp8hF+LoV8rHZzcNTz7V+pEIY7LQ==" + }, + "@redis/time-series": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.0.4.tgz", + "integrity": "sha512-ThUIgo2U/g7cCuZavucQTQzA9g9JbDDY2f64u3AbAoz/8vE2lt2U37LamDUVChhaDA3IRT9R6VvJwqnUfTJzng==" + }, "@tootallnate/once": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz", @@ -724,6 +759,11 @@ "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz", "integrity": "sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==" }, + "cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==" + }, "codependency": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/codependency/-/codependency-0.1.4.tgz", @@ -1668,6 +1708,11 @@ "integrity": "sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==", "dev": true }, + "generic-pool": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz", + "integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==" + }, "get-intrinsic": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.3.tgz", @@ -3191,6 +3236,19 @@ "backoff": "~2.5.0" } }, + "redis": { + "version": "4.6.4", + "resolved": "https://registry.npmjs.org/redis/-/redis-4.6.4.tgz", + "integrity": "sha512-wi2tgDdQ+Q8q+PR5FLRx4QvDiWaA+PoJbrzsyFqlClN5R4LplHqN3scs/aGjE//mbz++W19SgxiEnQ27jnCRaA==", + "requires": { + "@redis/bloom": "1.2.0", + "@redis/client": "1.5.5", + "@redis/graph": "1.1.0", + "@redis/json": "1.0.4", + "@redis/search": "1.1.1", + "@redis/time-series": "1.0.4" + } + }, "regenerator-runtime": { "version": "0.13.11", "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz", diff --git a/package.json b/package.json index 9c54488..ffef9fb 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "morgan": "^1.7.0", "neo4j-driver": "^5.3.0", "node-cache": "^5.1.2", + "redis": "^4.6.4", "swagger-ui-express": "^4.1.5", "tc-bus-api-wrapper": "topcoder-platform/tc-bus-api-wrapper.git#feature/auth0-proxy-server", "tc-core-library-js": "appirio-tech/tc-core-library-js.git#v2.6.4", diff --git a/src/common/helper.js b/src/common/helper.js index 8f6f5a6..02c4798 100644 --- a/src/common/helper.js +++ b/src/common/helper.js @@ -8,6 +8,7 @@ const config = require('config') const neo4j = require('neo4j-driver') const nodeCache = require('node-cache') const querystring = require('querystring') +const redis = require('redis') const uuid = require('uuid') const validate = require('uuid-validate') @@ -15,11 +16,9 @@ const errors = require('./errors') const logger = require('./logger') const constants = require('../../app-constants') -// Bus API Client -let busApiClient -// cache instance -let cache = null +let busApiClient // Bus API Client +let redisClient = null // redis client const driver = neo4j.driver(config.GRAPH_DB_URI, neo4j.auth.basic(config.GRAPH_DB_USER, config.GRAPH_DB_PASSWORD), { maxConnectionLifetime: 3 * 60 * 60 * 1000, @@ -389,15 +388,25 @@ async function deleteGroup(tx, group) { return groupsToDelete } -async function initiateCache() { - cache = new nodeCache({ - stdTTL: 432000, - checkperiod: 518400 - }); +async function acquireRedisClient() { + if (redisClient == null) { + logger.debug("Creating new redis client") + return createRedisClient() + } else { + const pong = await redisClient.ping() + if (!pong) { + logger.debug("Redis connection lost, creating new redis client") + return createRedisClient() + } + } } -async function getCacheInstance() { - return cache +async function createRedisClient() { + const con = redis.createClient({ url: config.REDIS_URL }) + con.on("error", function (err) { + logger.error(err) + }) + return con } module.exports = { @@ -416,6 +425,5 @@ module.exports = { postBusEvent, createGroup, deleteGroup, - initiateCache, - getCacheInstance + acquireRedisClient } diff --git a/src/services/GroupService.js b/src/services/GroupService.js index bd2b1eb..bd1423d 100644 --- a/src/services/GroupService.js +++ b/src/services/GroupService.js @@ -100,7 +100,7 @@ async function searchGroups(criteria, isAdmin) { const totalRes = await session.run(`${matchClause}${whereClause} RETURN COUNT(g)`) const total = totalRes.records[0].get(0).low || 0 - console.log(`${matchClause}${whereClause} RETURN g ORDER BY g.oldId SKIP ${(criteria.page - 1) * criteria.perPage} + logger.debug(`${matchClause}${whereClause} RETURN g ORDER BY g.oldId SKIP ${(criteria.page - 1) * criteria.perPage} LIMIT ${criteria.perPage}`) // query page of records @@ -232,6 +232,7 @@ createGroup.schema = { async function updateGroup(currentUser, groupId, data) { const session = helper.createDBSession() const tx = session.beginTransaction() + const redisClient = await helper.acquireRedisClient() try { logger.debug(`Update Group - user - ${currentUser} , data - ${JSON.stringify(data)}`) const group = await helper.ensureExists( @@ -271,8 +272,11 @@ async function updateGroup(currentUser, groupId, data) { await tx.commit() // update the cache only if the group has the `oldId` - const cache = await helper.getCacheInstance() - if (updateGroup.oldId && updateGroup.oldId.length > 0) cache.set(group.id, updatedGroup) + if (updatedGroup.oldId && updatedGroup.oldId.length > 0) { + await redisClient.connect() + await redisClient.set(`Group:${group.id}`, JSON.stringify(updatedGroup), { EX: config.CACHE_TTL, NX: true }) + await redisClient.disconnect() + } return updatedGroup } catch (error) { @@ -305,6 +309,7 @@ updateGroup.schema = { async function patchGroup(currentUser, groupId, data) { const session = helper.createDBSession() const tx = session.beginTransaction() + const redisClient = await helper.acquireRedisClient() try { logger.debug(`Patch Group - user - ${currentUser} , data - ${JSON.stringify(data)}`) const group = await helper.ensureExists( @@ -331,8 +336,8 @@ async function patchGroup(currentUser, groupId, data) { await tx.commit() // update the cache - const cache = await helper.getCacheInstance() - cache.set(group.id, updatedGroup) + await redisClient.connect() + await redisClient.set(`Group:${group.id}`, JSON.stringify(updatedGroup), { EX: config.CACHE_TTL, NX: true }) return updatedGroup } catch (error) { @@ -343,6 +348,7 @@ async function patchGroup(currentUser, groupId, data) { } finally { logger.debug('Session Close') await session.close() + await redisClient.disconnect() } } @@ -428,43 +434,49 @@ async function getGroup(currentUser, groupId, criteria) { } return session } - const cache = await helper.getCacheInstance() + + const redisClient = await helper.acquireRedisClient() + await redisClient.connect() let groupToReturn try { if (!criteria.skipCache) { // check for the availibility of the group in cache - groupToReturn = cache.get(groupId) + groupToReturn = JSON.parse(await redisClient.get(`Group:${groupId}`)) } if (!criteria.skipCache && groupToReturn) { + logger.debug('group found in cache') if (!isAdmin) delete groupToReturn.status // if the group is private, the user needs to be a member of the group, or an admin if (groupToReturn.privateGroup && currentUser !== 'M2M' && !helper.hasAdminRole(currentUser)) { - const cachedGroupMembers = cache.get(`${groupId}-members`) + const cachedGroupMembers = JSON.parse(await redisClient.get(`GroupMembers:${groupId}`)) if (!_.includes(cachedGroupMembers, currentUser.userId)) { await helper.ensureGroupMember(getSession(), groupId, currentUser.userId) - cachedGroupMembers.push(currentUser.userId) - cache.set(`${groupId}-members`, cachedGroupMembers) + await redisClient.set(`GroupMembers:${groupId}`, JSON.stringify(cachedGroupMembers), { EX: config.CACHE_TTL, NX: true }) } } } else { + logger.debug('group not found in cache, getting from DB') groupToReturn = await helper.ensureExists(getSession(), 'Group', groupId, isAdmin) // set the group in cache only if it is having the `oldId` - if (groupToReturn.oldId && groupToReturn.oldId.length > 0) cache.set(groupId, groupToReturn) + if (groupToReturn.oldId && groupToReturn.oldId.length > 0) { + logger.debug('saving group in cache') + await redisClient.set(`Group:${groupId}`, JSON.stringify(groupToReturn), { EX: config.CACHE_TTL, NX: true }) + } + if (!isAdmin) delete groupToReturn.status // if the group is private, the user needs to be a member of the group, or an admin if (groupToReturn.privateGroup && currentUser !== 'M2M' && !helper.hasAdminRole(currentUser)) { await helper.ensureGroupMember(getSession(), groupToReturn.id, currentUser.userId) - - cache.set(`${groupId}-members`, [currentUser.userId]) + await redisClient.set(`GroupMembers:${groupId}`, JSON.stringify([currentUser.userId]), { EX: config.CACHE_TTL, NX: true }) } } @@ -503,7 +515,8 @@ async function getGroup(currentUser, groupId, criteria) { groupToReturn.flattenGroupIdTree = flattenGroupIdTree // set the group in cache only if it is having the `oldId` - if (groupToReturn.oldId && groupToReturn.oldId.length > 0) cache.set(groupId, groupToReturn) + if (groupToReturn.oldId && groupToReturn.oldId.length > 0) + await redisClient.set(`Group:${groupId}`, JSON.stringify(groupToReturn), { EX: config.CACHE_TTL, NX: true }) } } else if (criteria.includeParentGroup && !groupToReturn.parentGroups) { // find parent groups @@ -537,6 +550,7 @@ async function getGroup(currentUser, groupId, criteria) { if (session) { await session.close() } + await redisClient.disconnect() } } @@ -562,6 +576,7 @@ getGroup.schema = { async function deleteGroup(groupId, isAdmin) { const session = helper.createDBSession() const tx = session.beginTransaction() + const redisClient = await helper.acquireRedisClient() try { logger.debug(`Delete Group - ${groupId}`) const group = await helper.ensureExists(tx, 'Group', groupId, isAdmin) @@ -574,9 +589,9 @@ async function deleteGroup(groupId, isAdmin) { await tx.commit() // delete the cache - const cache = await helper.getCacheInstance() - cache.del(group.id) - cache.del(`${group.id}-members`) + await redisClient.connect() + await redisClient.del(`Group:${group.id}`) + await redisClient.del(`GroupMembers:${group.id}`) return group } catch (error) { @@ -587,6 +602,7 @@ async function deleteGroup(groupId, isAdmin) { } finally { logger.debug('Session Close') await session.close() + await redisClient.disconnect() } } From 70e8fd6aefb1cbcfec1fb2ddfe76f87a25569935 Mon Sep 17 00:00:00 2001 From: bountyCoder Date: Fri, 10 Feb 2023 10:49:33 +0000 Subject: [PATCH 02/33] updated the config file for the cache TTL --- config/default.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/default.js b/config/default.js index c7bdfbf..bef09f5 100644 --- a/config/default.js +++ b/config/default.js @@ -55,5 +55,5 @@ module.exports = { }, REDIS_URL: process.env.REDIS_URL || 'redis://localhost:6379', - CACHE_TTL: process.env.CACHE_TTL || 3600 + CACHE_TTL: process.env.CACHE_TTL ? Number(process.env.CACHE_TTL) : 3600 } From eb916e65a700b3698cb396c815b3141ed07fcf2f Mon Sep 17 00:00:00 2001 From: bountyCoder Date: Fri, 10 Feb 2023 14:55:13 +0000 Subject: [PATCH 03/33] fixed missing - flush all cache --- src/controllers/GroupController.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/controllers/GroupController.js b/src/controllers/GroupController.js index f9e21db..86a8fd7 100644 --- a/src/controllers/GroupController.js +++ b/src/controllers/GroupController.js @@ -97,8 +97,10 @@ async function getGroupByOldId(req, res) { * @param res the response */ async function flushCache(req, res) { - const cachedData = await helper.getCacheInstance() - cachedData.flushAll() + const redisClient = await helper.acquireRedisClient() + await redisClient.connect() + await redisClient.FLUSHALL() + await redisClient.disconnect() res.send({ message: 'all cached data has been removed' From 1e3c76a958037c34f3db3407c82719fd0090c14d Mon Sep 17 00:00:00 2001 From: bountyCoder Date: Mon, 13 Feb 2023 17:24:54 +0000 Subject: [PATCH 04/33] incorporated the review comments --- src/common/helper.js | 26 +++++++++++--------------- src/controllers/GroupController.js | 2 -- src/services/GroupService.js | 20 ++++++-------------- 3 files changed, 17 insertions(+), 31 deletions(-) diff --git a/src/common/helper.js b/src/common/helper.js index 02c4798..771a586 100644 --- a/src/common/helper.js +++ b/src/common/helper.js @@ -389,24 +389,20 @@ async function deleteGroup(tx, group) { } async function acquireRedisClient() { + + if (redisClient == null) { - logger.debug("Creating new redis client") - return createRedisClient() - } else { - const pong = await redisClient.ping() - if (!pong) { - logger.debug("Redis connection lost, creating new redis client") - return createRedisClient() - } + logger.debug("creating new redis client") + redisClient = redis.createClient({ url: config.REDIS_URL }) + + redisClient.on('connect', () => logger.debug('redis client connected')); + redisClient.on('error', err => logger.error(new Date(), 'client error', err.message)); + redisClient.on('reconnecting', () => logger.debug('reconnecting')); + + await redisClient.connect() } -} -async function createRedisClient() { - const con = redis.createClient({ url: config.REDIS_URL }) - con.on("error", function (err) { - logger.error(err) - }) - return con + return redisClient } module.exports = { diff --git a/src/controllers/GroupController.js b/src/controllers/GroupController.js index 86a8fd7..b2c5485 100644 --- a/src/controllers/GroupController.js +++ b/src/controllers/GroupController.js @@ -98,9 +98,7 @@ async function getGroupByOldId(req, res) { */ async function flushCache(req, res) { const redisClient = await helper.acquireRedisClient() - await redisClient.connect() await redisClient.FLUSHALL() - await redisClient.disconnect() res.send({ message: 'all cached data has been removed' diff --git a/src/services/GroupService.js b/src/services/GroupService.js index bd1423d..7138105 100644 --- a/src/services/GroupService.js +++ b/src/services/GroupService.js @@ -273,9 +273,7 @@ async function updateGroup(currentUser, groupId, data) { // update the cache only if the group has the `oldId` if (updatedGroup.oldId && updatedGroup.oldId.length > 0) { - await redisClient.connect() - await redisClient.set(`Group:${group.id}`, JSON.stringify(updatedGroup), { EX: config.CACHE_TTL, NX: true }) - await redisClient.disconnect() + await redisClient.set(`Group:${group.id}`, JSON.stringify(updatedGroup), { EX: config.CACHE_TTL }) } return updatedGroup @@ -336,8 +334,7 @@ async function patchGroup(currentUser, groupId, data) { await tx.commit() // update the cache - await redisClient.connect() - await redisClient.set(`Group:${group.id}`, JSON.stringify(updatedGroup), { EX: config.CACHE_TTL, NX: true }) + await redisClient.set(`Group:${group.id}`, JSON.stringify(updatedGroup), { EX: config.CACHE_TTL }) return updatedGroup } catch (error) { @@ -348,7 +345,6 @@ async function patchGroup(currentUser, groupId, data) { } finally { logger.debug('Session Close') await session.close() - await redisClient.disconnect() } } @@ -436,7 +432,6 @@ async function getGroup(currentUser, groupId, criteria) { } const redisClient = await helper.acquireRedisClient() - await redisClient.connect() let groupToReturn @@ -457,7 +452,7 @@ async function getGroup(currentUser, groupId, criteria) { if (!_.includes(cachedGroupMembers, currentUser.userId)) { await helper.ensureGroupMember(getSession(), groupId, currentUser.userId) cachedGroupMembers.push(currentUser.userId) - await redisClient.set(`GroupMembers:${groupId}`, JSON.stringify(cachedGroupMembers), { EX: config.CACHE_TTL, NX: true }) + await redisClient.set(`GroupMembers:${groupId}`, JSON.stringify(cachedGroupMembers), { EX: config.CACHE_TTL }) } } } else { @@ -467,7 +462,7 @@ async function getGroup(currentUser, groupId, criteria) { // set the group in cache only if it is having the `oldId` if (groupToReturn.oldId && groupToReturn.oldId.length > 0) { logger.debug('saving group in cache') - await redisClient.set(`Group:${groupId}`, JSON.stringify(groupToReturn), { EX: config.CACHE_TTL, NX: true }) + await redisClient.set(`Group:${groupId}`, JSON.stringify(groupToReturn), { EX: config.CACHE_TTL }) } @@ -476,7 +471,7 @@ async function getGroup(currentUser, groupId, criteria) { // if the group is private, the user needs to be a member of the group, or an admin if (groupToReturn.privateGroup && currentUser !== 'M2M' && !helper.hasAdminRole(currentUser)) { await helper.ensureGroupMember(getSession(), groupToReturn.id, currentUser.userId) - await redisClient.set(`GroupMembers:${groupId}`, JSON.stringify([currentUser.userId]), { EX: config.CACHE_TTL, NX: true }) + await redisClient.set(`GroupMembers:${groupId}`, JSON.stringify([currentUser.userId]), { EX: config.CACHE_TTL }) } } @@ -516,7 +511,7 @@ async function getGroup(currentUser, groupId, criteria) { // set the group in cache only if it is having the `oldId` if (groupToReturn.oldId && groupToReturn.oldId.length > 0) - await redisClient.set(`Group:${groupId}`, JSON.stringify(groupToReturn), { EX: config.CACHE_TTL, NX: true }) + await redisClient.set(`Group:${groupId}`, JSON.stringify(groupToReturn), { EX: config.CACHE_TTL }) } } else if (criteria.includeParentGroup && !groupToReturn.parentGroups) { // find parent groups @@ -550,7 +545,6 @@ async function getGroup(currentUser, groupId, criteria) { if (session) { await session.close() } - await redisClient.disconnect() } } @@ -589,7 +583,6 @@ async function deleteGroup(groupId, isAdmin) { await tx.commit() // delete the cache - await redisClient.connect() await redisClient.del(`Group:${group.id}`) await redisClient.del(`GroupMembers:${group.id}`) @@ -602,7 +595,6 @@ async function deleteGroup(groupId, isAdmin) { } finally { logger.debug('Session Close') await session.close() - await redisClient.disconnect() } } From ecb156d704cfa79dbe7c8e60548f099283853d99 Mon Sep 17 00:00:00 2001 From: bountyCoder Date: Tue, 14 Feb 2023 18:42:21 +0000 Subject: [PATCH 05/33] incorporated the review comments --- config/default.js | 2 +- src/common/helper.js | 2 +- src/services/GroupService.js | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/config/default.js b/config/default.js index bef09f5..ca39510 100644 --- a/config/default.js +++ b/config/default.js @@ -54,6 +54,6 @@ module.exports = { User: 'user' }, - REDIS_URL: process.env.REDIS_URL || 'redis://localhost:6379', + REDIS_URL: process.env.REDIS_URL ? process.env.REDIS_URL : 'redis://localhost:6379', CACHE_TTL: process.env.CACHE_TTL ? Number(process.env.CACHE_TTL) : 3600 } diff --git a/src/common/helper.js b/src/common/helper.js index 771a586..fd3147b 100644 --- a/src/common/helper.js +++ b/src/common/helper.js @@ -391,7 +391,7 @@ async function deleteGroup(tx, group) { async function acquireRedisClient() { - if (redisClient == null) { + if (!redisClient) { logger.debug("creating new redis client") redisClient = redis.createClient({ url: config.REDIS_URL }) diff --git a/src/services/GroupService.js b/src/services/GroupService.js index 7138105..8c59b0b 100644 --- a/src/services/GroupService.js +++ b/src/services/GroupService.js @@ -583,8 +583,7 @@ async function deleteGroup(groupId, isAdmin) { await tx.commit() // delete the cache - await redisClient.del(`Group:${group.id}`) - await redisClient.del(`GroupMembers:${group.id}`) + await Promise.all([redisClient.del(`Group:${group.id}`), redisClient.del(`GroupMembers:${group.id}`)]) return group } catch (error) { From c04383a952b2f64c25b83b9144c07445f7d8fa41 Mon Sep 17 00:00:00 2001 From: bountyCoder Date: Thu, 16 Feb 2023 11:37:21 +0000 Subject: [PATCH 06/33] fixed the issue while adding the member to the group --- package-lock.json | 13 ------------- package.json | 1 - src/common/helper.js | 1 - src/services/GroupMembershipService.js | 14 +++++++++++--- 4 files changed, 11 insertions(+), 18 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4795b4e..8c4b729 100644 --- a/package-lock.json +++ b/package-lock.json @@ -754,11 +754,6 @@ "readdirp": "~3.6.0" } }, - "clone": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz", - "integrity": "sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==" - }, "cluster-key-slot": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", @@ -2742,14 +2737,6 @@ "resolved": "https://registry.npmjs.org/neo4j-driver-core/-/neo4j-driver-core-5.3.0.tgz", "integrity": "sha512-MGxHCU0Pt13ElxYZkObWoxZ8O5y0Guf/R3rcogDUktG6cp69rc+lWxZAR7HbsL+gc8NrdJlIxR7dIud7B3bhAA==" }, - "node-cache": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/node-cache/-/node-cache-5.1.2.tgz", - "integrity": "sha512-t1QzWwnk4sjLWaQAS8CHgOJ+RAfmHpxFWmc36IWTiWHQfs0w5JDMBS1b1ZxQteo0vVVuWJvIUKHDkkeK7vIGCg==", - "requires": { - "clone": "2.x" - } - }, "nodemon": { "version": "2.0.20", "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-2.0.20.tgz", diff --git a/package.json b/package.json index ffef9fb..dce6eda 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,6 @@ "lodash": "^4.17.19", "morgan": "^1.7.0", "neo4j-driver": "^5.3.0", - "node-cache": "^5.1.2", "redis": "^4.6.4", "swagger-ui-express": "^4.1.5", "tc-bus-api-wrapper": "topcoder-platform/tc-bus-api-wrapper.git#feature/auth0-proxy-server", diff --git a/src/common/helper.js b/src/common/helper.js index fd3147b..83fb9f3 100644 --- a/src/common/helper.js +++ b/src/common/helper.js @@ -6,7 +6,6 @@ const _ = require('lodash') const busApi = require('tc-bus-api-wrapper') const config = require('config') const neo4j = require('neo4j-driver') -const nodeCache = require('node-cache') const querystring = require('querystring') const redis = require('redis') const uuid = require('uuid') diff --git a/src/services/GroupMembershipService.js b/src/services/GroupMembershipService.js index 428c8b2..97064f7 100644 --- a/src/services/GroupMembershipService.js +++ b/src/services/GroupMembershipService.js @@ -11,6 +11,8 @@ const errors = require('../common/errors') const validate = require('uuid-validate') const constants = require('../../app-constants') +const GroupService = require('./GroupService') + /** * Add group member. * @param {Object} currentUser the current user @@ -63,11 +65,17 @@ async function addGroupMember(currentUser, groupId, data) { } // update the cache - const cache = await helper.getCacheInstance() - const cachedGroup = cache.get(group.id) + const redisClient = await helper.acquireRedisClient() + let cachedGroup = JSON.parse(await redisClient.get(`Group:${group.id}`)) + + if (!cachedGroup) { + await GroupService.getGroup(currentUser, group.id, { includeSubGroups: true }) + cachedGroup = JSON.parse(await redisClient.get(`Group:${group.id}`)) + } + cachedGroup.subGroups.push(childGroup) cachedGroup.flattenGroupIdTree.push(childGroup.id) - cache.set(group.id, cachedGroup) + await redisClient.set(`Group:${group.id}`, JSON.stringify(cachedGroup), { EX: config.CACHE_TTL }) } else { logger.debug(`Check for memberId ${memberId} exist or not`) await helper.ensureExists(tx, 'User', memberId) From 035f1e497d62b8b9d60899502356c9cf2a5921da Mon Sep 17 00:00:00 2001 From: bountyCoder Date: Thu, 23 Feb 2023 20:03:17 +0000 Subject: [PATCH 07/33] fix for the delayed sync for oldId --- src/services/GroupMembershipService.js | 5 +++ src/services/GroupService.js | 56 +++++++++++++------------- 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/src/services/GroupMembershipService.js b/src/services/GroupMembershipService.js index 97064f7..2b6556e 100644 --- a/src/services/GroupMembershipService.js +++ b/src/services/GroupMembershipService.js @@ -34,9 +34,14 @@ async function addGroupMember(currentUser, groupId, data) { groupId, currentUser !== 'M2M' && helper.hasAdminRole(currentUser) ) + data.oldId = group.oldId groupId = group.id + if (!group.oldId || group.oldId.length <= 0) { + throw new errors.ForbiddenError('Parent group is not ready yet, try after sometime') + } + const memberId = data.memberId ? data.memberId : data.universalUID if ( diff --git a/src/services/GroupService.js b/src/services/GroupService.js index 8c59b0b..f1f5f38 100644 --- a/src/services/GroupService.js +++ b/src/services/GroupService.js @@ -323,14 +323,12 @@ async function patchGroup(currentUser, groupId, data) { groupData.updatedBy = currentUser === 'M2M' ? '00000000' : currentUser.userId groupData.oldId = data.oldId ? data.oldId : '' - const updateRes = await tx.run( + await tx.run( 'MATCH (g:Group {id: {id}}) SET g.updatedAt={updatedAt}, g.updatedBy={updatedBy}, g.oldId={oldId} RETURN g', groupData ) - const updatedGroup = updateRes.records[0].get(0).properties - logger.debug(`Group = ${JSON.stringify(updatedGroup)}`) - + const updatedGroup = getGroup(currentUser, groupId, { includeSubGroups: true }) await tx.commit() // update the cache @@ -465,7 +463,6 @@ async function getGroup(currentUser, groupId, criteria) { await redisClient.set(`Group:${groupId}`, JSON.stringify(groupToReturn), { EX: config.CACHE_TTL }) } - if (!isAdmin) delete groupToReturn.status // if the group is private, the user needs to be a member of the group, or an admin @@ -513,40 +510,41 @@ async function getGroup(currentUser, groupId, criteria) { if (groupToReturn.oldId && groupToReturn.oldId.length > 0) await redisClient.set(`Group:${groupId}`, JSON.stringify(groupToReturn), { EX: config.CACHE_TTL }) } - } else if (criteria.includeParentGroup && !groupToReturn.parentGroups) { - // find parent groups - groupToExpand.parentGroups = await helper.getParentGroups(getSession(), groupToExpand.id) - // add parent groups to pending if needed - if (!criteria.oneLevel) { - _.forEach(groupToExpand.parentGroups, (g) => pending.push(g)) - } + } + } else if (criteria.includeParentGroup && !groupToReturn.parentGroups) { + // find parent groups + groupToExpand.parentGroups = await helper.getParentGroups(getSession(), groupToExpand.id) + // add parent groups to pending if needed + if (!criteria.oneLevel) { + _.forEach(groupToExpand.parentGroups, (g) => pending.push(g)) } } } + } if (fieldNames) { - fieldNames.push('subGroups') - fieldNames.push('parentGroups') - - groupToReturn = _.pick(groupToReturn, fieldNames) - } + fieldNames.push('subGroups') + fieldNames.push('parentGroups') - if (!criteria.includeSubGroups) delete groupToReturn.subGroups - if (!criteria.includeParentGroup) delete groupToReturn.parentGroups - if (!criteria.flattenGroupIdTree) delete groupToReturn.flattenGroupIdTree + groupToReturn = _.pick(groupToReturn, fieldNames) + } - return groupToReturn - } catch (error) { - logger.error(error) - throw error - } finally { - logger.debug('Session Close') - if (session) { - await session.close() - } + if (!criteria.includeSubGroups) delete groupToReturn.subGroups + if (!criteria.includeParentGroup) delete groupToReturn.parentGroups + if (!criteria.flattenGroupIdTree) delete groupToReturn.flattenGroupIdTree + + return groupToReturn +} catch (error) { + logger.error(error) + throw error +} finally { + logger.debug('Session Close') + if (session) { + await session.close() } } +} getGroup.schema = { currentUser: Joi.any(), From fcd762e5b955720ce4d24a4012572c59d9bd63e4 Mon Sep 17 00:00:00 2001 From: bountyCoder Date: Thu, 23 Feb 2023 20:33:08 +0000 Subject: [PATCH 08/33] fixing bug --- src/services/GroupService.js | 48 ++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/src/services/GroupService.js b/src/services/GroupService.js index f1f5f38..664efe5 100644 --- a/src/services/GroupService.js +++ b/src/services/GroupService.js @@ -511,40 +511,40 @@ async function getGroup(currentUser, groupId, criteria) { await redisClient.set(`Group:${groupId}`, JSON.stringify(groupToReturn), { EX: config.CACHE_TTL }) } } - } else if (criteria.includeParentGroup && !groupToReturn.parentGroups) { - // find parent groups - groupToExpand.parentGroups = await helper.getParentGroups(getSession(), groupToExpand.id) - // add parent groups to pending if needed - if (!criteria.oneLevel) { - _.forEach(groupToExpand.parentGroups, (g) => pending.push(g)) + else if (criteria.includeParentGroup && !groupToReturn.parentGroups) { + // find parent groups + groupToExpand.parentGroups = await helper.getParentGroups(getSession(), groupToExpand.id) + // add parent groups to pending if needed + if (!criteria.oneLevel) { + _.forEach(groupToExpand.parentGroups, (g) => pending.push(g)) + } } } } - } if (fieldNames) { - fieldNames.push('subGroups') - fieldNames.push('parentGroups') + fieldNames.push('subGroups') + fieldNames.push('parentGroups') - groupToReturn = _.pick(groupToReturn, fieldNames) - } + groupToReturn = _.pick(groupToReturn, fieldNames) + } - if (!criteria.includeSubGroups) delete groupToReturn.subGroups - if (!criteria.includeParentGroup) delete groupToReturn.parentGroups - if (!criteria.flattenGroupIdTree) delete groupToReturn.flattenGroupIdTree - - return groupToReturn -} catch (error) { - logger.error(error) - throw error -} finally { - logger.debug('Session Close') - if (session) { - await session.close() + if (!criteria.includeSubGroups) delete groupToReturn.subGroups + if (!criteria.includeParentGroup) delete groupToReturn.parentGroups + if (!criteria.flattenGroupIdTree) delete groupToReturn.flattenGroupIdTree + + return groupToReturn + } catch (error) { + logger.error(error) + throw error + } finally { + logger.debug('Session Close') + if (session) { + await session.close() + } } } -} getGroup.schema = { currentUser: Joi.any(), From c342d6fa8df6ea669ef9a742d81817b1af79ed93 Mon Sep 17 00:00:00 2001 From: bountyCoder Date: Fri, 24 Feb 2023 19:10:56 +0000 Subject: [PATCH 09/33] fix the cache mismatch issue and added the logs --- src/services/GroupService.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/services/GroupService.js b/src/services/GroupService.js index 664efe5..7c2a52e 100644 --- a/src/services/GroupService.js +++ b/src/services/GroupService.js @@ -328,9 +328,13 @@ async function patchGroup(currentUser, groupId, data) { groupData ) - const updatedGroup = getGroup(currentUser, groupId, { includeSubGroups: true }) + const updatedGroup = await getGroup(currentUser, groupId, { includeSubGroups: true }) await tx.commit() + logger.debug('*****') + logger.debug(JSON.stringify(updateGroup)) + logger.debug('*****') + // update the cache await redisClient.set(`Group:${group.id}`, JSON.stringify(updatedGroup), { EX: config.CACHE_TTL }) From ad2f5abd0a9eb2cea7ec636f051394e7c331adca Mon Sep 17 00:00:00 2001 From: bountyCoder Date: Fri, 24 Feb 2023 19:20:52 +0000 Subject: [PATCH 10/33] removed the debug log lines --- src/services/GroupService.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/services/GroupService.js b/src/services/GroupService.js index 7c2a52e..e379db2 100644 --- a/src/services/GroupService.js +++ b/src/services/GroupService.js @@ -331,10 +331,6 @@ async function patchGroup(currentUser, groupId, data) { const updatedGroup = await getGroup(currentUser, groupId, { includeSubGroups: true }) await tx.commit() - logger.debug('*****') - logger.debug(JSON.stringify(updateGroup)) - logger.debug('*****') - // update the cache await redisClient.set(`Group:${group.id}`, JSON.stringify(updatedGroup), { EX: config.CACHE_TTL }) From 38a9b8e81a5eb81c2cbe37cf2a639ff4a89d75be Mon Sep 17 00:00:00 2001 From: bountyCoder Date: Mon, 27 Feb 2023 16:51:56 +0000 Subject: [PATCH 11/33] updated the commit seq --- src/services/GroupService.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/services/GroupService.js b/src/services/GroupService.js index e379db2..9087382 100644 --- a/src/services/GroupService.js +++ b/src/services/GroupService.js @@ -278,7 +278,6 @@ async function updateGroup(currentUser, groupId, data) { return updatedGroup } catch (error) { - logger.error(error) logger.debug('Transaction Rollback') await tx.rollback() throw error @@ -327,9 +326,9 @@ async function patchGroup(currentUser, groupId, data) { 'MATCH (g:Group {id: {id}}) SET g.updatedAt={updatedAt}, g.updatedBy={updatedBy}, g.oldId={oldId} RETURN g', groupData ) + await tx.commit() const updatedGroup = await getGroup(currentUser, groupId, { includeSubGroups: true }) - await tx.commit() // update the cache await redisClient.set(`Group:${group.id}`, JSON.stringify(updatedGroup), { EX: config.CACHE_TTL }) From aa368c601cbe752b6762ddb6c851d58b03adb8e2 Mon Sep 17 00:00:00 2001 From: bountyCoder Date: Mon, 27 Feb 2023 18:07:39 +0000 Subject: [PATCH 12/33] skip cache in multiple patch calls --- src/services/GroupService.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/GroupService.js b/src/services/GroupService.js index 9087382..10b5014 100644 --- a/src/services/GroupService.js +++ b/src/services/GroupService.js @@ -328,7 +328,7 @@ async function patchGroup(currentUser, groupId, data) { ) await tx.commit() - const updatedGroup = await getGroup(currentUser, groupId, { includeSubGroups: true }) + const updatedGroup = await getGroup(currentUser, groupId, { includeSubGroups: true, skipCache: true }) // update the cache await redisClient.set(`Group:${group.id}`, JSON.stringify(updatedGroup), { EX: config.CACHE_TTL }) From bd3cc076bb8deed6eca0926ab25911999faa3730 Mon Sep 17 00:00:00 2001 From: bountyCoder Date: Tue, 28 Feb 2023 17:17:44 +0000 Subject: [PATCH 13/33] fixed the issue for adding member --- src/services/GroupMembershipService.js | 3 ++- src/services/GroupService.js | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/services/GroupMembershipService.js b/src/services/GroupMembershipService.js index 2b6556e..d7a2627 100644 --- a/src/services/GroupMembershipService.js +++ b/src/services/GroupMembershipService.js @@ -74,7 +74,8 @@ async function addGroupMember(currentUser, groupId, data) { let cachedGroup = JSON.parse(await redisClient.get(`Group:${group.id}`)) if (!cachedGroup) { - await GroupService.getGroup(currentUser, group.id, { includeSubGroups: true }) + logger.debug('inside getting group from DB') + await GroupService.getGroup(currentUser, group.id, { includeSubGroups: true, flattenGroupIdTree: true }) cachedGroup = JSON.parse(await redisClient.get(`Group:${group.id}`)) } diff --git a/src/services/GroupService.js b/src/services/GroupService.js index 10b5014..c59ed69 100644 --- a/src/services/GroupService.js +++ b/src/services/GroupService.js @@ -328,7 +328,7 @@ async function patchGroup(currentUser, groupId, data) { ) await tx.commit() - const updatedGroup = await getGroup(currentUser, groupId, { includeSubGroups: true, skipCache: true }) + const updatedGroup = await getGroup(currentUser, groupId, { includeSubGroups: true, flattenGroupIdTree: true, skipCache: true }) // update the cache await redisClient.set(`Group:${group.id}`, JSON.stringify(updatedGroup), { EX: config.CACHE_TTL }) From e6420306cb3fbd20589e309f52d0b49c9306401b Mon Sep 17 00:00:00 2001 From: bountyCoder Date: Wed, 1 Mar 2023 16:42:11 +0000 Subject: [PATCH 14/33] fix for internal server error --- src/services/GroupService.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/GroupService.js b/src/services/GroupService.js index c59ed69..87d2bc4 100644 --- a/src/services/GroupService.js +++ b/src/services/GroupService.js @@ -444,7 +444,7 @@ async function getGroup(currentUser, groupId, criteria) { // if the group is private, the user needs to be a member of the group, or an admin if (groupToReturn.privateGroup && currentUser !== 'M2M' && !helper.hasAdminRole(currentUser)) { - const cachedGroupMembers = JSON.parse(await redisClient.get(`GroupMembers:${groupId}`)) + const cachedGroupMembers = JSON.parse(await redisClient.get(`GroupMembers:${groupId}`)) || [] if (!_.includes(cachedGroupMembers, currentUser.userId)) { await helper.ensureGroupMember(getSession(), groupId, currentUser.userId) From 6df84138b877b3fc6d1e2bac87567afd4a08e73f Mon Sep 17 00:00:00 2001 From: bountyCoder Date: Tue, 7 Mar 2023 17:56:28 +0000 Subject: [PATCH 15/33] added new role for creating group --- app-constants.js | 3 ++- src/routes.js | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/app-constants.js b/app-constants.js index c770831..6cca7f0 100644 --- a/app-constants.js +++ b/app-constants.js @@ -3,7 +3,8 @@ */ const UserRoles = { Admin: 'Administrator', - User: 'Topcoder User' + User: 'Topcoder User', + TGAdmin: 'tgadmin' } const MembershipTypes = { diff --git a/src/routes.js b/src/routes.js index 6feb464..2c6b0c4 100644 --- a/src/routes.js +++ b/src/routes.js @@ -17,7 +17,7 @@ module.exports = { controller: 'GroupController', method: 'createGroup', auth: 'jwt', - access: [constants.UserRoles.Admin], + access: [constants.UserRoles.Admin, constants.UserRoles.TGAdmin], scopes: ['write:groups', 'all:groups'] } }, From fdc76195f65cc8a6e53e55b0a2c991d7a3a43b24 Mon Sep 17 00:00:00 2001 From: bountyCoder Date: Thu, 9 Mar 2023 19:48:24 +0000 Subject: [PATCH 16/33] updated the caching mechanism --- src/common/helper.js | 35 +++++- src/services/GroupMembershipService.js | 14 +-- src/services/GroupService.js | 156 +++++++++---------------- 3 files changed, 88 insertions(+), 117 deletions(-) diff --git a/src/common/helper.js b/src/common/helper.js index 83fb9f3..e0d9548 100644 --- a/src/common/helper.js +++ b/src/common/helper.js @@ -172,15 +172,18 @@ async function getChildGroups(session, groupId) { /** * Get parent groups. - * @param {Object} session the db session * @param {String} groupId the group id * @returns {Array} the parent groups */ -async function getParentGroups(session, groupId) { +async function getParentGroups(groupId) { + const session = createDBSession() + const res = await session.run( 'MATCH (g:Group)-[r:GroupContains]->(c:Group {id: {groupId}}) RETURN g ORDER BY g.oldId', { groupId } ) + + await session.close() return _.map(res.records, (record) => record.get(0).properties) } @@ -363,7 +366,7 @@ async function deleteGroup(tx, group) { continue } // delete child if it doesn't belong to other group - const parents = await getParentGroups(tx, child.id) + const parents = await getParentGroups(child.id) if (parents.length <= 1) { groupsToDelete.push(child) } @@ -388,8 +391,6 @@ async function deleteGroup(tx, group) { } async function acquireRedisClient() { - - if (!redisClient) { logger.debug("creating new redis client") redisClient = redis.createClient({ url: config.REDIS_URL }) @@ -404,6 +405,26 @@ async function acquireRedisClient() { return redisClient } +async function getCacheKey(criteria) { + let criteriaStr = '' + Object.keys(criteria).forEach(c => { + if (c != 'skipCache') criteriaStr = criteriaStr + c + '=' + criteria[c] + ',' + }) + return criteriaStr.replace(/,\s*$/, "") +} + +async function invalidateCache(group) { + const redisClient = await helper.acquireRedisClient() + const parentGroups = await getParentGroups(group.id) + parentGroups.forEach(async g => { + await redisClient.del(`group:${g.id}:*`) + await redisClient.del(`group:${g.oldId}:*`) + }) + + await redisClient.del(`group:${group.id}:*`) + await redisClient.del(`group:${group.oldId}:*`) +} + module.exports = { wrapExpress, autoWrapExpress, @@ -420,5 +441,7 @@ module.exports = { postBusEvent, createGroup, deleteGroup, - acquireRedisClient + acquireRedisClient, + getCacheKey, + invalidateCache } diff --git a/src/services/GroupMembershipService.js b/src/services/GroupMembershipService.js index d7a2627..a16923a 100644 --- a/src/services/GroupMembershipService.js +++ b/src/services/GroupMembershipService.js @@ -69,19 +69,7 @@ async function addGroupMember(currentUser, groupId, data) { throw new errors.ConflictError('Parent group is private, the child group must be private too.') } - // update the cache - const redisClient = await helper.acquireRedisClient() - let cachedGroup = JSON.parse(await redisClient.get(`Group:${group.id}`)) - - if (!cachedGroup) { - logger.debug('inside getting group from DB') - await GroupService.getGroup(currentUser, group.id, { includeSubGroups: true, flattenGroupIdTree: true }) - cachedGroup = JSON.parse(await redisClient.get(`Group:${group.id}`)) - } - - cachedGroup.subGroups.push(childGroup) - cachedGroup.flattenGroupIdTree.push(childGroup.id) - await redisClient.set(`Group:${group.id}`, JSON.stringify(cachedGroup), { EX: config.CACHE_TTL }) + await helper.invalidateCache(group) } else { logger.debug(`Check for memberId ${memberId} exist or not`) await helper.ensureExists(tx, 'User', memberId) diff --git a/src/services/GroupService.js b/src/services/GroupService.js index 87d2bc4..42c7e75 100644 --- a/src/services/GroupService.js +++ b/src/services/GroupService.js @@ -123,7 +123,7 @@ async function searchGroups(criteria, isAdmin) { if (criteria.includeParentGroup) { for (let i = 0; i < result.length; i += 1) { const group = result[i] - group.parentGroups = await helper.getParentGroups(session, group.id) + group.parentGroups = await helper.getParentGroups(group.id) } } @@ -232,7 +232,6 @@ createGroup.schema = { async function updateGroup(currentUser, groupId, data) { const session = helper.createDBSession() const tx = session.beginTransaction() - const redisClient = await helper.acquireRedisClient() try { logger.debug(`Update Group - user - ${currentUser} , data - ${JSON.stringify(data)}`) const group = await helper.ensureExists( @@ -271,10 +270,7 @@ async function updateGroup(currentUser, groupId, data) { await helper.postBusEvent(config.KAFKA_GROUP_UPDATE_TOPIC, updatedGroup) await tx.commit() - // update the cache only if the group has the `oldId` - if (updatedGroup.oldId && updatedGroup.oldId.length > 0) { - await redisClient.set(`Group:${group.id}`, JSON.stringify(updatedGroup), { EX: config.CACHE_TTL }) - } + await helper.invalidateCache(updatedGroup) return updatedGroup } catch (error) { @@ -306,7 +302,6 @@ updateGroup.schema = { async function patchGroup(currentUser, groupId, data) { const session = helper.createDBSession() const tx = session.beginTransaction() - const redisClient = await helper.acquireRedisClient() try { logger.debug(`Patch Group - user - ${currentUser} , data - ${JSON.stringify(data)}`) const group = await helper.ensureExists( @@ -331,7 +326,7 @@ async function patchGroup(currentUser, groupId, data) { const updatedGroup = await getGroup(currentUser, groupId, { includeSubGroups: true, flattenGroupIdTree: true, skipCache: true }) // update the cache - await redisClient.set(`Group:${group.id}`, JSON.stringify(updatedGroup), { EX: config.CACHE_TTL }) + await helper.invalidateCache(updateGroup) return updatedGroup } catch (error) { @@ -361,7 +356,8 @@ patchGroup.schema = { * @param {Object} criteria the query criteria * @returns {Object} the group */ -async function getGroup(currentUser, groupId, criteria) { + +async function getGroup (currentUser, groupId, criteria) { const isAdmin = currentUser === 'M2M' || helper.hasAdminRole(currentUser) logger.debug( `Get Group - admin - ${isAdmin} - user - ${currentUser} , groupId - ${groupId} , criteria - ${JSON.stringify( @@ -369,10 +365,6 @@ async function getGroup(currentUser, groupId, criteria) { )}` ) - if (!_.has(criteria, 'skipCache')) { - criteria.skipCache = false - } - if (criteria.includeSubGroups && criteria.includeParentGroup) { throw new errors.BadRequestError('includeSubGroups and includeParentGroup can not be both true') } @@ -404,71 +396,45 @@ async function getGroup(currentUser, groupId, criteria) { 'oldId' ] - if (_.uniq(fieldNames).length !== fieldNames.length) { - throw new errors.BadRequestError(`duplicate field names are not allowed`) - } - - const extraFields = _.difference(fieldNames, allowedFieldNames) - if (extraFields.length > 0) { - throw new errors.BadRequestError( - `${_.toString(extraFields)} is/are not allowed, allowed field names: ${JSON.stringify( - allowedFieldNames, - null, - 4 - )}` - ) - } - } - - let session - const getSession = () => { - if (!session) { - session = helper.createDBSession() + for (let i = 0; i < fieldNames.length; i += 1) { + if (!_.includes(allowedFieldNames, fieldNames[i])) { + throw new errors.BadRequestError( + `Field name ${fieldNames[i]} is not allowed, allowed field names: ${JSON.stringify( + allowedFieldNames, + null, + 4 + )}` + ) + } + for (let j = i + 1; j < fieldNames.length; j += 1) { + if (fieldNames[i] === fieldNames[j]) { + throw new errors.BadRequestError(`There are duplicate field names: ${fieldNames[i]}`) + } + } } - return session } + const session = helper.createDBSession() const redisClient = await helper.acquireRedisClient() - let groupToReturn - try { - if (!criteria.skipCache) { + const cacheCriteria = await helper.getCacheKey(criteria) + const cacheKey = `group:${groupId}:${cacheCriteria}` + + // logger.debug(redisClient.exists(cacheKey)) + if (!criteria.skipCache && await redisClient.exists(cacheKey)) { + logger.debug('returning from cache') // check for the availibility of the group in cache - groupToReturn = JSON.parse(await redisClient.get(`Group:${groupId}`)) + return JSON.parse(await redisClient.get(cacheKey)) } - if (!criteria.skipCache && groupToReturn) { - logger.debug('group found in cache') - if (!isAdmin) delete groupToReturn.status - - // if the group is private, the user needs to be a member of the group, or an admin - if (groupToReturn.privateGroup && currentUser !== 'M2M' && !helper.hasAdminRole(currentUser)) { - const cachedGroupMembers = JSON.parse(await redisClient.get(`GroupMembers:${groupId}`)) || [] + let group = await helper.ensureExists(session, 'Group', groupId, isAdmin) - if (!_.includes(cachedGroupMembers, currentUser.userId)) { - await helper.ensureGroupMember(getSession(), groupId, currentUser.userId) - cachedGroupMembers.push(currentUser.userId) - await redisClient.set(`GroupMembers:${groupId}`, JSON.stringify(cachedGroupMembers), { EX: config.CACHE_TTL }) - } - } - } else { - logger.debug('group not found in cache, getting from DB') - groupToReturn = await helper.ensureExists(getSession(), 'Group', groupId, isAdmin) - - // set the group in cache only if it is having the `oldId` - if (groupToReturn.oldId && groupToReturn.oldId.length > 0) { - logger.debug('saving group in cache') - await redisClient.set(`Group:${groupId}`, JSON.stringify(groupToReturn), { EX: config.CACHE_TTL }) - } + if (!isAdmin) delete group.status - if (!isAdmin) delete groupToReturn.status - - // if the group is private, the user needs to be a member of the group, or an admin - if (groupToReturn.privateGroup && currentUser !== 'M2M' && !helper.hasAdminRole(currentUser)) { - await helper.ensureGroupMember(getSession(), groupToReturn.id, currentUser.userId) - await redisClient.set(`GroupMembers:${groupId}`, JSON.stringify([currentUser.userId]), { EX: config.CACHE_TTL }) - } + // if the group is private, the user needs to be a member of the group, or an admin + if (group.privateGroup && currentUser !== 'M2M' && !helper.hasAdminRole(currentUser)) { + await helper.ensureGroupMember(session, group.id, currentUser.userId) } // get parent or sub groups using breadth first search algorithm, @@ -480,8 +446,11 @@ async function getGroup(currentUser, groupId, criteria) { const pending = [] const expanded = [] if (criteria.includeSubGroups || criteria.includeParentGroup || criteria.flattenGroupIdTree) { - pending.push(groupToReturn) + pending.push(group) + + let flattenGroupIdTree = [] while (pending.length > 0) { + const groupIdTree = [] const groupToExpand = pending.shift() const found = _.find(expanded, (g) => g.id === groupToExpand.id) if (found) { @@ -491,57 +460,50 @@ async function getGroup(currentUser, groupId, criteria) { continue } expanded.push(groupToExpand) - if ((criteria.includeSubGroups && !groupToReturn.subGroups) || (criteria.flattenGroupIdTree && !groupToReturn.flattenGroupIdTree)) { - const flattenGroupIdTree = [] - + if (criteria.includeSubGroups) { // find child groups - groupToExpand.subGroups = await helper.getChildGroups(getSession(), groupToExpand.id) + groupToExpand.subGroups = await helper.getChildGroups(session, groupToExpand.id) + _.forEach(groupToExpand.subGroups, (g) => groupIdTree.push(g.id)) + groupToExpand.flattenGroupIdTree = groupIdTree // add child groups to pending if needed if (!criteria.oneLevel) { - _.forEach(groupToExpand.subGroups, (g) => { - pending.push(g) - flattenGroupIdTree.push(g.id) - }) - - groupToReturn.flattenGroupIdTree = flattenGroupIdTree - - // set the group in cache only if it is having the `oldId` - if (groupToReturn.oldId && groupToReturn.oldId.length > 0) - await redisClient.set(`Group:${groupId}`, JSON.stringify(groupToReturn), { EX: config.CACHE_TTL }) + _.forEach(groupToExpand.subGroups, (g) => pending.push(g)) } - } - else if (criteria.includeParentGroup && !groupToReturn.parentGroups) { + } else { // find parent groups - groupToExpand.parentGroups = await helper.getParentGroups(getSession(), groupToExpand.id) + groupToExpand.parentGroups = await helper.getParentGroups(groupToExpand.id) // add parent groups to pending if needed if (!criteria.oneLevel) { _.forEach(groupToExpand.parentGroups, (g) => pending.push(g)) } } + + flattenGroupIdTree = flattenGroupIdTree.concat(groupIdTree) } + group.flattenGroupIdTree = flattenGroupIdTree } - if (fieldNames) { fieldNames.push('subGroups') fieldNames.push('parentGroups') - - groupToReturn = _.pick(groupToReturn, fieldNames) + + group = _.pick(group, fieldNames) } - if (!criteria.includeSubGroups) delete groupToReturn.subGroups - if (!criteria.includeParentGroup) delete groupToReturn.parentGroups - if (!criteria.flattenGroupIdTree) delete groupToReturn.flattenGroupIdTree + // set the value in redis with UUID + await redisClient.set(`group:${group.id}:${cacheCriteria}`, JSON.stringify(group), { EX: config.CACHE_TTL }) + + // set the value in redis with OldID + if (group.oldId) + await redisClient.set(`group:${group.oldId}:${cacheCriteria}`, JSON.stringify(group), { EX: config.CACHE_TTL }) - return groupToReturn + return group } catch (error) { logger.error(error) throw error } finally { logger.debug('Session Close') - if (session) { - await session.close() - } + await session.close() } } @@ -567,7 +529,6 @@ getGroup.schema = { async function deleteGroup(groupId, isAdmin) { const session = helper.createDBSession() const tx = session.beginTransaction() - const redisClient = await helper.acquireRedisClient() try { logger.debug(`Delete Group - ${groupId}`) const group = await helper.ensureExists(tx, 'Group', groupId, isAdmin) @@ -579,8 +540,7 @@ async function deleteGroup(groupId, isAdmin) { await helper.postBusEvent(config.KAFKA_GROUP_DELETE_TOPIC, kafkaPayload) await tx.commit() - // delete the cache - await Promise.all([redisClient.del(`Group:${group.id}`), redisClient.del(`GroupMembers:${group.id}`)]) + groupsToDelete.forEach(async group => await helper.invalidateCache(group)) return group } catch (error) { From 938f15d86a97413d3292d98a18d82ddd7af1ce44 Mon Sep 17 00:00:00 2001 From: bountyCoder Date: Wed, 15 Mar 2023 19:41:46 +0100 Subject: [PATCH 17/33] allowed tgadmin to create groups --- app-constants.js | 3 ++- src/routes.js | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/app-constants.js b/app-constants.js index c770831..6cca7f0 100644 --- a/app-constants.js +++ b/app-constants.js @@ -3,7 +3,8 @@ */ const UserRoles = { Admin: 'Administrator', - User: 'Topcoder User' + User: 'Topcoder User', + TGAdmin: 'tgadmin' } const MembershipTypes = { diff --git a/src/routes.js b/src/routes.js index 6feb464..2c6b0c4 100644 --- a/src/routes.js +++ b/src/routes.js @@ -17,7 +17,7 @@ module.exports = { controller: 'GroupController', method: 'createGroup', auth: 'jwt', - access: [constants.UserRoles.Admin], + access: [constants.UserRoles.Admin, constants.UserRoles.TGAdmin], scopes: ['write:groups', 'all:groups'] } }, From 4dc9b559beb3e9ff90e42c1ff6b0717b42efe241 Mon Sep 17 00:00:00 2001 From: bountyCoder Date: Wed, 15 Mar 2023 20:59:40 +0100 Subject: [PATCH 18/33] allowed tgadmin to add group role --- src/routes.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes.js b/src/routes.js index 2c6b0c4..14524b8 100644 --- a/src/routes.js +++ b/src/routes.js @@ -170,7 +170,7 @@ module.exports = { controller: 'GroupRoleController', method: 'addGroupRole', auth: 'jwt', - access: [constants.UserRoles.Admin], + access: [constants.UserRoles.Admin, constants.UserRoles.TGAdmin], scopes: ['write:groups', 'all:groups'] }, delete: { From 0c818755ae12ae44666d4cbfa5e43144fa4f8229 Mon Sep 17 00:00:00 2001 From: bountyCoder Date: Thu, 16 Mar 2023 17:09:38 +0000 Subject: [PATCH 19/33] fixed the group-role issue --- src/routes.js | 2 +- src/services/GroupService.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/routes.js b/src/routes.js index 2c6b0c4..14524b8 100644 --- a/src/routes.js +++ b/src/routes.js @@ -170,7 +170,7 @@ module.exports = { controller: 'GroupRoleController', method: 'addGroupRole', auth: 'jwt', - access: [constants.UserRoles.Admin], + access: [constants.UserRoles.Admin, constants.UserRoles.TGAdmin], scopes: ['write:groups', 'all:groups'] }, delete: { diff --git a/src/services/GroupService.js b/src/services/GroupService.js index 42c7e75..e560548 100644 --- a/src/services/GroupService.js +++ b/src/services/GroupService.js @@ -460,7 +460,7 @@ async function getGroup (currentUser, groupId, criteria) { continue } expanded.push(groupToExpand) - if (criteria.includeSubGroups) { + if (criteria.includeSubGroups || criteria.flattenGroupIdTree) { // find child groups groupToExpand.subGroups = await helper.getChildGroups(session, groupToExpand.id) _.forEach(groupToExpand.subGroups, (g) => groupIdTree.push(g.id)) From 6e3e3c76858f068a0572eab3204e10bb909972d4 Mon Sep 17 00:00:00 2001 From: bountyCoder Date: Thu, 16 Mar 2023 17:37:55 +0000 Subject: [PATCH 20/33] fixed the error --- src/common/helper.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/helper.js b/src/common/helper.js index e0d9548..6adb42d 100644 --- a/src/common/helper.js +++ b/src/common/helper.js @@ -414,7 +414,7 @@ async function getCacheKey(criteria) { } async function invalidateCache(group) { - const redisClient = await helper.acquireRedisClient() + const redisClient = await acquireRedisClient() const parentGroups = await getParentGroups(group.id) parentGroups.forEach(async g => { await redisClient.del(`group:${g.id}:*`) From fcbcb26406adea5225cf221a6878d3bd598b5c7a Mon Sep 17 00:00:00 2001 From: bountyCoder Date: Thu, 16 Mar 2023 19:08:22 +0000 Subject: [PATCH 21/33] fixed the cache refactoring issues --- src/common/helper.js | 22 ++++++++++++++++------ src/services/GroupService.js | 16 ++++++++-------- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/src/common/helper.js b/src/common/helper.js index 6adb42d..5ea9c52 100644 --- a/src/common/helper.js +++ b/src/common/helper.js @@ -410,19 +410,29 @@ async function getCacheKey(criteria) { Object.keys(criteria).forEach(c => { if (c != 'skipCache') criteriaStr = criteriaStr + c + '=' + criteria[c] + ',' }) - return criteriaStr.replace(/,\s*$/, "") + + return criteriaStr.replace(/,\s*$/, "").replace(/,/g, '|').replace(/=/g, '-') } async function invalidateCache(group) { - const redisClient = await acquireRedisClient() const parentGroups = await getParentGroups(group.id) parentGroups.forEach(async g => { - await redisClient.del(`group:${g.id}:*`) - await redisClient.del(`group:${g.oldId}:*`) + await deleteKeys(`group:${g.id}:*`) + await deleteKeys(`group:${g.oldId}:*`) }) - await redisClient.del(`group:${group.id}:*`) - await redisClient.del(`group:${group.oldId}:*`) + await deleteKeys(`group:${group.id}:*`) + await deleteKeys(`group:${group.oldId}:*`) +} + +async function deleteKeys(key) { + const redisClient = await acquireRedisClient() + const keys = await redisClient.keys(key) + + keys.forEach(async (key) => { + const test = await redisClient.del(key) + logger.debug(test) + }) } module.exports = { diff --git a/src/services/GroupService.js b/src/services/GroupService.js index e560548..4d9acf8 100644 --- a/src/services/GroupService.js +++ b/src/services/GroupService.js @@ -317,18 +317,17 @@ async function patchGroup(currentUser, groupId, data) { groupData.updatedBy = currentUser === 'M2M' ? '00000000' : currentUser.userId groupData.oldId = data.oldId ? data.oldId : '' - await tx.run( + const patchRes = await tx.run( 'MATCH (g:Group {id: {id}}) SET g.updatedAt={updatedAt}, g.updatedBy={updatedBy}, g.oldId={oldId} RETURN g', groupData ) await tx.commit() - const updatedGroup = await getGroup(currentUser, groupId, { includeSubGroups: true, flattenGroupIdTree: true, skipCache: true }) - + const patchedGroup = patchRes.records[0].get(0).properties // update the cache - await helper.invalidateCache(updateGroup) + await helper.invalidateCache(patchedGroup) - return updatedGroup + return patchedGroup } catch (error) { logger.error(error) logger.debug('Transaction Rollback') @@ -491,6 +490,7 @@ async function getGroup (currentUser, groupId, criteria) { } // set the value in redis with UUID + logger.debug(`adding group:${group.id}:${cacheCriteria} key in cache`) await redisClient.set(`group:${group.id}:${cacheCriteria}`, JSON.stringify(group), { EX: config.CACHE_TTL }) // set the value in redis with OldID @@ -532,7 +532,9 @@ async function deleteGroup(groupId, isAdmin) { try { logger.debug(`Delete Group - ${groupId}`) const group = await helper.ensureExists(tx, 'Group', groupId, isAdmin) - + + await helper.invalidateCache(group) + const groupsToDelete = await helper.deleteGroup(tx, group) const kafkaPayload = {} @@ -540,8 +542,6 @@ async function deleteGroup(groupId, isAdmin) { await helper.postBusEvent(config.KAFKA_GROUP_DELETE_TOPIC, kafkaPayload) await tx.commit() - groupsToDelete.forEach(async group => await helper.invalidateCache(group)) - return group } catch (error) { logger.error(error) From f02d6379ccce893ec4db17f8db6157e3384dde8a Mon Sep 17 00:00:00 2001 From: bountyCoder Date: Thu, 16 Mar 2023 19:13:46 +0000 Subject: [PATCH 22/33] removed extra log lines --- src/common/helper.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/common/helper.js b/src/common/helper.js index 5ea9c52..9a02849 100644 --- a/src/common/helper.js +++ b/src/common/helper.js @@ -430,8 +430,7 @@ async function deleteKeys(key) { const keys = await redisClient.keys(key) keys.forEach(async (key) => { - const test = await redisClient.del(key) - logger.debug(test) + await redisClient.del(key) }) } From 2553c3bef367f1a6bc87d9808ab1c2849cec3f9c Mon Sep 17 00:00:00 2001 From: bountyCoder Date: Wed, 29 Mar 2023 19:42:09 +0000 Subject: [PATCH 23/33] fix the issue of group update without desc --- src/services/GroupService.js | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/services/GroupService.js b/src/services/GroupService.js index 4d9acf8..142fe91 100644 --- a/src/services/GroupService.js +++ b/src/services/GroupService.js @@ -234,21 +234,21 @@ async function updateGroup(currentUser, groupId, data) { const tx = session.beginTransaction() try { logger.debug(`Update Group - user - ${currentUser} , data - ${JSON.stringify(data)}`) - const group = await helper.ensureExists( - tx, - 'Group', - groupId, - currentUser === 'M2M' || helper.hasAdminRole(currentUser) - ) + const group = await helper.ensureExists(tx, 'Group', groupId, currentUser === 'M2M' || helper.hasAdminRole(currentUser)) - const groupData = data + const groupData = {} groupData.id = groupId + groupData.name = data.name ? data.name : group.name + groupData.description = data.description ? data.description : group.description + groupData.privateGroup = data.privateGroup ? data.privateGroup : group.privateGroup + groupData.selfRegister = data.selfRegister ? data.selfRegister : group.selfRegister + groupData.status = data.status ? data.status : group.status groupData.updatedAt = new Date().toISOString() groupData.updatedBy = currentUser === 'M2M' ? '00000000' : currentUser.userId - groupData.domain = data.domain ? data.domain : '' - groupData.ssoId = data.ssoId ? data.ssoId : '' - groupData.organizationId = data.organizationId ? data.organizationId : '' - groupData.oldId = data.oldId ? data.oldId : '' + groupData.domain = data.domain ? data.domain : group.domain + groupData.ssoId = data.ssoId ? data.ssoId : group.ssoId + groupData.organizationId = data.organizationId ? data.organizationId : group.organizationId + groupData.oldId = data.oldId ? data.oldId : group.oldId let updateRes if (groupData.status) { From 72224f9e8476cedbca9b43ec78ba276721bc1ee1 Mon Sep 17 00:00:00 2001 From: bountyCoder Date: Wed, 5 Apr 2023 17:34:26 +0000 Subject: [PATCH 24/33] default fetching the subgroups for oneLevel only --- src/services/GroupService.js | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/services/GroupService.js b/src/services/GroupService.js index 142fe91..7410aa2 100644 --- a/src/services/GroupService.js +++ b/src/services/GroupService.js @@ -165,7 +165,7 @@ searchGroups.schema = { privateGroup: Joi.boolean(), includeSubGroups: Joi.boolean().default(false), includeParentGroup: Joi.boolean().default(false), - oneLevel: Joi.boolean(), + oneLevel: Joi.boolean().default(true), status: Joi.string() .valid([constants.GroupStatus.Active, constants.GroupStatus.InActive]) .default(constants.GroupStatus.Active) @@ -368,14 +368,6 @@ async function getGroup (currentUser, groupId, criteria) { throw new errors.BadRequestError('includeSubGroups and includeParentGroup can not be both true') } - if (_.isNil(criteria.oneLevel)) { - if (criteria.includeSubGroups) { - criteria.oneLevel = false - } else if (criteria.includeParentGroup) { - criteria.oneLevel = true - } - } - let fieldNames = null if (criteria.fields) { fieldNames = criteria.fields.split(',') @@ -515,7 +507,7 @@ getGroup.schema = { includeParentGroup: Joi.boolean().default(false), flattenGroupIdTree: Joi.boolean().default(false), skipCache: Joi.boolean().default(false), - oneLevel: Joi.boolean(), + oneLevel: Joi.boolean().default(true), fields: Joi.string() }) } From 65a39a6d0365b63d4bf2795162bda947b532bdff Mon Sep 17 00:00:00 2001 From: bountyCoder Date: Mon, 17 Apr 2023 18:22:08 +0000 Subject: [PATCH 25/33] updated for the group delete functionality --- config/default.js | 3 ++ package-lock.json | 51 +++++++++++++++++++++++------- package.json | 1 + src/common/helper.js | 12 ++++++- src/controllers/GroupController.js | 2 +- src/services/GroupService.js | 26 +++++++++++++-- 6 files changed, 79 insertions(+), 16 deletions(-) diff --git a/config/default.js b/config/default.js index ca39510..6a5a829 100644 --- a/config/default.js +++ b/config/default.js @@ -28,6 +28,9 @@ module.exports = { BUSAPI_URL: process.env.BUSAPI_URL || 'https://api.topcoder-dev.com/v5', KAFKA_ERROR_TOPIC: process.env.KAFKA_ERROR_TOPIC || 'common.error.reporting', + // challenge api + CHALLENGE_API: process.env.CHALLENGE_API || 'https://api.topcoder-dev.com/v5/challenges/', + // health check timeout in milliseconds HEALTH_CHECK_TIMEOUT: process.env.HEALTH_CHECK_TIMEOUT || 60000, diff --git a/package-lock.json b/package-lock.json index 8c4b729..92a9c55 100644 --- a/package-lock.json +++ b/package-lock.json @@ -537,11 +537,25 @@ "integrity": "sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==" }, "axios": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.12.0.tgz", - "integrity": "sha512-FyH6bSfRAKChMa6yvHVFcnaBj6zcbKFCZMvNsG+q0r+n2XplEIhxu6JPq73I6wL196aAzUxUYktcayWKAlbiPQ==", + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.3.5.tgz", + "integrity": "sha512-glL/PvG/E+xCWwV8S6nCHcrfg1exGx7vxyUIivIA1iL7BIh6bePylCfVHwp6k13ao7SATxB6imau2kqY+I67kw==", "requires": { - "follow-redirects": "0.0.7" + "follow-redirects": "^1.15.0", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + }, + "dependencies": { + "form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + } + } } }, "babel-runtime": { @@ -1630,13 +1644,9 @@ "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==" }, "follow-redirects": { - "version": "0.0.7", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-0.0.7.tgz", - "integrity": "sha512-RxpX808lAA4IZ2cNqzRedcsPfVuo2AJEL8mmGvGeN0KGLJWZf5fidmUkcB0DWUCrmLD+GAQ0J2WOBORw8BS/Uw==", - "requires": { - "debug": "^2.2.0", - "stream-consume": "^0.1.0" - } + "version": "1.15.2", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", + "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==" }, "forever-agent": { "version": "0.6.1", @@ -3694,6 +3704,15 @@ "tc-core-library-js": "github:appirio-tech/tc-core-library-js#v2.6" }, "dependencies": { + "follow-redirects": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-0.0.7.tgz", + "integrity": "sha512-RxpX808lAA4IZ2cNqzRedcsPfVuo2AJEL8mmGvGeN0KGLJWZf5fidmUkcB0DWUCrmLD+GAQ0J2WOBORw8BS/Uw==", + "requires": { + "debug": "^2.2.0", + "stream-consume": "^0.1.0" + } + }, "hoek": { "version": "5.0.4", "resolved": "https://registry.npmjs.org/hoek/-/hoek-5.0.4.tgz", @@ -3722,6 +3741,16 @@ "lodash": "^4.17.10", "millisecond": "^0.1.2", "request": "^2.88.0" + }, + "dependencies": { + "axios": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.12.0.tgz", + "integrity": "sha512-FyH6bSfRAKChMa6yvHVFcnaBj6zcbKFCZMvNsG+q0r+n2XplEIhxu6JPq73I6wL196aAzUxUYktcayWKAlbiPQ==", + "requires": { + "follow-redirects": "0.0.7" + } + } } } } diff --git a/package.json b/package.json index dce6eda..ac6b28b 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "standard": "^17.0.0" }, "dependencies": { + "axios": "^1.3.5", "bluebird": "^3.7.2", "body-parser": "^1.19.0", "config": "^3.2.4", diff --git a/src/common/helper.js b/src/common/helper.js index 9a02849..4c01118 100644 --- a/src/common/helper.js +++ b/src/common/helper.js @@ -7,6 +7,8 @@ const busApi = require('tc-bus-api-wrapper') const config = require('config') const neo4j = require('neo4j-driver') const querystring = require('querystring') +const m2mAuth = require('tc-core-library-js').auth +const m2m = m2mAuth.m2m(_.pick(config, ['AUTH0_URL', 'AUTH0_AUDIENCE', 'TOKEN_CACHE_TIME', 'AUTH0_PROXY_SERVER_URL'])) const redis = require('redis') const uuid = require('uuid') const validate = require('uuid-validate') @@ -434,6 +436,13 @@ async function deleteKeys(key) { }) } +/* Function to get M2M token + * @returns m2m token + */ +async function getM2Mtoken() { + return m2m.getMachineToken(config.AUTH0_CLIENT_ID, config.AUTH0_CLIENT_SECRET) +} + module.exports = { wrapExpress, autoWrapExpress, @@ -452,5 +461,6 @@ module.exports = { deleteGroup, acquireRedisClient, getCacheKey, - invalidateCache + invalidateCache, + getM2Mtoken } diff --git a/src/controllers/GroupController.js b/src/controllers/GroupController.js index b2c5485..0fcdd49 100644 --- a/src/controllers/GroupController.js +++ b/src/controllers/GroupController.js @@ -74,7 +74,7 @@ async function deleteGroup(req, res) { req.params.groupId, req.authUser.isMachine || helper.hasAdminRole(req.authUser) ) - res.send(result) + result.statusCode ? res.status(result.statusCode).send(result.data) : res.send(result) } /** diff --git a/src/services/GroupService.js b/src/services/GroupService.js index 7410aa2..52ba086 100644 --- a/src/services/GroupService.js +++ b/src/services/GroupService.js @@ -2,6 +2,7 @@ * This service provides operations of groups */ const _ = require('lodash') +const axios = require('axios') const config = require('config') const Joi = require('joi') const helper = require('../common/helper') @@ -524,13 +525,32 @@ async function deleteGroup(groupId, isAdmin) { try { logger.debug(`Delete Group - ${groupId}`) const group = await helper.ensureExists(tx, 'Group', groupId, isAdmin) + + //check if group is associated with challenges or not; if yes, don't delete the group else delete + const token = await helper.getM2Mtoken() + const challengeFilterURL = config.CHALLENGE_API + `?groups=["${groupId}"]` + const challenges = await axios.get(challengeFilterURL, { + headers: { + 'User-Agent': 'Request-Promise', + 'Content-Type': 'application/json', + Authorization: `Bearer ${token}` + } + }) + + if (challenges.data.length > 0) { + return { + statusCode: 406, + data: { + message: `group ${groupId} is associated with challenges and can not be deleted` + } + } + } await helper.invalidateCache(group) - const groupsToDelete = await helper.deleteGroup(tx, group) - + const deletedGroups = await helper.deleteGroup(tx, group) const kafkaPayload = {} - kafkaPayload.groups = groupsToDelete + kafkaPayload.groups = deletedGroups await helper.postBusEvent(config.KAFKA_GROUP_DELETE_TOPIC, kafkaPayload) await tx.commit() From b4a10e641b178f1762ead47469eff3e065e09fb8 Mon Sep 17 00:00:00 2001 From: Marios Kranitsas Date: Wed, 17 May 2023 16:17:57 +0300 Subject: [PATCH 26/33] Fix case issue --- src/common/helper.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/helper.js b/src/common/helper.js index 4c01118..daf439f 100644 --- a/src/common/helper.js +++ b/src/common/helper.js @@ -327,7 +327,7 @@ async function postBusEvent(topic, payload) { async function createGroup(tx, data, currentUser) { // check whether group name is already used - const nameCheckRes = await tx.run('MATCH (g:Group {name: {name}}) RETURN g LIMIT 1', { + const nameCheckRes = await tx.run('MATCH (g:Group) WHERE tolower(g.name) = tolower($name) RETURN g LIMIT 1', { name: data.name }) if (nameCheckRes.records.length > 0) { From a48978804f8defd5d740d279a068f3b32b7a993e Mon Sep 17 00:00:00 2001 From: Marios Kranitsas Date: Wed, 17 May 2023 17:45:16 +0300 Subject: [PATCH 27/33] Add validation on the update method --- src/services/GroupService.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/services/GroupService.js b/src/services/GroupService.js index 52ba086..5451705 100644 --- a/src/services/GroupService.js +++ b/src/services/GroupService.js @@ -237,6 +237,13 @@ async function updateGroup(currentUser, groupId, data) { logger.debug(`Update Group - user - ${currentUser} , data - ${JSON.stringify(data)}`) const group = await helper.ensureExists(tx, 'Group', groupId, currentUser === 'M2M' || helper.hasAdminRole(currentUser)) + const nameCheckRes = await tx.run('MATCH (g:Group) WHERE tolower(g.name) = tolower($name) RETURN g LIMIT 1', { + name: data.name + }) + if (nameCheckRes.records.length > 0) { + throw new errors.ConflictError(`The group name ${data.name} is already used`) + } + const groupData = {} groupData.id = groupId groupData.name = data.name ? data.name : group.name From d1cd6d55952d977cd9db5a304e22d5a75d605b48 Mon Sep 17 00:00:00 2001 From: Gunasekar-K Date: Wed, 24 May 2023 12:02:21 +0530 Subject: [PATCH 28/33] read-only-root-file-system-fix --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8f9916b..c80e723 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,7 +15,7 @@ install_dependency: &install_dependency install_deploysuite: &install_deploysuite name: Installation of install_deploysuite. command: | - git clone --branch v1.4.14 https://github.com/topcoder-platform/tc-deploy-scripts ../buildscript + git clone --branch v1.4.15 https://github.com/topcoder-platform/tc-deploy-scripts ../buildscript cp ./../buildscript/master_deploy.sh . cp ./../buildscript/buildenv.sh . cp ./../buildscript/awsconfiguration.sh . From dcc955daa5966b5dfef66c82b775151cc5b5d38f Mon Sep 17 00:00:00 2001 From: marioskranitsas <82266709+marioskranitsas@users.noreply.github.com> Date: Mon, 29 May 2023 10:47:21 +0300 Subject: [PATCH 29/33] Update README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 8a9c96c..e9eff39 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,6 @@ - Heroku CLI - Heroku account - ## Configuration Configuration for the application is at `config/default.js` and `config/production.js`. From 3a4f0992c88310a7d82a0ad641eac1ab567ef112 Mon Sep 17 00:00:00 2001 From: Marios Kranitsas Date: Tue, 1 Aug 2023 14:24:49 +0300 Subject: [PATCH 30/33] Fix PLAT-3280 --- src/services/GroupService.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/services/GroupService.js b/src/services/GroupService.js index 5451705..5f55092 100644 --- a/src/services/GroupService.js +++ b/src/services/GroupService.js @@ -185,6 +185,8 @@ async function createGroup(currentUser, data) { try { logger.debug(`Create Group - user - ${currentUser} , data - ${JSON.stringify(data)}`) + data.name = data.name.trim() + const group = await helper.createGroup(tx, data, currentUser) logger.debug(`Group = ${JSON.stringify(group)}`) @@ -238,7 +240,7 @@ async function updateGroup(currentUser, groupId, data) { const group = await helper.ensureExists(tx, 'Group', groupId, currentUser === 'M2M' || helper.hasAdminRole(currentUser)) const nameCheckRes = await tx.run('MATCH (g:Group) WHERE tolower(g.name) = tolower($name) RETURN g LIMIT 1', { - name: data.name + name: data.name.trim() }) if (nameCheckRes.records.length > 0) { throw new errors.ConflictError(`The group name ${data.name} is already used`) From 08831dc38fa9afe81c12fd7f36c72f9866d3cca7 Mon Sep 17 00:00:00 2001 From: Marios Kranitsas Date: Tue, 1 Aug 2023 15:41:14 +0300 Subject: [PATCH 31/33] Add joi validation --- src/services/GroupService.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/services/GroupService.js b/src/services/GroupService.js index 5f55092..c2fd5a1 100644 --- a/src/services/GroupService.js +++ b/src/services/GroupService.js @@ -185,8 +185,6 @@ async function createGroup(currentUser, data) { try { logger.debug(`Create Group - user - ${currentUser} , data - ${JSON.stringify(data)}`) - data.name = data.name.trim() - const group = await helper.createGroup(tx, data, currentUser) logger.debug(`Group = ${JSON.stringify(group)}`) @@ -211,7 +209,7 @@ createGroup.schema = { currentUser: Joi.any(), data: Joi.object() .keys({ - name: Joi.string().min(3).max(150).required(), + name: Joi.string().trim().min(3).max(150).required(), description: Joi.string().min(3).max(2048), privateGroup: Joi.boolean().required(), selfRegister: Joi.boolean().required(), @@ -240,7 +238,7 @@ async function updateGroup(currentUser, groupId, data) { const group = await helper.ensureExists(tx, 'Group', groupId, currentUser === 'M2M' || helper.hasAdminRole(currentUser)) const nameCheckRes = await tx.run('MATCH (g:Group) WHERE tolower(g.name) = tolower($name) RETURN g LIMIT 1', { - name: data.name.trim() + name: data.name }) if (nameCheckRes.records.length > 0) { throw new errors.ConflictError(`The group name ${data.name} is already used`) From 418fc24af32cc10d006323ebc67fb48e7c41360e Mon Sep 17 00:00:00 2001 From: Gunasekar-K Date: Thu, 19 Oct 2023 12:36:49 +0530 Subject: [PATCH 32/33] Update Dockerfile --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index bb91338..25b0bed 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -10,4 +10,4 @@ WORKDIR /groups-api # Install the dependencies from package.json RUN npm install -CMD npm start +CMD node app.js From 38818e155ac61d29be392948fb92eff60710f269 Mon Sep 17 00:00:00 2001 From: Gunasekar-K Date: Thu, 20 Feb 2025 19:57:52 +0530 Subject: [PATCH 33/33] Update config.yml CORE-1115 [skip ci] --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c80e723..b6ad0d5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -43,7 +43,7 @@ builddeploy_steps: &builddeploy_steps source awsenvconf ./buildenv.sh -e $DEPLOY_ENV -b ${DEPLOY_ENV}-groups-api-deployvar source buildenvvar - ./master_deploy.sh -d ECS -e $DEPLOY_ENV -t latest -s ${APP_ENV}-global-appvar,${DEPLOY_ENV}-groups-api-appvar -i groups-api + ./master_deploy.sh -d ECS -e $DEPLOY_ENV -t latest -s ${APP_ENV}-global-appvar,${DEPLOY_ENV}-groups-api-appvar -i groups-api -p FARGATE #./master_deploy.sh -d ECS -e $DEPLOY_ENV -t latest -s groups-api -i groups-api jobs: