From 211f395cb98aa283e8e7eed98976ed14438328a4 Mon Sep 17 00:00:00 2001 From: Mithun Kamath Date: Tue, 20 Oct 2020 12:25:34 +0530 Subject: [PATCH 1/5] Merge winning submission from enrich contest --- README.md | 3 +- config/default.js | 4 +- package-lock.json | 317 +++++++++++++++++++++++---------------- package.json | 4 +- scripts/constants.js | 5 + scripts/db/dumpDbToEs.js | 13 +- scripts/db/genData.js | 15 +- src/common/es-client.js | 13 +- src/common/es-helper.js | 95 ++---------- 9 files changed, 236 insertions(+), 233 deletions(-) diff --git a/README.md b/README.md index ce89626..d4bb982 100755 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ - node 12.x - npm 6.x - docker -- elasticsearch 6.x +- elasticsearch 7.7 ## Configuration @@ -36,7 +36,6 @@ Configuration for the application is at config/default.js and config/production. - UBAHN_DELETE_TOPIC: Kafka topic for delete message - UBAHN_AGGREGATE_TOPIC: Kafka topic that is used to combine all create, update and delete message(s) - ES.HOST: Elasticsearch host -- ES.API_VERSION: Elasticsearch API version - ES.DOCUMENTS: Elasticsearch index, type and id mapping for resources. For `ES.DOCUMENTS` configuration, you will find multiple other configurations below it. Each has default values that you can override using the environment variables diff --git a/config/default.js b/config/default.js index b46f98b..e824552 100755 --- a/config/default.js +++ b/config/default.js @@ -50,8 +50,8 @@ module.exports = { // ElasticSearch ES: { - HOST: process.env.ES_HOST || 'localhost:9200', - API_VERSION: process.env.ES_API_VERSION || '7.4', + HOST: process.env.ES_HOST || 'http://localhost:9200', + ENRICH_USER_PIPELINE_NAME: process.env.ENRICH_USER_PIPELINE_NAME || 'enrich_user', // es mapping: _index, _type, _id DOCUMENTS: { achievementprovider: { diff --git a/package-lock.json b/package-lock.json index 5a98387..2a51a95 100644 --- a/package-lock.json +++ b/package-lock.json @@ -30,6 +30,33 @@ "js-tokens": "^4.0.0" } }, + "@elastic/elasticsearch": { + "version": "7.9.1", + "resolved": "https://registry.npmjs.org/@elastic/elasticsearch/-/elasticsearch-7.9.1.tgz", + "integrity": "sha512-NfPADbm9tRK/4ohpm9+aBtJ8WPKQqQaReyBKT225pi2oKQO1IzRlfM+OPplAvbhoH1efrSj1NKk27L+4BCrzXQ==", + "requires": { + "debug": "^4.1.1", + "decompress-response": "^4.2.0", + "ms": "^2.1.1", + "pump": "^3.0.0", + "secure-json-parse": "^2.1.0" + }, + "dependencies": { + "debug": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } + }, "@hapi/address": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/@hapi/address/-/address-2.1.4.tgz", @@ -78,6 +105,11 @@ "@hapi/hoek": "^8.3.0" } }, + "@tootallnate/once": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz", + "integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==" + }, "@types/body-parser": { "version": "1.19.0", "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.0.tgz", @@ -102,9 +134,9 @@ } }, "@types/express": { - "version": "4.17.7", - "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.7.tgz", - "integrity": "sha512-dCOT5lcmV/uC2J9k0rPafATeeyz+99xTt54ReX11/LObZgfzJqZNcW27zGhYyX+9iSEGXGt5qLPwRSvBZcLvtQ==", + "version": "4.17.8", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.8.tgz", + "integrity": "sha512-wLhcKh3PMlyA2cNAB9sjM1BntnhPMiM0JOBwPBqttjHev2428MLEB4AYVN+d8s2iyCVZac+o41Pflm/ZH5vLXQ==", "requires": { "@types/body-parser": "*", "@types/express-serve-static-core": "*", @@ -122,9 +154,9 @@ } }, "@types/express-serve-static-core": { - "version": "4.17.8", - "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.8.tgz", - "integrity": "sha512-1SJZ+R3Q/7mLkOD9ewCBDYD2k0WyZQtWYqF/2VvoNN2/uhI49J9CDN4OAm+wGMA0DbArA4ef27xl4+JwMtGggw==", + "version": "4.17.13", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.13.tgz", + "integrity": "sha512-RgDi5a4nuzam073lRGKTUIaL3eF2+H7LJvJ8eUnCI0wA6SNjXc44DCmWNiTLs/AZ7QlsFWZiw/gTG3nSQGL0fA==", "requires": { "@types/node": "*", "@types/qs": "*", @@ -140,9 +172,9 @@ } }, "@types/mime": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@types/mime/-/mime-2.0.2.tgz", - "integrity": "sha512-4kPlzbljFcsttWEq6aBW0OZe6BDajAmyvr2xknBG92tejQnvdGtT9+kXSZ580DqpxY9qG2xeQVF9Dq0ymUTo5Q==" + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@types/mime/-/mime-2.0.3.tgz", + "integrity": "sha512-Jus9s4CDbqwocc5pOAnh8ShfrnMcPHuJYzVcSUU7lrh8Ni5HuIqX3oilL86p3dlTrk0LzHRCgA/GQ7uNCw6l2Q==" }, "@types/node": { "version": "12.0.2", @@ -150,9 +182,9 @@ "integrity": "sha512-5tabW/i+9mhrfEOUcLDu2xBPsHJ+X5Orqy9FKpale3SjDA17j5AEpYq5vfy3oAeAHGcvANRCO3NV3d2D6q3NiA==" }, "@types/qs": { - "version": "6.9.3", - "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.3.tgz", - "integrity": "sha512-7s9EQWupR1fTc2pSMtXRQ9w9gLOcrJn+h7HOXw4evxyvVqMi4f+q7d2tnFe3ng3SNHjtK+0EzGMGFUQX4/AQRA==" + "version": "6.9.5", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.5.tgz", + "integrity": "sha512-/JHkVHtx/REVG0VVToGRGH2+23hsYLHdyG+GrvoUGlGAd0ErauXDyvHtRI/7H7mzLm+tBCKA7pfcpkQ1lf58iQ==" }, "@types/range-parser": { "version": "1.2.3", @@ -160,9 +192,9 @@ "integrity": "sha512-ewFXqrQHlFsgc09MK5jP5iR7vumV/BYayNC6PgJO2LPe8vrnNFyjQjSppfEngITi0qvfKtzFvgKymGheFM9UOA==" }, "@types/serve-static": { - "version": "1.13.4", - "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.4.tgz", - "integrity": "sha512-jTDt0o/YbpNwZbQmE/+2e+lfjJEJJR0I3OFaKQKPWkASkCoW3i6fsUnqudSMcNAfbtmADGu8f4MV4q+GqULmug==", + "version": "1.13.5", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.5.tgz", + "integrity": "sha512-6M64P58N+OXjU432WoLLBQxbA0LRGBCRm7aAGQJ+SMC1IMl0dgRVi9EFfoDcS2a7Xogygk/eGN94CfwU9UF7UQ==", "requires": { "@types/express-serve-static-core": "*", "@types/mime": "*" @@ -189,18 +221,34 @@ "integrity": "sha512-HiUX/+K2YpkpJ+SzBffkM/AQ2YE03S0U1kjTLVpoJdhZMOWy8qvXVN9JdLqv2QsaQ6MPYQIuNmwD8zOiYUofLQ==", "dev": true }, - "agentkeepalive": { - "version": "3.5.2", - "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-3.5.2.tgz", - "integrity": "sha512-e0L/HNe6qkQ7H19kTlRRqUibEAwDK5AFk6y3PtMsuut2VAH6+Q4xZml1tNDJD7kSAyqmbG/K08K5WEJYtUrSlQ==", + "agent-base": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.1.tgz", + "integrity": "sha512-01q25QQDwLSsyfhrKbn8yuur+JNw0H+0Y4JiGIKd3z9aYk/w/2kxD/Upc+t2ZBBSUNff50VjPsSW2YxM8QYKVg==", "requires": { - "humanize-ms": "^1.2.1" + "debug": "4" + }, + "dependencies": { + "debug": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } } }, "ajv": { "version": "6.12.2", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.2.tgz", "integrity": "sha512-k+V+hzjm5q/Mr8ef/1Y9goCmlsK4I6Sm74teeyGvFk1XrOsbsKLjEdrvny42CZ+a8sXbk8KWpY/bDwS+FLL2UQ==", + "dev": true, "requires": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -306,6 +354,14 @@ "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" }, + "aws-elasticsearch-connector": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/aws-elasticsearch-connector/-/aws-elasticsearch-connector-9.0.0.tgz", + "integrity": "sha512-O+A9HEa14gOiKTAp6U6Ha1RRJvQjc046wIn9CJ69wc+c1c5CfPE4xl4Av6Zyv6dgzs+RVGxdetjm8RpSlTUmhQ==", + "requires": { + "aws4": "^1.10.0" + } + }, "aws-sdk": { "version": "2.668.0", "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.668.0.tgz", @@ -335,9 +391,9 @@ "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" }, "aws4": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.10.0.tgz", - "integrity": "sha512-3YDiu347mtVtjpyV3u5kVqQLP242c06zwDOgpeRnybmXlYYsLbtTrUBUm8i8srONt+FWobl5aibnU1030PeeuA==" + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.10.1.tgz", + "integrity": "sha512-zg7Hz2k5lI8kb7U32998pRRFin7zJlkfezGJjUc2heaD4Pw2wObakCDVzkKztTm/Ln7eiVvYsjqak0Ed4LkMDA==" }, "axios": { "version": "0.19.2", @@ -665,6 +721,14 @@ "integrity": "sha1-IwdjLUwEOCuN+KMvcLiVBG1SdF8=", "dev": true }, + "decompress-response": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-4.2.1.tgz", + "integrity": "sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==", + "requires": { + "mimic-response": "^2.0.0" + } + }, "deep-is": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", @@ -781,53 +845,6 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, - "elasticsearch": { - "version": "16.7.1", - "resolved": "https://registry.npmjs.org/elasticsearch/-/elasticsearch-16.7.1.tgz", - "integrity": "sha512-PL/BxB03VGbbghJwISYvVcrR9KbSSkuQ7OM//jHJg/End/uC2fvXg4QI7RXLvCGbhBuNQ8dPue7DOOPra73PCw==", - "requires": { - "agentkeepalive": "^3.4.1", - "chalk": "^1.0.0", - "lodash": "^4.17.10" - }, - "dependencies": { - "ansi-regex": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" - }, - "ansi-styles": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", - "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=" - }, - "chalk": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", - "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", - "requires": { - "ansi-styles": "^2.2.1", - "escape-string-regexp": "^1.0.2", - "has-ansi": "^2.0.0", - "strip-ansi": "^3.0.0", - "supports-color": "^2.0.0" - } - }, - "strip-ansi": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", - "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", - "requires": { - "ansi-regex": "^2.0.0" - } - }, - "supports-color": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", - "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=" - } - } - }, "emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", @@ -847,6 +864,14 @@ "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" }, + "end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "requires": { + "once": "^1.4.0" + } + }, "env-variable": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/env-variable/-/env-variable-0.0.6.tgz", @@ -907,7 +932,8 @@ "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", + "dev": true }, "eslint": { "version": "6.8.0", @@ -1516,12 +1542,25 @@ "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" }, "har-validator": { - "version": "5.1.3", - "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", - "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", + "version": "5.1.5", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.5.tgz", + "integrity": "sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==", "requires": { - "ajv": "^6.5.5", + "ajv": "^6.12.3", "har-schema": "^2.0.0" + }, + "dependencies": { + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + } } }, "has": { @@ -1533,21 +1572,6 @@ "function-bind": "^1.1.1" } }, - "has-ansi": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", - "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=", - "requires": { - "ansi-regex": "^2.0.0" - }, - "dependencies": { - "ansi-regex": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" - } - } - }, "has-flag": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", @@ -1571,11 +1595,6 @@ "integrity": "sha512-f/wzC2QaWBs7t9IYqB4T3sR1xviIViXJRJTWBlx2Gf3g0Xi5vI7Yy4koXQ1c9OYDGHN9sBy1DQ2AB8fqZBWhUg==", "dev": true }, - "http-aws-es": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/http-aws-es/-/http-aws-es-6.0.0.tgz", - "integrity": "sha512-g+qp7J110/m4aHrR3iit4akAlnW0UljZ6oTq/rCcbsI8KP9x+95vqUtx49M2XQ2JMpwJio3B6gDYx+E8WDxqiA==" - }, "http-errors": { "version": "1.7.2", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", @@ -1588,6 +1607,31 @@ "toidentifier": "1.0.0" } }, + "http-proxy-agent": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz", + "integrity": "sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg==", + "requires": { + "@tootallnate/once": "1", + "agent-base": "6", + "debug": "4" + }, + "dependencies": { + "debug": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } + }, "http-signature": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", @@ -1598,12 +1642,28 @@ "sshpk": "^1.7.0" } }, - "humanize-ms": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", - "integrity": "sha1-xG4xWaKT9riW2ikxbYtv6Lt5u+0=", + "https-proxy-agent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", + "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==", "requires": { - "ms": "^2.0.0" + "agent-base": "6", + "debug": "4" + }, + "dependencies": { + "debug": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } } }, "iconv-lite": { @@ -1992,13 +2052,15 @@ } }, "jwks-rsa": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/jwks-rsa/-/jwks-rsa-1.8.1.tgz", - "integrity": "sha512-CcE8ypsATHwGmzELwzeFjLzPBXTXTrMmDYbn92LTQwYsZdOedp+ZIuYTofUdrWreu8CKRuXmhk17+6/li2sR6g==", + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/jwks-rsa/-/jwks-rsa-1.10.1.tgz", + "integrity": "sha512-UmjOsATVu7eQr17wbBCS+BSoz5LFtl57PtNXHbHFeT1WKomHykCHtn7c8inWVI7tpnsy6CZ1KOMJTgipFwXPig==", "requires": { "@types/express-jwt": "0.0.42", "axios": "^0.19.2", "debug": "^4.1.0", + "http-proxy-agent": "^4.0.1", + "https-proxy-agent": "^5.0.0", "jsonwebtoken": "^8.5.1", "limiter": "^1.1.5", "lru-memoizer": "^2.1.2", @@ -2006,11 +2068,11 @@ }, "dependencies": { "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", "requires": { - "ms": "^2.1.1" + "ms": "2.1.2" } }, "ms": { @@ -2209,6 +2271,11 @@ "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", "dev": true }, + "mimic-response": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.1.0.tgz", + "integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==" + }, "minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", @@ -2258,9 +2325,9 @@ } }, "nan": { - "version": "2.14.1", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.1.tgz", - "integrity": "sha512-isWHgVjnFjh2x2yuJ/tj3JbwoHu3UC2dX5G/88Cm24yB6YopVgxvBObDY7n5xW6ExmFhJpSEQqFPvq9zaXc8Jw==", + "version": "2.14.2", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.2.tgz", + "integrity": "sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ==", "optional": true }, "natural-compare": { @@ -2683,6 +2750,15 @@ "resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz", "integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==" }, + "pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "requires": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "punycode": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", @@ -2915,6 +2991,11 @@ "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" }, + "secure-json-parse": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.1.0.tgz", + "integrity": "sha512-GckO+MS/wT4UogDyoI/H/S1L0MCcKS1XX/vp48wfmU7Nw4woBmb8mIpu4zPBQjKlRT88/bt9xdoV4111jPpNJA==" + }, "semaphore-async-await": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/semaphore-async-await/-/semaphore-async-await-1.5.1.tgz", @@ -3337,22 +3418,6 @@ "lodash": "^4.17.15", "superagent": "^3.8.3", "tc-core-library-js": "github:appirio-tech/tc-core-library-js#v2.6.4" - }, - "dependencies": { - "tc-core-library-js": { - "version": "github:appirio-tech/tc-core-library-js#df0b36c51cf80918194cbff777214b3c0cf5a151", - "from": "github:appirio-tech/tc-core-library-js#v2.6.4", - "requires": { - "axios": "^0.19.0", - "bunyan": "^1.8.12", - "jsonwebtoken": "^8.5.1", - "jwks-rsa": "^1.6.0", - "lodash": "^4.17.15", - "millisecond": "^0.1.2", - "r7insight_node": "^1.8.4", - "request": "^2.88.0" - } - } } }, "tc-core-library-js": { diff --git a/package.json b/package.json index 4fa45ef..ce8a809 100644 --- a/package.json +++ b/package.json @@ -15,18 +15,18 @@ "url": "" }, "dependencies": { + "@elastic/elasticsearch": "^7.9.1", "@hapi/joi": "^16.1.8", "@hapi/joi-date": "^2.0.1", "amazon-qldb-driver-nodejs": "^0.1.1-preview.2", + "aws-elasticsearch-connector": "^9.0.0", "aws-sdk": "^2.627.0", "axios": "^0.19.2", "body-parser": "^1.19.0", "config": "^3.2.4", "cors": "^2.8.5", - "elasticsearch": "^16.7.1", "express": "^4.17.1", "get-parameter-names": "^0.3.0", - "http-aws-es": "^6.0.0", "ion-js": "^3.1.2", "js-yaml": "^3.13.1", "lodash": "^4.17.19", diff --git a/scripts/constants.js b/scripts/constants.js index 755b650..42acd44 100644 --- a/scripts/constants.js +++ b/scripts/constants.js @@ -8,10 +8,12 @@ const config = require('config') const topResources = { achievementprovider: { index: config.get('ES.DOCUMENTS.achievementprovider.index'), + enrichPolicy: 'achievementprovider-policy', type: config.get('ES.DOCUMENTS.achievementprovider.type') }, attribute: { index: config.get('ES.DOCUMENTS.attribute.index'), + enrichPolicy: 'attribute-policy', type: config.get('ES.DOCUMENTS.attribute.type') }, attributegroup: { @@ -20,14 +22,17 @@ const topResources = { }, organization: { index: config.get('ES.DOCUMENTS.organization.index'), + enrichPolicy: 'organization-policy', type: config.get('ES.DOCUMENTS.organization.type') }, role: { index: config.get('ES.DOCUMENTS.role.index'), + enrichPolicy: 'role-policy', type: config.get('ES.DOCUMENTS.role.type') }, skill: { index: config.get('ES.DOCUMENTS.skill.index'), + enrichPolicy: 'skill-policy', type: config.get('ES.DOCUMENTS.skill.type') }, skillprovider: { diff --git a/scripts/db/dumpDbToEs.js b/scripts/db/dumpDbToEs.js index 17c51cf..0d12e73 100644 --- a/scripts/db/dumpDbToEs.js +++ b/scripts/db/dumpDbToEs.js @@ -1,4 +1,5 @@ const _ = require('lodash') +const config = require('config') const models = require('../../src/models') const logger = require('../../src/common/logger') const { getESClient } = require('../../src/common/es-client') @@ -38,10 +39,13 @@ async function insertIntoES (modelName, body) { body, refresh: 'true' }) + if (topResources[esResourceName].enrichPolicy) { + await client.enrich.executePolicy({ name: topResources[esResourceName].enrichPolicy }) + } } else if (_.includes(_.keys(userResources), esResourceName)) { const userResource = userResources[esResourceName] - const user = await client.getSource({ + const { body: user } = await client.getSource({ index: topResources.user.index, type: topResources.user.type, id: body.userId @@ -73,18 +77,19 @@ async function insertIntoES (modelName, body) { logger.error(`Can't create existing ${esResourceName} with the ${userResource.relateKey}: ${relateId}, userId: ${body.userId}`) } else { user[userResource.propertyName].push(body) - await client.update({ + await client.index({ index: topResources.user.index, type: topResources.user.type, id: body.userId, - body: { doc: user }, + body: user, + pipeline: config.get('ES.ENRICH_USER_PIPELINE_NAME'), refresh: 'true' }) } } else if (_.includes(_.keys(organizationResources), esResourceName)) { const orgResource = organizationResources[esResourceName] - const organization = await client.getSource({ + const { body: organization } = await client.getSource({ index: topResources.organization.index, type: topResources.organization.type, id: body.organizationId diff --git a/scripts/db/genData.js b/scripts/db/genData.js index 2f501a1..e11dca9 100644 --- a/scripts/db/genData.js +++ b/scripts/db/genData.js @@ -1,4 +1,5 @@ const _ = require('lodash') +const config = require('config') const models = require('../../src/models') const logger = require('../../src/common/logger') const { getESClient } = require('../../src/common/es-client') @@ -28,10 +29,13 @@ async function insertIntoES (modelName, body) { body, refresh: 'true' }) + if (topResources[esResourceName].enrichPolicy) { + await client.enrich.executePolicy({ name: topResources[esResourceName].enrichPolicy }) + } } else if (_.includes(_.keys(userResources), esResourceName)) { const userResource = userResources[esResourceName] - const user = await client.getSource({ + const { body: user } = await client.getSource({ index: topResources.user.index, type: topResources.user.type, id: body.userId @@ -63,18 +67,19 @@ async function insertIntoES (modelName, body) { logger.error(`Can't create existing ${esResourceName} with the ${userResource.relateKey}: ${relateId}, userId: ${body.userId}`) } else { user[userResource.propertyName].push(body) - await client.update({ + await client.index({ index: topResources.user.index, type: topResources.user.type, id: body.userId, - body: { doc: user }, - refresh: 'true' + body: user, + pipeline: config.get('ES.ENRICH_USER_PIPELINE_NAME'), + refresh: 'wait_for' }) } } else if (_.includes(_.keys(organizationResources), esResourceName)) { const orgResource = organizationResources[esResourceName] - const organization = await client.getSource({ + const { body: organization } = await client.getSource({ index: topResources.organization.index, type: topResources.organization.type, id: body.organizationId diff --git a/src/common/es-client.js b/src/common/es-client.js index 2ffc3f1..20c1bb7 100644 --- a/src/common/es-client.js +++ b/src/common/es-client.js @@ -1,6 +1,7 @@ const config = require('config') const AWS = require('aws-sdk') -const elasticsearch = require('elasticsearch') +const elasticsearch = require('@elastic/elasticsearch') +const createAwsElasticsearchConnector = require('aws-elasticsearch-connector') AWS.config.region = config.AWS_REGION @@ -16,20 +17,16 @@ function getESClient () { return esClient } const host = config.ES.HOST - const apiVersion = config.ES.API_VERSION - if (!esClient) { // AWS ES configuration is different from other providers if (/.*amazonaws.*/.test(host)) { esClient = new elasticsearch.Client({ - apiVersion, - host, - connectionClass: require('http-aws-es') // eslint-disable-line global-require + ...createAwsElasticsearchConnector(AWS.config), + node: host }) } else { esClient = new elasticsearch.Client({ - apiVersion, - host + node: host }) } } diff --git a/src/common/es-helper.js b/src/common/es-helper.js index 0b395c5..581ce3f 100644 --- a/src/common/es-helper.js +++ b/src/common/es-helper.js @@ -376,68 +376,6 @@ function parseEnrichFilter (params) { return filters } -/** - * Enrich a resource recursively by following enrich path. - * @param resource the resource to enrich - * @param enrichIdProp the id property of child resource in parent object - * @param item the parent object - * @returns {Promise} the promise of enriched parent object - */ -async function enrichResource (resource, enrichIdProp, item) { - const subDoc = DOCUMENTS[resource] - const filterChain = FILTER_CHAIN[resource] - const subResult = await esClient.getSource({ - index: subDoc.index, - type: subDoc.type, - id: item[enrichIdProp] - }) - - if (filterChain.enrichNext) { - const enrichIdProp = filterChain.idField - // return error if any id is missing in enrich path - if (!subResult[enrichIdProp]) { - throw new Error(`The parent ${resource} is missing id value of child resource ${filterChain.enrichNext}`) - } - // enrich next child resource recursively - await enrichResource(filterChain.enrichNext, enrichIdProp, subResult) - } - item[resource] = subResult -} - -/** - * Enrich a user. - * @param user the user object to enrich - * @returns {Promise<*>} the promise of enriched user - */ -async function enrichUser (user) { - for (const subProp of Object.keys(SUB_USER_DOCUMENTS)) { - const subDoc = SUB_USER_DOCUMENTS[subProp] - const subData = user[subDoc.userField] - const filterChain = FILTER_CHAIN[subProp] - if (subData && subData.length > 0) { - // enrich next level sub resources - for (const subItem of subData) { - await enrichResource(filterChain.enrichNext, filterChain.idField, subItem) - } - } - } - return user -} - -/** - * Enrich users. - * @param users list of users from ES search - * @returns {Promise<*>} the enriched users - */ -async function enrichUsers (users) { - const enrichedUsers = [] - for (const user of users) { - const enriched = await enrichUser(user) - enrichedUsers.push(enriched) - } - return enrichedUsers -} - /** * Get a resource by Id from ES. * @param resource the resource to get @@ -494,12 +432,9 @@ async function getFromElasticSearch (resource, ...args) { logger.debug(`ES query for get ${resource}: ${JSON.stringify(esQuery, null, 2)}`) // query ES - const result = await esClient.getSource(esQuery) + const { body: result } = await esClient.getSource(esQuery) - if (params.enrich && resource === 'user') { - const user = await enrichUser(result) - return user - } else if (subUserDoc) { + if (subUserDoc) { // find top sub doc by sub.id const found = result[subUserDoc.userField].find(sub => sub[filterChain.idField] === params[filterChain.idField]) if (found) { @@ -713,7 +648,7 @@ async function searchSkills (keyword, skillProviderIds) { } logger.debug(`ES query for searching skills: ${JSON.stringify(esQuery, null, 2)}`) - const results = await esClient.search(esQuery) + const { body: results } = await esClient.search(esQuery) return results.hits.hits.map(hit => hit._source) } @@ -1059,7 +994,7 @@ async function resolveResFilter (filter, initialRes) { // query ES with filter const esQuery = buildEsQueryFromFilter(filter) - const result = await esClient.search(esQuery) + const { body: result } = await esClient.search(esQuery) const numHits = getTotalCount(result.hits.total) @@ -1288,7 +1223,7 @@ async function searchElasticSearch (resource, ...args) { } logger.debug(`ES query for search ${resource}: ${JSON.stringify(esQuery, null, 2)}`) - const docs = await esClient.search(esQuery) + const { body: docs } = await esClient.search(esQuery) if (docs.hits && getTotalCount(docs.hits.total) === 0) { return { total: docs.hits.total, @@ -1299,10 +1234,7 @@ async function searchElasticSearch (resource, ...args) { } let result = [] - if (resource === 'user' && params.enrich) { - const users = docs.hits.hits.map(hit => hit._source) - result = await enrichUsers(users) - } else if (topUserSubDoc) { + if (topUserSubDoc) { result = docs.hits.hits[0]._source[topUserSubDoc.userField] // for sub-resource query, it returns all sub-resource items in one user, // so needs filtering and also page size @@ -1420,14 +1352,9 @@ async function searchUsers (authUser, filter, params) { logger.debug(`ES query for searching users: ${JSON.stringify(esQuery, null, 2)}`) console.time('mainesquery') - const docs = await esClient.search(esQuery) + const { body: docs } = await esClient.search(esQuery) console.timeEnd('mainesquery') - const users = docs.hits.hits.map(hit => hit._source) - - logger.debug('Enrich users') - console.time('enrichUsers') - const result = await enrichUsers(users) - console.timeEnd('enrichUsers') + const result = docs.hits.hits.map(hit => hit._source) return { total: getTotalCount(docs.hits.total), @@ -1448,7 +1375,7 @@ async function searchSkillsInOrganization ({ organizationId, keyword }) { const esQueryToGetSkillProviders = buildEsQueryToGetSkillProviderIds(organizationId) logger.debug(`ES query to get skill provider ids: ${JSON.stringify(esQueryToGetSkillProviders, null, 2)}`) - const esResultOfQueryToGetSkillProviders = await esClient.search(esQueryToGetSkillProviders) + const { body: esResultOfQueryToGetSkillProviders } = await esClient.search(esQueryToGetSkillProviders) logger.debug(`ES result: ${JSON.stringify(esResultOfQueryToGetSkillProviders, null, 2)}`) const skillProviderIds = _.flatten(esResultOfQueryToGetSkillProviders.hits.hits.map(hit => hit._source.skillProviders == null ? [] : hit._source.skillProviders.map(sp => sp.skillProviderId))) @@ -1474,7 +1401,7 @@ async function searchAttributeValues ({ attributeId, attributeValue }) { const esQuery = buildEsQueryToGetAttributeValues(attributeId, querystring.unescape(attributeValue), 5) logger.debug(`ES query for searching attribute values: ${JSON.stringify(esQuery, null, 2)}`) - const esResult = await esClient.search(esQuery) + const { body: esResult } = await esClient.search(esQuery) logger.debug(`ES Result: ${JSON.stringify(esResult, null, 2)}`) const result = [] const attributes = esResult.aggregations.attributes.ids.buckets @@ -1502,7 +1429,7 @@ async function searchAchievementValues ({ organizationId, keyword }) { const esQuery = buildEsQueryToGetAchievements(organizationId, querystring.unescape(keyword), 5) logger.debug(`ES query for searching achievement values; ${JSON.stringify(esQuery, null, 2)}`) - const esResult = await esClient.search(esQuery) + const { body: esResult } = await esClient.search(esQuery) logger.debug(`ES response ${JSON.stringify(esResult, null, 2)}`) const result = esResult.aggregations.achievements.buckets.map(a => { const achievementName = a.key From 25ea241145d65b8b72c8d53f9cc30b25598f7205 Mon Sep 17 00:00:00 2001 From: Mithun Kamath Date: Wed, 21 Oct 2020 13:49:20 +0530 Subject: [PATCH 2/5] Working data insertion script --- scripts/constants.js | 149 ++++++++++++++++++++++++++++------ scripts/db/genData.js | 183 +++++++++++++++++++++++++++++++++++------- 2 files changed, 277 insertions(+), 55 deletions(-) diff --git a/scripts/constants.js b/scripts/constants.js index 42acd44..1b0a5b3 100644 --- a/scripts/constants.js +++ b/scripts/constants.js @@ -6,42 +6,136 @@ const config = require('config') const topResources = { + skillprovider: { + index: config.get('ES.DOCUMENTS.skillprovider.index'), + type: config.get('ES.DOCUMENTS.skillprovider.type'), + enrich: { + policyName: 'skillprovider-policy', + matchField: 'id', + enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy'] + }, + pipeline: { + id: 'skillprovider-pipeline', + field: 'skillProviderId', + targetField: 'skillprovider', + maxMatches: '1' + } + }, + + role: { + index: config.get('ES.DOCUMENTS.role.index'), + type: config.get('ES.DOCUMENTS.role.type'), + enrich: { + policyName: 'role-policy', + matchField: 'id', + enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy'] + } + }, + achievementprovider: { index: config.get('ES.DOCUMENTS.achievementprovider.index'), - enrichPolicy: 'achievementprovider-policy', - type: config.get('ES.DOCUMENTS.achievementprovider.type') - }, - attribute: { - index: config.get('ES.DOCUMENTS.attribute.index'), - enrichPolicy: 'attribute-policy', - type: config.get('ES.DOCUMENTS.attribute.type') + type: config.get('ES.DOCUMENTS.achievementprovider.type'), + enrich: { + policyName: 'achievementprovider-policy', + matchField: 'id', + enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy'] + } }, + attributegroup: { index: config.get('ES.DOCUMENTS.attributegroup.index'), - type: config.get('ES.DOCUMENTS.attributegroup.type') - }, - organization: { - index: config.get('ES.DOCUMENTS.organization.index'), - enrichPolicy: 'organization-policy', - type: config.get('ES.DOCUMENTS.organization.type') - }, - role: { - index: config.get('ES.DOCUMENTS.role.index'), - enrichPolicy: 'role-policy', - type: config.get('ES.DOCUMENTS.role.type') + type: config.get('ES.DOCUMENTS.attributegroup.type'), + enrich: { + policyName: 'attributegroup-policy', + matchField: 'id', + enrichFields: ['id', 'name', 'organizationId', 'created', 'updated', 'createdBy', 'updatedBy'] + }, + pipeline: { + id: 'attributegroup-pipeline', + field: 'attributeGroupId', + targetField: 'attributegroup', + maxMatches: '1' + } }, + skill: { index: config.get('ES.DOCUMENTS.skill.index'), - enrichPolicy: 'skill-policy', - type: config.get('ES.DOCUMENTS.skill.type') + type: config.get('ES.DOCUMENTS.skill.type'), + enrich: { + policyName: 'skill-policy', + matchField: 'id', + enrichFields: ['id', 'skillProviderId', 'name', 'externalId', 'uri', 'created', 'updated', 'createdBy', 'updatedBy', 'skillprovider'] + }, + ingest: { + pipeline: { + id: 'skillprovider-pipeline' + } + } }, - skillprovider: { - index: config.get('ES.DOCUMENTS.skillprovider.index'), - type: config.get('ES.DOCUMENTS.skillprovider.type') + + attribute: { + index: config.get('ES.DOCUMENTS.attribute.index'), + type: config.get('ES.DOCUMENTS.attribute.type'), + enrich: { + policyName: 'attribute-policy', + matchField: 'id', + enrichFields: ['id', 'name', 'attributeGroupId', 'created', 'updated', 'createdBy', 'updatedBy', 'attributegroup'] + }, + ingest: { + pipeline: { + id: 'attributegroup-pipeline' + } + } }, + + organization: { + index: config.get('ES.DOCUMENTS.organization.index'), + type: config.get('ES.DOCUMENTS.organization.type'), + }, + user: { index: config.get('ES.DOCUMENTS.user.index'), - type: config.get('ES.DOCUMENTS.user.type') + type: config.get('ES.DOCUMENTS.user.type'), + pipeline: { + id: 'user-pipeline', + processors: [ + { + referenceField: config.get('ES.DOCUMENTS.achievement.userField'), + enrichPolicyName: 'achievementprovider-policy', + field: '_ingest._value.achievementsProviderId', + targetField: '_ingest._value.achievementprovider', + maxMatches: '1' + }, + { + referenceField: config.get('ES.DOCUMENTS.externalprofile.userField'), + enrichPolicyName: 'organization-policy', + field: '_ingest._value.organizationId', + targetField: '_ingest._value.organization', + maxMatches: '1' + }, + { + referenceField: config.get('ES.DOCUMENTS.userattribute.userField'), + enrichPolicyName: 'attribute-policy', + field: '_ingest._value.attributeId', + targetField: '_ingest._value.attribute', + maxMatches: '1' + }, + { + referenceField: config.get('ES.DOCUMENTS.userrole.userField'), + enrichPolicyName: 'role-policy', + field: '_ingest._value.roleId', + targetField: '_ingest._value.role', + maxMatches: '1' + }, + { + referenceField: config.get('ES.DOCUMENTS.userskill.userField'), + enrichPolicyName: 'skill-policy', + field: '_ingest._value.skillId', + targetField: '_ingest._value.skill', + maxMatches: '1' + } + ] + } } } @@ -72,7 +166,12 @@ const userResources = { const organizationResources = { organizationskillprovider: { propertyName: config.get('ES.DOCUMENTS.organizationskillprovider.orgField'), - relateKey: 'skillProviderId' + relateKey: 'skillProviderId', + enrich: { + policyName: 'organization-policy', + matchField: 'id', + enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy', 'skillProviders'] + } } } diff --git a/scripts/db/genData.js b/scripts/db/genData.js index e11dca9..b1cfa0e 100644 --- a/scripts/db/genData.js +++ b/scripts/db/genData.js @@ -10,6 +10,26 @@ const { modelToESIndexMapping } = require('../constants') +// Declares the ordering of the resource data insertion, to ensure that enrichment happens correctly +const RESOURCES_IN_ORDER = [ + 'skillprovider', + 'role', + 'achievementprovider', + 'attributegroup', + 'skill', + 'attribute', + 'organization', + 'organizationskillprovider', + 'user', + 'userskill', + 'achievement', + 'userrole', + 'externalprofile', + 'userattribute' +] + +const client = getESClient() + async function insertIntoES (modelName, body) { const esResourceName = modelToESIndexMapping[modelName] @@ -19,19 +39,15 @@ async function insertIntoES (modelName, body) { return } - const client = getESClient() - if (_.includes(_.keys(topResources), esResourceName)) { - await client.create({ + await client.index({ index: topResources[esResourceName].index, type: topResources[esResourceName].type, id: body.id, body, - refresh: 'true' + pipeline: topResources[esResourceName].ingest ? topResources[esResourceName].ingest.pipeline.id : undefined, + refresh: 'wait_for' }) - if (topResources[esResourceName].enrichPolicy) { - await client.enrich.executePolicy({ name: topResources[esResourceName].enrichPolicy }) - } } else if (_.includes(_.keys(userResources), esResourceName)) { const userResource = userResources[esResourceName] @@ -72,7 +88,7 @@ async function insertIntoES (modelName, body) { type: topResources.user.type, id: body.userId, body: user, - pipeline: config.get('ES.ENRICH_USER_PIPELINE_NAME'), + pipeline: topResources.user.pipeline.id, refresh: 'wait_for' }) } @@ -95,12 +111,102 @@ async function insertIntoES (modelName, body) { logger.error(`Can't create existing ${esResourceName} with the ${orgResource.relateKey}: ${relateId}, organizationId: ${body.organizationId}`) } else { organization[orgResource.propertyName].push(body) - await client.update({ + await client.index({ index: topResources.organization.index, type: topResources.organization.type, id: body.organizationId, - body: { doc: organization }, - refresh: 'true' + body: organization, + refresh: 'wait_for' + }) + } + } +} + +/** + * Creates and executes the enrich policy for the provided model + * @param {String} modelName The model name + */ +async function createAndExecuteEnrichPolicy (modelName) { + const esResourceName = modelToESIndexMapping[modelName] + + if (_.includes(_.keys(topResources), esResourceName) && topResources[esResourceName].enrich) { + await client.enrich.putPolicy({ + name: topResources[esResourceName].enrich.policyName, + body: { + match: { + indices: topResources[esResourceName].index, + match_field: topResources[esResourceName].enrich.matchField, + enrich_fields: topResources[esResourceName].enrich.enrichFields + } + } + }) + await client.enrich.executePolicy({ name: topResources[esResourceName].enrich.policyName }) + } else if (_.includes(_.keys(organizationResources), esResourceName)) { + // For organization, execute enrich policy AFTER the sub documents on the org (namely orgskillprovider) is in + // This is because external profile on user is enriched with org, and it needs to have the orgskillprovider details in it + await client.enrich.putPolicy({ + name: organizationResources[esResourceName].enrich.policyName, + body: { + match: { + indices: topResources.organization.index, + match_field: organizationResources[esResourceName].enrich.matchField, + enrich_fields: organizationResources[esResourceName].enrich.enrichFields + } + } + }) + await client.enrich.executePolicy({ name: organizationResources[esResourceName].enrich.policyName }) + } +} + +/** + * Creates the ingest pipeline using the enrich policy + * @param {String} modelName The model name + */ +async function createEnrichProcessor (modelName) { + const esResourceName = modelToESIndexMapping[modelName] + + if (_.includes(_.keys(topResources), esResourceName) && topResources[esResourceName].pipeline) { + if (topResources[esResourceName].pipeline.processors) { + const processors = [] + + for (let i = 0; i < topResources[esResourceName].pipeline.processors.length; i++) { + const ep = topResources[esResourceName].pipeline.processors[i] + processors.push({ + foreach: { + field: ep.referenceField, + ignore_missing: true, + processor: { + enrich: { + policy_name: ep.enrichPolicyName, + ignore_missing: true, + field: ep.field, + target_field: ep.targetField, + max_matches: ep.maxMatches + } + } + } + }) + } + + await client.ingest.putPipeline({ + id: topResources[esResourceName].pipeline.id, + body: { + processors + } + }) + } else { + await client.ingest.putPipeline({ + id: topResources[esResourceName].pipeline.id, + body: { + processors: [{ + enrich: { + policy_name: topResources[esResourceName].enrich.policyName, + field: topResources[esResourceName].pipeline.field, + target_field: topResources[esResourceName].pipeline.targetField, + max_matches: topResources[esResourceName].pipeline.maxMatches + } + }] + } }) } } @@ -114,32 +220,49 @@ async function main () { await models.init() let keys = Object.keys(models) - keys = _.orderBy(keys, k => { - const esResourceName = modelToESIndexMapping[k] - // Create parent data first - if (_.includes(_.keys(topResources), esResourceName)) { - return -1 + // Sort the models in the order of insertion (for correct enrichment) + const temp = Array(keys.length).fill(null) + keys.forEach(k => { + if (models[k].tableName) { + const esResourceName = modelToESIndexMapping[k] + const index = RESOURCES_IN_ORDER.indexOf(esResourceName) + temp[index] = k } - - return 1 }) + keys = _.compact(temp) for (let i = 0; i < keys.length; i++) { const key = keys[i] - if (models[key].tableName) { - try { - const data = require(`./data/${key}.json`) - await models.DBHelper.clear(models[key]) - for (let i = 0; i < data.length; i++) { - await models.DBHelper.save(models[key], new models[key]().from(data[i]), true) - await insertIntoES(key, data[i]) - } - logger.info('import data for ' + key + ' done') - } catch (e) { - logger.error(e) - logger.warn('import data for ' + key + ' failed') + try { + const data = require(`./data/${key}.json`) + await models.DBHelper.clear(models[key]) + for (let i = 0; i < data.length; i++) { + logger.info(`Inserting data ${i + 1} of ${data.length}`) + await models.DBHelper.save(models[key], new models[key]().from(data[i]), true) + await insertIntoES(key, data[i]) } + logger.info('import data for ' + key + ' done') + } catch (e) { + logger.error(e) + logger.warn('import data for ' + key + ' failed') + continue + } + + try { + await createAndExecuteEnrichPolicy(key) + logger.info('create and execute enrich policy for ' + key + ' done') + } catch (e) { + logger.error(e) + logger.warn('create and execute enrich policy for ' + key + ' failed') + } + + try { + await createEnrichProcessor(key) + logger.info('create enrich processor (pipeline) for ' + key + ' done') + } catch (e) { + logger.error(e) + logger.warn('create enrich processor (pipeline) for ' + key + ' failed') } } logger.info('all done') From 7f7a6cb007221349fcd110de57e9f0fc35481240 Mon Sep 17 00:00:00 2001 From: Mithun Kamath Date: Wed, 21 Oct 2020 19:06:15 +0530 Subject: [PATCH 3/5] Working db dump script and misc --- README.md | 3 + config/default.js | 10 +- scripts/constants.js | 12 +- scripts/db/dropAll.js | 41 ++++++- scripts/db/dumpDbToEs.js | 225 +++++++++++++++++++++++++++++------ scripts/db/genData.js | 1 - src/common/es-helper.js | 8 -- src/common/service-helper.js | 4 - 8 files changed, 243 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index d4bb982..861c35c 100755 --- a/README.md +++ b/README.md @@ -37,6 +37,9 @@ Configuration for the application is at config/default.js and config/production. - UBAHN_AGGREGATE_TOPIC: Kafka topic that is used to combine all create, update and delete message(s) - ES.HOST: Elasticsearch host - ES.DOCUMENTS: Elasticsearch index, type and id mapping for resources. +- ATTRIBUTE_GROUP_PIPELINE_ID: The pipeline id for enrichment with attribute group. Default is `attributegroup-pipeline` +- SKILL_PROVIDER_PIPELINE_ID: The pipeline id for enrichment with skill provider. Default is `skillprovider-pipeline` +- USER_PIPELINE_ID: The pipeline id for enrichment of user details. Default is `user-pipeline` For `ES.DOCUMENTS` configuration, you will find multiple other configurations below it. Each has default values that you can override using the environment variables diff --git a/config/default.js b/config/default.js index e824552..c5e9f3a 100755 --- a/config/default.js +++ b/config/default.js @@ -51,7 +51,6 @@ module.exports = { // ElasticSearch ES: { HOST: process.env.ES_HOST || 'http://localhost:9200', - ENRICH_USER_PIPELINE_NAME: process.env.ENRICH_USER_PIPELINE_NAME || 'enrich_user', // es mapping: _index, _type, _id DOCUMENTS: { achievementprovider: { @@ -64,7 +63,8 @@ module.exports = { }, attributegroup: { index: process.env.ATTRIBUTE_GROUP_INDEX || 'attribute_group', - type: '_doc' + type: '_doc', + pipelineId: process.env.ATTRIBUTE_GROUP_PIPELINE_ID || 'attributegroup-pipeline' }, organization: { index: process.env.ORGANIZATION_INDEX || 'organization', @@ -80,11 +80,13 @@ module.exports = { }, skillprovider: { index: process.env.SKILL_PROVIDER_INDEX || 'skill_provider', - type: '_doc' + type: '_doc', + pipelineId: process.env.SKILL_PROVIDER_PIPELINE_ID || 'skillprovider-pipeline' }, user: { index: process.env.USER_INDEX || 'user', - type: '_doc' + type: '_doc', + pipelineId: process.env.USER_PIPELINE_ID || 'user-pipeline' }, // sub resources under user achievement: { diff --git a/scripts/constants.js b/scripts/constants.js index 1b0a5b3..de5d6af 100644 --- a/scripts/constants.js +++ b/scripts/constants.js @@ -15,7 +15,7 @@ const topResources = { enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy'] }, pipeline: { - id: 'skillprovider-pipeline', + id: config.get('ES.DOCUMENTS.skillprovider.pipelineId'), field: 'skillProviderId', targetField: 'skillprovider', maxMatches: '1' @@ -51,7 +51,7 @@ const topResources = { enrichFields: ['id', 'name', 'organizationId', 'created', 'updated', 'createdBy', 'updatedBy'] }, pipeline: { - id: 'attributegroup-pipeline', + id: config.get('ES.DOCUMENTS.attributegroup.pipelineId'), field: 'attributeGroupId', targetField: 'attributegroup', maxMatches: '1' @@ -68,7 +68,7 @@ const topResources = { }, ingest: { pipeline: { - id: 'skillprovider-pipeline' + id: config.get('ES.DOCUMENTS.skillprovider.pipelineId') } } }, @@ -83,21 +83,21 @@ const topResources = { }, ingest: { pipeline: { - id: 'attributegroup-pipeline' + id: config.get('ES.DOCUMENTS.attributegroup.pipelineId') } } }, organization: { index: config.get('ES.DOCUMENTS.organization.index'), - type: config.get('ES.DOCUMENTS.organization.type'), + type: config.get('ES.DOCUMENTS.organization.type') }, user: { index: config.get('ES.DOCUMENTS.user.index'), type: config.get('ES.DOCUMENTS.user.type'), pipeline: { - id: 'user-pipeline', + id: config.get('ES.DOCUMENTS.user.pipelineId'), processors: [ { referenceField: config.get('ES.DOCUMENTS.achievement.userField'), diff --git a/scripts/db/dropAll.js b/scripts/db/dropAll.js index cef1924..a30a2cf 100644 --- a/scripts/db/dropAll.js +++ b/scripts/db/dropAll.js @@ -4,11 +4,33 @@ const _ = require('lodash') const models = require('../../src/models') const logger = require('../../src/common/logger') -const { topResources, modelToESIndexMapping } = require('../constants') +const { + topResources, + organizationResources, + modelToESIndexMapping +} = require('../constants') const { getESClient } = require('../../src/common/es-client') async function main () { const client = getESClient() + + try { + logger.info('Deleting all pipelines...') + await client.ingest.deletePipeline({ + id: topResources.user.pipeline.id + }) + await client.ingest.deletePipeline({ + id: topResources.skillprovider.pipeline.id + }) + await client.ingest.deletePipeline({ + id: topResources.attributegroup.pipeline.id + }) + logger.info('Successfully deleted') + } catch (e) { + console.error(e) + logger.warn('Delete all ingest pipelines failed') + } + const keys = Object.keys(models) for (let i = 0; i < keys.length; i++) { const key = keys[i] @@ -16,12 +38,29 @@ async function main () { const esResourceName = modelToESIndexMapping[key] try { if (_.includes(_.keys(topResources), esResourceName)) { + if (topResources[esResourceName].enrich) { + logger.info(`Deleting enrich policy for ${esResourceName}`) + await client.enrich.deletePolicy({ + name: topResources[esResourceName].enrich.policyName + }) + logger.info(`Successfully deleted enrich policy for ${esResourceName}`) + } + logger.info(`Deleting index for ${esResourceName}`) await client.indices.delete({ index: topResources[esResourceName].index }) + logger.info(`Successfully deleted enrich policy for ${esResourceName}`) + } else if (_.includes(_.keys(organizationResources), esResourceName)) { + logger.info('Deleting enrich policy for organization') + await client.enrich.deletePolicy({ + name: organizationResources[esResourceName].enrich.policyName + }) + logger.info('Successfully deleted enrich policy for organization') } + logger.info(`Deleting data in QLDB for ${esResourceName}`) await models.DBHelper.clear(models[key]) + logger.info(`Successfully deleted data in QLDB for ${esResourceName}`) } catch (e) { console.error(e) logger.warn(`drop table ${key} failed`) diff --git a/scripts/db/dumpDbToEs.js b/scripts/db/dumpDbToEs.js index 0d12e73..c5a23fd 100644 --- a/scripts/db/dumpDbToEs.js +++ b/scripts/db/dumpDbToEs.js @@ -1,5 +1,4 @@ const _ = require('lodash') -const config = require('config') const models = require('../../src/models') const logger = require('../../src/common/logger') const { getESClient } = require('../../src/common/es-client') @@ -10,14 +9,62 @@ const { modelToESIndexMapping } = require('../constants') -async function cleanupES () { - const client = getESClient() +// Declares the ordering of the resource data insertion, to ensure that enrichment happens correctly +const RESOURCES_IN_ORDER = [ + 'skillprovider', + 'role', + 'achievementprovider', + 'attributegroup', + 'skill', + 'attribute', + 'organization', + 'organizationskillprovider', + 'user', + 'userskill', + 'achievement', + 'userrole', + 'externalprofile', + 'userattribute' +] - await client.indices.delete({ - index: '_all' - }) +const client = getESClient() - console.log('Existing indices have been deleted!') +/** + * Cleans up the data in elasticsearch + * @param {Array} keys Array of models + */ +async function cleanupES (keys) { + const client = getESClient() + await client.ingest.deletePipeline({ + id: topResources.user.pipeline.id + }) + await client.ingest.deletePipeline({ + id: topResources.skillprovider.pipeline.id + }) + await client.ingest.deletePipeline({ + id: topResources.attributegroup.pipeline.id + }) + for (let i = 0; i < keys.length; i++) { + const key = keys[i] + if (models[key].tableName) { + const esResourceName = modelToESIndexMapping[key] + if (_.includes(_.keys(topResources), esResourceName)) { + if (topResources[esResourceName].enrich) { + await client.enrich.deletePolicy({ + name: topResources[esResourceName].enrich.policyName + }) + } + await client.indices.delete({ + index: topResources[esResourceName].index + }) + } else if (_.includes(_.keys(organizationResources), esResourceName)) { + await client.enrich.deletePolicy({ + name: organizationResources[esResourceName].enrich.policyName + }) + } + } + } + console.log('Existing data in elasticsearch has been deleted!') } async function insertIntoES (modelName, body) { @@ -29,19 +76,15 @@ async function insertIntoES (modelName, body) { return } - const client = getESClient() - if (_.includes(_.keys(topResources), esResourceName)) { - await client.create({ + await client.index({ index: topResources[esResourceName].index, type: topResources[esResourceName].type, id: body.id, body, - refresh: 'true' + pipeline: topResources[esResourceName].ingest ? topResources[esResourceName].ingest.pipeline.id : undefined, + refresh: 'wait_for' }) - if (topResources[esResourceName].enrichPolicy) { - await client.enrich.executePolicy({ name: topResources[esResourceName].enrichPolicy }) - } } else if (_.includes(_.keys(userResources), esResourceName)) { const userResource = userResources[esResourceName] @@ -82,8 +125,8 @@ async function insertIntoES (modelName, body) { type: topResources.user.type, id: body.userId, body: user, - pipeline: config.get('ES.ENRICH_USER_PIPELINE_NAME'), - refresh: 'true' + pipeline: topResources.user.pipeline.id, + refresh: 'wait_for' }) } } else if (_.includes(_.keys(organizationResources), esResourceName)) { @@ -105,12 +148,102 @@ async function insertIntoES (modelName, body) { logger.error(`Can't create existing ${esResourceName} with the ${orgResource.relateKey}: ${relateId}, organizationId: ${body.organizationId}`) } else { organization[orgResource.propertyName].push(body) - await client.update({ + await client.index({ index: topResources.organization.index, type: topResources.organization.type, id: body.organizationId, - body: { doc: organization }, - refresh: 'true' + body: organization, + refresh: 'wait_for' + }) + } + } +} + +/** + * Creates and executes the enrich policy for the provided model + * @param {String} modelName The model name + */ +async function createAndExecuteEnrichPolicy (modelName) { + const esResourceName = modelToESIndexMapping[modelName] + + if (_.includes(_.keys(topResources), esResourceName) && topResources[esResourceName].enrich) { + await client.enrich.putPolicy({ + name: topResources[esResourceName].enrich.policyName, + body: { + match: { + indices: topResources[esResourceName].index, + match_field: topResources[esResourceName].enrich.matchField, + enrich_fields: topResources[esResourceName].enrich.enrichFields + } + } + }) + await client.enrich.executePolicy({ name: topResources[esResourceName].enrich.policyName }) + } else if (_.includes(_.keys(organizationResources), esResourceName)) { + // For organization, execute enrich policy AFTER the sub documents on the org (namely orgskillprovider) is in + // This is because external profile on user is enriched with org, and it needs to have the orgskillprovider details in it + await client.enrich.putPolicy({ + name: organizationResources[esResourceName].enrich.policyName, + body: { + match: { + indices: topResources.organization.index, + match_field: organizationResources[esResourceName].enrich.matchField, + enrich_fields: organizationResources[esResourceName].enrich.enrichFields + } + } + }) + await client.enrich.executePolicy({ name: organizationResources[esResourceName].enrich.policyName }) + } +} + +/** + * Creates the ingest pipeline using the enrich policy + * @param {String} modelName The model name + */ +async function createEnrichProcessor (modelName) { + const esResourceName = modelToESIndexMapping[modelName] + + if (_.includes(_.keys(topResources), esResourceName) && topResources[esResourceName].pipeline) { + if (topResources[esResourceName].pipeline.processors) { + const processors = [] + + for (let i = 0; i < topResources[esResourceName].pipeline.processors.length; i++) { + const ep = topResources[esResourceName].pipeline.processors[i] + processors.push({ + foreach: { + field: ep.referenceField, + ignore_missing: true, + processor: { + enrich: { + policy_name: ep.enrichPolicyName, + ignore_missing: true, + field: ep.field, + target_field: ep.targetField, + max_matches: ep.maxMatches + } + } + } + }) + } + + await client.ingest.putPipeline({ + id: topResources[esResourceName].pipeline.id, + body: { + processors + } + }) + } else { + await client.ingest.putPipeline({ + id: topResources[esResourceName].pipeline.id, + body: { + processors: [{ + enrich: { + policy_name: topResources[esResourceName].enrich.policyName, + field: topResources[esResourceName].pipeline.field, + target_field: topResources[esResourceName].pipeline.targetField, + max_matches: topResources[esResourceName].pipeline.maxMatches + } + }] + } }) } } @@ -122,32 +255,50 @@ async function insertIntoES (modelName, body) { */ async function main () { let keys = Object.keys(models) - keys = _.orderBy(keys, k => { - const esResourceName = modelToESIndexMapping[k] - // Create parent data first - if (_.includes(_.keys(topResources), esResourceName)) { - return -1 + // Sort the models in the order of insertion (for correct enrichment) + const temp = Array(keys.length).fill(null) + keys.forEach(k => { + if (models[k].tableName) { + const esResourceName = modelToESIndexMapping[k] + const index = RESOURCES_IN_ORDER.indexOf(esResourceName) + temp[index] = k } - - return 1 }) + keys = _.compact(temp) - await cleanupES() + await cleanupES(keys) for (let i = 0; i < keys.length; i++) { const key = keys[i] - if (models[key].tableName) { - try { - const data = await models.DBHelper.find(models[key], []) - for (let i = 0; i < data.length; i++) { - await insertIntoES(key, data[i]) - } - logger.info('import data for ' + key + ' done') - } catch (e) { - logger.error(e) - logger.warn('import data for ' + key + ' failed') + try { + const data = await models.DBHelper.find(models[key], []) + + for (let i = 0; i < data.length; i++) { + logger.info(`Inserting data ${i + 1} of ${data.length}`) + await insertIntoES(key, data[i]) } + logger.info('import data for ' + key + ' done') + } catch (e) { + logger.error(e) + logger.warn('import data for ' + key + ' failed') + continue + } + + try { + await createAndExecuteEnrichPolicy(key) + logger.info('create and execute enrich policy for ' + key + ' done') + } catch (e) { + logger.error(e) + logger.warn('create and execute enrich policy for ' + key + ' failed') + } + + try { + await createEnrichProcessor(key) + logger.info('create enrich processor (pipeline) for ' + key + ' done') + } catch (e) { + logger.error(e) + logger.warn('create enrich processor (pipeline) for ' + key + ' failed') } } logger.info('all done') diff --git a/scripts/db/genData.js b/scripts/db/genData.js index b1cfa0e..97681ac 100644 --- a/scripts/db/genData.js +++ b/scripts/db/genData.js @@ -1,5 +1,4 @@ const _ = require('lodash') -const config = require('config') const models = require('../../src/models') const logger = require('../../src/common/logger') const { getESClient } = require('../../src/common/es-client') diff --git a/src/common/es-helper.js b/src/common/es-helper.js index 581ce3f..3cdf833 100644 --- a/src/common/es-helper.js +++ b/src/common/es-helper.js @@ -215,13 +215,11 @@ const FILTER_CHAIN = { skill: { filterNext: 'userskill', queryField: 'skillId', - enrichNext: 'skillprovider', idField: 'skillProviderId' }, attribute: { filterNext: 'userattribute', queryField: 'attributeId', - enrichNext: 'attributegroup', idField: 'attributeGroupId' }, attributegroup: { @@ -247,29 +245,23 @@ const FILTER_CHAIN = { // sub resource userskill: { queryField: 'skillId', - enrichNext: 'skill', idField: 'skillId' }, userrole: { queryField: 'roleId', - enrichNext: 'role', idField: 'roleId' }, externalprofile: { - enrichNext: 'organization', idField: 'organizationId' }, achievement: { - enrichNext: 'achievementprovider', idField: 'achievementsProviderId' }, userattribute: { - enrichNext: 'attribute', idField: 'attributeId' }, organizationskillprovider: { queryField: 'skillProviderId', - enrichNext: 'skillprovider', idField: 'skillProviderId' } } diff --git a/src/common/service-helper.js b/src/common/service-helper.js index 072a0b8..56870e5 100644 --- a/src/common/service-helper.js +++ b/src/common/service-helper.js @@ -259,10 +259,6 @@ function getServiceMethods (Model, createSchema, patchSchema, searchSchema, buil try { return await esHelper.searchElasticSearch(resource, query, auth) } catch (err) { - // return error if enrich fails - if (resource === 'user' && query.enrich) { - throw errors.elasticSearchEnrichError(err.message) - } logger.logFullError(err) } From 8ac769bc4604e866977dfc8295c881dbc5c70fca Mon Sep 17 00:00:00 2001 From: Mithun Kamath Date: Wed, 21 Oct 2020 19:12:41 +0530 Subject: [PATCH 4/5] Restore deleted code --- src/common/service-helper.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/common/service-helper.js b/src/common/service-helper.js index 56870e5..072a0b8 100644 --- a/src/common/service-helper.js +++ b/src/common/service-helper.js @@ -259,6 +259,10 @@ function getServiceMethods (Model, createSchema, patchSchema, searchSchema, buil try { return await esHelper.searchElasticSearch(resource, query, auth) } catch (err) { + // return error if enrich fails + if (resource === 'user' && query.enrich) { + throw errors.elasticSearchEnrichError(err.message) + } logger.logFullError(err) } From 310adc9fdb564212799ddc5849c7a1396aeee569 Mon Sep 17 00:00:00 2001 From: Mithun Kamath Date: Thu, 22 Oct 2020 14:41:43 +0530 Subject: [PATCH 5/5] 1. Set the enrich policy name as a config 2. Fix issue where migrate data from db to es would not save boolean fields 3. Fix issue where updating skill or attribute would result in referenced fields to also be passed in bus api --- README.md | 6 +++ config/default.js | 21 +++++--- scripts/constants.js | 14 +++--- scripts/db/dumpDbToEs.js | 98 +++++++++++++++++++++++++++--------- src/common/helper.js | 3 ++ src/common/service-helper.js | 29 ++++++----- 6 files changed, 119 insertions(+), 52 deletions(-) diff --git a/README.md b/README.md index 861c35c..a0a2dc4 100755 --- a/README.md +++ b/README.md @@ -40,6 +40,12 @@ Configuration for the application is at config/default.js and config/production. - ATTRIBUTE_GROUP_PIPELINE_ID: The pipeline id for enrichment with attribute group. Default is `attributegroup-pipeline` - SKILL_PROVIDER_PIPELINE_ID: The pipeline id for enrichment with skill provider. Default is `skillprovider-pipeline` - USER_PIPELINE_ID: The pipeline id for enrichment of user details. Default is `user-pipeline` +- ATTRIBUTE_GROUP_ENRICH_POLICYNAME: The enrich policy for attribute group. Default is `attributegroup-policy` +- SKILL_PROVIDER_ENRICH_POLICYNAME: The enrich policy for skill provider. Default is `skillprovider-policy` +- ROLE_ENRICH_POLICYNAME: The enrich policy for role. Default is `role-policy` +- ACHIEVEMENT_PROVIDER_ENRICH_POLICYNAME: The enrich policy for achievement provider. Default is `achievementprovider-policy` +- SKILL_ENRICH_POLICYNAME: The enrich policy for skill. Default is `skill-policy` +- ATTRIBUTE_ENRICH_POLICYNAME: The enrich policy for skill. Default is `attribute-policy` For `ES.DOCUMENTS` configuration, you will find multiple other configurations below it. Each has default values that you can override using the environment variables diff --git a/config/default.js b/config/default.js index c5e9f3a..66ed133 100755 --- a/config/default.js +++ b/config/default.js @@ -55,33 +55,40 @@ module.exports = { DOCUMENTS: { achievementprovider: { index: process.env.ACHIEVEMENT_PROVIDER_INDEX || 'achievement_provider', - type: '_doc' + type: '_doc', + enrichPolicyName: process.env.ACHIEVEMENT_PROVIDER_ENRICH_POLICYNAME || 'achievementprovider-policy' }, attribute: { index: process.env.ATTRIBUTE_INDEX || 'attribute', - type: '_doc' + type: '_doc', + enrichPolicyName: process.env.ATTRIBUTE_ENRICH_POLICYNAME || 'attribute-policy' }, attributegroup: { index: process.env.ATTRIBUTE_GROUP_INDEX || 'attribute_group', type: '_doc', - pipelineId: process.env.ATTRIBUTE_GROUP_PIPELINE_ID || 'attributegroup-pipeline' + pipelineId: process.env.ATTRIBUTE_GROUP_PIPELINE_ID || 'attributegroup-pipeline', + enrichPolicyName: process.env.ATTRIBUTE_GROUP_ENRICH_POLICYNAME || 'attributegroup-policy' }, organization: { index: process.env.ORGANIZATION_INDEX || 'organization', - type: '_doc' + type: '_doc', + enrichPolicyName: process.env.ORGANIZATION_ENRICH_POLICYNAME || 'organization-policy' }, role: { index: process.env.ROLE_INDEX || 'role', - type: '_doc' + type: '_doc', + enrichPolicyName: process.env.ROLE_ENRICH_POLICYNAME || 'role-policy' }, skill: { index: process.env.SKILL_INDEX || 'skill', - type: '_doc' + type: '_doc', + enrichPolicyName: process.env.SKILL_ENRICH_POLICYNAME || 'skill-policy' }, skillprovider: { index: process.env.SKILL_PROVIDER_INDEX || 'skill_provider', type: '_doc', - pipelineId: process.env.SKILL_PROVIDER_PIPELINE_ID || 'skillprovider-pipeline' + pipelineId: process.env.SKILL_PROVIDER_PIPELINE_ID || 'skillprovider-pipeline', + enrichPolicyName: process.env.SKILL_PROVIDER_ENRICH_POLICYNAME || 'skillprovider-policy' }, user: { index: process.env.USER_INDEX || 'user', diff --git a/scripts/constants.js b/scripts/constants.js index de5d6af..c5d976c 100644 --- a/scripts/constants.js +++ b/scripts/constants.js @@ -10,7 +10,7 @@ const topResources = { index: config.get('ES.DOCUMENTS.skillprovider.index'), type: config.get('ES.DOCUMENTS.skillprovider.type'), enrich: { - policyName: 'skillprovider-policy', + policyName: config.get('ES.DOCUMENTS.skillprovider.enrichPolicyName'), matchField: 'id', enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy'] }, @@ -26,7 +26,7 @@ const topResources = { index: config.get('ES.DOCUMENTS.role.index'), type: config.get('ES.DOCUMENTS.role.type'), enrich: { - policyName: 'role-policy', + policyName: config.get('ES.DOCUMENTS.role.enrichPolicyName'), matchField: 'id', enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy'] } @@ -36,7 +36,7 @@ const topResources = { index: config.get('ES.DOCUMENTS.achievementprovider.index'), type: config.get('ES.DOCUMENTS.achievementprovider.type'), enrich: { - policyName: 'achievementprovider-policy', + policyName: config.get('ES.DOCUMENTS.achievementprovider.enrichPolicyName'), matchField: 'id', enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy'] } @@ -46,7 +46,7 @@ const topResources = { index: config.get('ES.DOCUMENTS.attributegroup.index'), type: config.get('ES.DOCUMENTS.attributegroup.type'), enrich: { - policyName: 'attributegroup-policy', + policyName: config.get('ES.DOCUMENTS.attributegroup.enrichPolicyName'), matchField: 'id', enrichFields: ['id', 'name', 'organizationId', 'created', 'updated', 'createdBy', 'updatedBy'] }, @@ -62,7 +62,7 @@ const topResources = { index: config.get('ES.DOCUMENTS.skill.index'), type: config.get('ES.DOCUMENTS.skill.type'), enrich: { - policyName: 'skill-policy', + policyName: config.get('ES.DOCUMENTS.skill.enrichPolicyName'), matchField: 'id', enrichFields: ['id', 'skillProviderId', 'name', 'externalId', 'uri', 'created', 'updated', 'createdBy', 'updatedBy', 'skillprovider'] }, @@ -77,7 +77,7 @@ const topResources = { index: config.get('ES.DOCUMENTS.attribute.index'), type: config.get('ES.DOCUMENTS.attribute.type'), enrich: { - policyName: 'attribute-policy', + policyName: config.get('ES.DOCUMENTS.attribute.enrichPolicyName'), matchField: 'id', enrichFields: ['id', 'name', 'attributeGroupId', 'created', 'updated', 'createdBy', 'updatedBy', 'attributegroup'] }, @@ -168,7 +168,7 @@ const organizationResources = { propertyName: config.get('ES.DOCUMENTS.organizationskillprovider.orgField'), relateKey: 'skillProviderId', enrich: { - policyName: 'organization-policy', + policyName: config.get('ES.DOCUMENTS.organization.enrichPolicyName'), matchField: 'id', enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy', 'skillProviders'] } diff --git a/scripts/db/dumpDbToEs.js b/scripts/db/dumpDbToEs.js index c5a23fd..006f0bb 100644 --- a/scripts/db/dumpDbToEs.js +++ b/scripts/db/dumpDbToEs.js @@ -29,40 +29,88 @@ const RESOURCES_IN_ORDER = [ const client = getESClient() +const RESOURCE_NOT_FOUND = 'resource_not_found_exception' +const INDEX_NOT_FOUND = 'index_not_found_exception' + /** * Cleans up the data in elasticsearch * @param {Array} keys Array of models */ async function cleanupES (keys) { const client = getESClient() - await client.ingest.deletePipeline({ - id: topResources.user.pipeline.id - }) - await client.ingest.deletePipeline({ - id: topResources.skillprovider.pipeline.id - }) - await client.ingest.deletePipeline({ - id: topResources.attributegroup.pipeline.id - }) - for (let i = 0; i < keys.length; i++) { - const key = keys[i] - if (models[key].tableName) { - const esResourceName = modelToESIndexMapping[key] - if (_.includes(_.keys(topResources), esResourceName)) { - if (topResources[esResourceName].enrich) { - await client.enrich.deletePolicy({ - name: topResources[esResourceName].enrich.policyName - }) + try { + await client.ingest.deletePipeline({ + id: topResources.user.pipeline.id + }) + } catch (e) { + if (e.meta && e.meta.body.error.type !== RESOURCE_NOT_FOUND) { + throw e + } + } + + try { + await client.ingest.deletePipeline({ + id: topResources.skillprovider.pipeline.id + }) + } catch (e) { + if (e.meta && e.meta.body.error.type !== RESOURCE_NOT_FOUND) { + throw e + } + } + + try { + await client.ingest.deletePipeline({ + id: topResources.attributegroup.pipeline.id + }) + } catch (e) { + if (e.meta && e.meta.body.error.type !== RESOURCE_NOT_FOUND) { + throw e + } + } + + try { + for (let i = 0; i < keys.length; i++) { + const key = keys[i] + if (models[key].tableName) { + const esResourceName = modelToESIndexMapping[key] + if (_.includes(_.keys(topResources), esResourceName)) { + if (topResources[esResourceName].enrich) { + try { + await client.enrich.deletePolicy({ + name: topResources[esResourceName].enrich.policyName + }) + } catch (e) { + if (e.meta && e.meta.body.error.type !== RESOURCE_NOT_FOUND) { + throw e + } + } + } + + try { + await client.indices.delete({ + index: topResources[esResourceName].index + }) + } catch (e) { + if (e.meta && e.meta.body.error.type !== INDEX_NOT_FOUND) { + throw e + } + } + } else if (_.includes(_.keys(organizationResources), esResourceName)) { + try { + await client.enrich.deletePolicy({ + name: organizationResources[esResourceName].enrich.policyName + }) + } catch (e) { + if (e.meta && e.meta.body.error.type !== RESOURCE_NOT_FOUND) { + throw e + } + } } - await client.indices.delete({ - index: topResources[esResourceName].index - }) - } else if (_.includes(_.keys(organizationResources), esResourceName)) { - await client.enrich.deletePolicy({ - name: organizationResources[esResourceName].enrich.policyName - }) } } + } catch (e) { + console.log(JSON.stringify(e)) + throw e } console.log('Existing data in elasticsearch has been deleted!') } diff --git a/src/common/helper.js b/src/common/helper.js index 604d0fc..56e269e 100644 --- a/src/common/helper.js +++ b/src/common/helper.js @@ -109,6 +109,9 @@ function readerToJson (reader) { toRealValue(r, setValue(name, [])) r.stepOut() break + case IonTypes.BOOL: + setValue(name, r.booleanValue()) + break } nextT = reader.next() } diff --git a/src/common/service-helper.js b/src/common/service-helper.js index 072a0b8..10eb361 100644 --- a/src/common/service-helper.js +++ b/src/common/service-helper.js @@ -190,7 +190,7 @@ function getServiceMethods (Model, createSchema, patchSchema, searchSchema, buil async function patch (id, entity, auth, params) { await makeSureRefExist(entity) - const dbEntity = await get(id, auth, params) + const dbEntity = await get(id, auth, params, {}, true) const newEntity = new Model() _.extend(newEntity, dbEntity, entity) newEntity.updated = new Date() @@ -218,23 +218,26 @@ function getServiceMethods (Model, createSchema, patchSchema, searchSchema, buil * @param auth the auth obj * @param params the path parameters * @param query the query parameters + * @param fromDb Should we bypass Elasticsearch for the record and fetch from db instead? * @return {Promise} the db device */ - async function get (id, auth, params, query = {}) { + async function get (id, auth, params, query = {}, fromDb = false) { let recordObj // Merge path and query params const trueParams = _.assign(params, query) - try { - const result = await esHelper.getFromElasticSearch(resource, id, auth, trueParams) - // check permission - permissionCheck(auth, result) - return result - } catch (err) { - // return error if enrich fails or permission fails - if ((resource === 'user' && trueParams.enrich) || (err.status && err.status === 403)) { - throw errors.elasticSearchEnrichError(err.message) + if (!fromDb) { + try { + const result = await esHelper.getFromElasticSearch(resource, id, auth, trueParams) + // check permission + permissionCheck(auth, result) + return result + } catch (err) { + // return error if enrich fails or permission fails + if ((resource === 'user' && trueParams.enrich) || (err.status && err.status === 403)) { + throw errors.elasticSearchEnrichError(err.message) + } + logger.logFullError(err) } - logger.logFullError(err) } if (_.isNil(trueParams) || _.isEmpty(trueParams)) { recordObj = await models.DBHelper.get(Model, id) @@ -302,7 +305,7 @@ function getServiceMethods (Model, createSchema, patchSchema, searchSchema, buil */ async function remove (id, auth, params) { let payload - await get(id, auth, params) // check exist + await get(id, auth, params, {}, true) // check exist await models.DBHelper.delete(Model, id, buildQueryByParams(params)) if (SUB_USER_DOCUMENTS[resource] || SUB_ORG_DOCUMENTS[resource]) { payload = _.assign({}, params)