From d6ffc57b9082c8da7f668dfb92bc84da648f15b3 Mon Sep 17 00:00:00 2001 From: Unknown Date: Fri, 29 Mar 2019 14:28:52 -0500 Subject: [PATCH 1/8] New deps for some reason... --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 703908e..ccf7d9d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,7 @@ install_dependency: &install_dependency command: | apk update apk add --no-cache bash - apk add --no-cache jq py-pip sudo curl + apk add --no-cache jq py-pip sudo curl gcc python-dev musl-dev libffi-dev make openssl-dev apk upgrade pip install --upgrade pip sudo pip install awscli --upgrade From d48ef44043919bf15aee38960980c9ebc658f871 Mon Sep 17 00:00:00 2001 From: Gunasekar-K Date: Wed, 3 Apr 2019 12:22:16 +0530 Subject: [PATCH 2/8] Update config.yml [skip ci] --- .circleci/config.yml | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ccf7d9d..99ce2be 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,18 +1,14 @@ version: 2 defaults: &defaults - docker: - - image: docker:18.06.0-ce-git + docker: + - image: circleci/python:2.7-stretch-browsers install_dependency: &install_dependency name: Installation of build and deployment dependencies. command: | - apk update - apk add --no-cache bash - apk add --no-cache jq py-pip sudo curl gcc python-dev musl-dev libffi-dev make openssl-dev - apk upgrade - pip install --upgrade pip + sudo apt install jq sudo pip install awscli --upgrade - sudo curl -o /usr/local/bin/ecs-cli https://s3.amazonaws.com/amazon-ecs-cli/ecs-cli-linux-amd64-latest sudo pip install docker-compose + sudo curl -o /usr/local/bin/ecs-cli https://s3.amazonaws.com/amazon-ecs-cli/ecs-cli-linux-amd64-latest sudo chmod +x /usr/local/bin/ecs-cli install_deploysuite: &install_deploysuite @@ -22,7 +18,6 @@ install_deploysuite: &install_deploysuite cp ./../buildscript/master_deploy.sh . cp ./../buildscript/buildenv.sh . cp ./../buildscript/awsconfiguration.sh . - run_build: &run_build name: Installation of build dependencies. command: ./build.sh @@ -65,7 +60,6 @@ jobs: ./buildenv.sh -e PROD -b submission-processor-es_deployvar source buildenvvar ./master_deploy.sh -d ECS -e PROD -t latest -s prod-global-appvar,submission-processor-es_appvar -i submission-processor-es - workflows: version: 2 build: @@ -80,4 +74,4 @@ workflows: context : org-global filters: branches: - only: master + only: master From 5b491e813add455590acff9c5fc09cc4e35078a0 Mon Sep 17 00:00:00 2001 From: Mithun Kamath Date: Wed, 7 Aug 2019 20:37:23 +0530 Subject: [PATCH 3/8] Merge a new review summation with existing review summations on a submission correctly --- src/services/ProcessorService.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/services/ProcessorService.js b/src/services/ProcessorService.js index 4732d62..9dcf1fc 100755 --- a/src/services/ProcessorService.js +++ b/src/services/ProcessorService.js @@ -53,8 +53,8 @@ function * create (message) { const submission = yield getESData(message.payload.submissionId) let reviewSummationArr = [] reviewSummationArr.push(_.omit(message.payload, ['resource'])) - if (submission.reviewSummationArr) { - reviewSummationArr = reviewSummationArr.concat(submission.review) + if (submission.reviewSummation) { + reviewSummationArr = reviewSummationArr.concat(submission.reviewSummation) } yield client.update({ index: config.get('esConfig.ES_INDEX'), From dabed6df1dd20025e8a6a26a369ecb0086666853 Mon Sep 17 00:00:00 2001 From: Mithun Kamath Date: Mon, 29 Jun 2020 12:18:19 +0530 Subject: [PATCH 4/8] https://github.com/topcoder-platform/submissions-api/issues/179 - Use new index --- README.md | 4 +++- config/default.js | 1 + src/init-es.js | 12 ++++++------ src/services/ProcessorService.js | 20 ++++++++++---------- 4 files changed, 20 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 4dea34c..2f78d5a 100755 --- a/README.md +++ b/README.md @@ -26,6 +26,8 @@ The following parameters can be set in config files or in env variables: Refer to `esConfig` variable in `config/default.js` for ES related configuration. +**NOTE** Kindly refer to Issue 179 of submission api. We had to create a new index, copy the mapping over of the old index, and then the data. The new index is defined under `ES_INDEX_V2` environment variable. This new index is now used for all operations + Also note that there is a `/health` endpoint that checks for the health of the app. This sets up an expressjs server and listens on the environment variable `PORT`. It's not part of the configuration file and needs to be passed as an environment variable ## Local Kafka setup @@ -225,4 +227,4 @@ npm run view-data a12a4180-65aa-42ec-a945-5fd21dec0501 ``` { "topic":"submission.notification.delete", "originator":"submission-api", "timestamp":"2018-08-06T15:46:05.575Z", "mime-type":"application/json", "payload":{ "resource":"review", "id": "d34d4180-65aa-42ec-a945-5fd21dec0503" } } -``` \ No newline at end of file +``` diff --git a/config/default.js b/config/default.js index 7e77087..6c948d3 100755 --- a/config/default.js +++ b/config/default.js @@ -21,6 +21,7 @@ module.exports = { AWS_REGION: process.env.AWS_REGION || 'us-east-1', // AWS Region to be used if we use AWS ES API_VERSION: process.env.ES_API_VERSION || '6.3', ES_INDEX: process.env.ES_INDEX || 'submission', + ES_INDEX_V2: process.env.ES_INDEX_V2 || 'new_submission', ES_TYPE: process.env.ES_TYPE || '_doc' // ES 6.x accepts only 1 Type per index and it's mandatory to define it } } diff --git a/src/init-es.js b/src/init-es.js index bc62453..202e751 100755 --- a/src/init-es.js +++ b/src/init-es.js @@ -15,20 +15,20 @@ const client = helper.getESClient() co(function * () { if (process.argv.length === 3 && process.argv[2] === 'force') { - logger.info(`Delete index ${config.esConfig.ES_INDEX} if any.`) + logger.info(`Delete index ${config.esConfig.ES_INDEX_V2} if any.`) try { - yield client.indices.delete({ index: config.esConfig.ES_INDEX }) + yield client.indices.delete({ index: config.esConfig.ES_INDEX_V2 }) } catch (err) { // ignore } } - const exists = yield client.indices.exists({ index: config.esConfig.ES_INDEX }) + const exists = yield client.indices.exists({ index: config.esConfig.ES_INDEX_V2 }) if (exists) { - logger.info(`The index ${config.esConfig.ES_INDEX} exists.`) + logger.info(`The index ${config.esConfig.ES_INDEX_V2} exists.`) } else { - logger.info(`The index ${config.esConfig.ES_INDEX} will be created.`) - yield client.indices.create({ index: config.esConfig.ES_INDEX }) + logger.info(`The index ${config.esConfig.ES_INDEX_V2} will be created.`) + yield client.indices.create({ index: config.esConfig.ES_INDEX_V2 }) } }).then(() => { logger.info('done') diff --git a/src/services/ProcessorService.js b/src/services/ProcessorService.js index 9dcf1fc..d1fec75 100755 --- a/src/services/ProcessorService.js +++ b/src/services/ProcessorService.js @@ -17,7 +17,7 @@ const client = helper.getESClient() */ function * getESData (id) { return yield client.getSource({ - index: config.get('esConfig.ES_INDEX'), + index: config.get('esConfig.ES_INDEX_V2'), type: config.get('esConfig.ES_TYPE'), id }) @@ -29,7 +29,7 @@ function * getESData (id) { */ function * create (message) { yield client.create({ - index: config.get('esConfig.ES_INDEX'), + index: config.get('esConfig.ES_INDEX_V2'), type: config.get('esConfig.ES_TYPE'), id: message.payload.id, body: message.payload @@ -44,7 +44,7 @@ function * create (message) { reviewArr = reviewArr.concat(submission.review) } yield client.update({ - index: config.get('esConfig.ES_INDEX'), + index: config.get('esConfig.ES_INDEX_V2'), type: config.get('esConfig.ES_TYPE'), id: message.payload.submissionId, body: { doc: {review: reviewArr} } @@ -57,7 +57,7 @@ function * create (message) { reviewSummationArr = reviewSummationArr.concat(submission.reviewSummation) } yield client.update({ - index: config.get('esConfig.ES_INDEX'), + index: config.get('esConfig.ES_INDEX_V2'), type: config.get('esConfig.ES_TYPE'), id: message.payload.submissionId, body: { doc: {reviewSummation: reviewSummationArr} } @@ -84,7 +84,7 @@ create.schema = { */ function * update (message) { yield client.update({ - index: config.get('esConfig.ES_INDEX'), + index: config.get('esConfig.ES_INDEX_V2'), type: config.get('esConfig.ES_TYPE'), id: message.payload.id, body: { doc: message.payload } @@ -98,7 +98,7 @@ function * update (message) { _.remove(submission.review, {id: message.payload.id}) submission.review.push(reviewToBeUpdated) yield client.update({ - index: config.get('esConfig.ES_INDEX'), + index: config.get('esConfig.ES_INDEX_V2'), type: config.get('esConfig.ES_TYPE'), id: submission.id, body: { doc: {review: submission.review} } @@ -111,7 +111,7 @@ function * update (message) { _.remove(submission.reviewSummation, {id: message.payload.id}) submission.reviewSummation.push(reviewSummationToBeUpdated) yield client.update({ - index: config.get('esConfig.ES_INDEX'), + index: config.get('esConfig.ES_INDEX_V2'), type: config.get('esConfig.ES_TYPE'), id: submission.id, body: { doc: {reviewSummation: submission.reviewSummation} } @@ -132,7 +132,7 @@ function * remove (message) { const submission = yield getESData(review.submissionId) _.remove(submission.review, {id: message.payload.id}) yield client.update({ - index: config.get('esConfig.ES_INDEX'), + index: config.get('esConfig.ES_INDEX_V2'), type: config.get('esConfig.ES_TYPE'), id: submission.id, body: { doc: {review: submission.review} } @@ -142,7 +142,7 @@ function * remove (message) { const submission = yield getESData(reviewSummation.submissionId) _.remove(submission.reviewSummation, {id: message.payload.id}) yield client.update({ - index: config.get('esConfig.ES_INDEX'), + index: config.get('esConfig.ES_INDEX_V2'), type: config.get('esConfig.ES_TYPE'), id: submission.id, body: { doc: {reviewSummation: submission.reviewSummation} } @@ -150,7 +150,7 @@ function * remove (message) { } yield client.delete({ - index: config.get('esConfig.ES_INDEX'), + index: config.get('esConfig.ES_INDEX_V2'), type: config.get('esConfig.ES_TYPE'), id: message.payload.id }) From cc68f8cb5928dce74ad3342ecbfde019f55a9aeb Mon Sep 17 00:00:00 2001 From: Mithun Kamath Date: Tue, 30 Jun 2020 16:54:20 +0530 Subject: [PATCH 5/8] Debug index --- src/services/ProcessorService.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/services/ProcessorService.js b/src/services/ProcessorService.js index d1fec75..b82fcfb 100755 --- a/src/services/ProcessorService.js +++ b/src/services/ProcessorService.js @@ -28,6 +28,7 @@ function * getESData (id) { * @param {Object} message the message */ function * create (message) { + logger.debug(`Creating data in index ${config.get('esConfig.ES_INDEX_V2')}`) yield client.create({ index: config.get('esConfig.ES_INDEX_V2'), type: config.get('esConfig.ES_TYPE'), From 64702bbf4a9af84e3e9eead4ee8c9b1c46ab2b40 Mon Sep 17 00:00:00 2001 From: Mithun Kamath Date: Tue, 30 Jun 2020 17:28:41 +0530 Subject: [PATCH 6/8] Update default index to match the one used by submission api --- config/default.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/default.js b/config/default.js index 6c948d3..6ecf196 100755 --- a/config/default.js +++ b/config/default.js @@ -21,7 +21,7 @@ module.exports = { AWS_REGION: process.env.AWS_REGION || 'us-east-1', // AWS Region to be used if we use AWS ES API_VERSION: process.env.ES_API_VERSION || '6.3', ES_INDEX: process.env.ES_INDEX || 'submission', - ES_INDEX_V2: process.env.ES_INDEX_V2 || 'new_submission', + ES_INDEX_V2: process.env.ES_INDEX_V2 || 'submission-index-v2', ES_TYPE: process.env.ES_TYPE || '_doc' // ES 6.x accepts only 1 Type per index and it's mandatory to define it } } From ad872636f27f9f1589fa96a7037b8090ab42f0ad Mon Sep 17 00:00:00 2001 From: Mithun Kamath Date: Tue, 30 Jun 2020 21:51:25 +0530 Subject: [PATCH 7/8] Undo changes made for submission api Issue 179 --- README.md | 2 -- config/default.js | 1 - src/init-es.js | 12 ++++++------ src/services/ProcessorService.js | 21 ++++++++++----------- 4 files changed, 16 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 2f78d5a..036427f 100755 --- a/README.md +++ b/README.md @@ -26,8 +26,6 @@ The following parameters can be set in config files or in env variables: Refer to `esConfig` variable in `config/default.js` for ES related configuration. -**NOTE** Kindly refer to Issue 179 of submission api. We had to create a new index, copy the mapping over of the old index, and then the data. The new index is defined under `ES_INDEX_V2` environment variable. This new index is now used for all operations - Also note that there is a `/health` endpoint that checks for the health of the app. This sets up an expressjs server and listens on the environment variable `PORT`. It's not part of the configuration file and needs to be passed as an environment variable ## Local Kafka setup diff --git a/config/default.js b/config/default.js index 6ecf196..7e77087 100755 --- a/config/default.js +++ b/config/default.js @@ -21,7 +21,6 @@ module.exports = { AWS_REGION: process.env.AWS_REGION || 'us-east-1', // AWS Region to be used if we use AWS ES API_VERSION: process.env.ES_API_VERSION || '6.3', ES_INDEX: process.env.ES_INDEX || 'submission', - ES_INDEX_V2: process.env.ES_INDEX_V2 || 'submission-index-v2', ES_TYPE: process.env.ES_TYPE || '_doc' // ES 6.x accepts only 1 Type per index and it's mandatory to define it } } diff --git a/src/init-es.js b/src/init-es.js index 202e751..bc62453 100755 --- a/src/init-es.js +++ b/src/init-es.js @@ -15,20 +15,20 @@ const client = helper.getESClient() co(function * () { if (process.argv.length === 3 && process.argv[2] === 'force') { - logger.info(`Delete index ${config.esConfig.ES_INDEX_V2} if any.`) + logger.info(`Delete index ${config.esConfig.ES_INDEX} if any.`) try { - yield client.indices.delete({ index: config.esConfig.ES_INDEX_V2 }) + yield client.indices.delete({ index: config.esConfig.ES_INDEX }) } catch (err) { // ignore } } - const exists = yield client.indices.exists({ index: config.esConfig.ES_INDEX_V2 }) + const exists = yield client.indices.exists({ index: config.esConfig.ES_INDEX }) if (exists) { - logger.info(`The index ${config.esConfig.ES_INDEX_V2} exists.`) + logger.info(`The index ${config.esConfig.ES_INDEX} exists.`) } else { - logger.info(`The index ${config.esConfig.ES_INDEX_V2} will be created.`) - yield client.indices.create({ index: config.esConfig.ES_INDEX_V2 }) + logger.info(`The index ${config.esConfig.ES_INDEX} will be created.`) + yield client.indices.create({ index: config.esConfig.ES_INDEX }) } }).then(() => { logger.info('done') diff --git a/src/services/ProcessorService.js b/src/services/ProcessorService.js index b82fcfb..9dcf1fc 100755 --- a/src/services/ProcessorService.js +++ b/src/services/ProcessorService.js @@ -17,7 +17,7 @@ const client = helper.getESClient() */ function * getESData (id) { return yield client.getSource({ - index: config.get('esConfig.ES_INDEX_V2'), + index: config.get('esConfig.ES_INDEX'), type: config.get('esConfig.ES_TYPE'), id }) @@ -28,9 +28,8 @@ function * getESData (id) { * @param {Object} message the message */ function * create (message) { - logger.debug(`Creating data in index ${config.get('esConfig.ES_INDEX_V2')}`) yield client.create({ - index: config.get('esConfig.ES_INDEX_V2'), + index: config.get('esConfig.ES_INDEX'), type: config.get('esConfig.ES_TYPE'), id: message.payload.id, body: message.payload @@ -45,7 +44,7 @@ function * create (message) { reviewArr = reviewArr.concat(submission.review) } yield client.update({ - index: config.get('esConfig.ES_INDEX_V2'), + index: config.get('esConfig.ES_INDEX'), type: config.get('esConfig.ES_TYPE'), id: message.payload.submissionId, body: { doc: {review: reviewArr} } @@ -58,7 +57,7 @@ function * create (message) { reviewSummationArr = reviewSummationArr.concat(submission.reviewSummation) } yield client.update({ - index: config.get('esConfig.ES_INDEX_V2'), + index: config.get('esConfig.ES_INDEX'), type: config.get('esConfig.ES_TYPE'), id: message.payload.submissionId, body: { doc: {reviewSummation: reviewSummationArr} } @@ -85,7 +84,7 @@ create.schema = { */ function * update (message) { yield client.update({ - index: config.get('esConfig.ES_INDEX_V2'), + index: config.get('esConfig.ES_INDEX'), type: config.get('esConfig.ES_TYPE'), id: message.payload.id, body: { doc: message.payload } @@ -99,7 +98,7 @@ function * update (message) { _.remove(submission.review, {id: message.payload.id}) submission.review.push(reviewToBeUpdated) yield client.update({ - index: config.get('esConfig.ES_INDEX_V2'), + index: config.get('esConfig.ES_INDEX'), type: config.get('esConfig.ES_TYPE'), id: submission.id, body: { doc: {review: submission.review} } @@ -112,7 +111,7 @@ function * update (message) { _.remove(submission.reviewSummation, {id: message.payload.id}) submission.reviewSummation.push(reviewSummationToBeUpdated) yield client.update({ - index: config.get('esConfig.ES_INDEX_V2'), + index: config.get('esConfig.ES_INDEX'), type: config.get('esConfig.ES_TYPE'), id: submission.id, body: { doc: {reviewSummation: submission.reviewSummation} } @@ -133,7 +132,7 @@ function * remove (message) { const submission = yield getESData(review.submissionId) _.remove(submission.review, {id: message.payload.id}) yield client.update({ - index: config.get('esConfig.ES_INDEX_V2'), + index: config.get('esConfig.ES_INDEX'), type: config.get('esConfig.ES_TYPE'), id: submission.id, body: { doc: {review: submission.review} } @@ -143,7 +142,7 @@ function * remove (message) { const submission = yield getESData(reviewSummation.submissionId) _.remove(submission.reviewSummation, {id: message.payload.id}) yield client.update({ - index: config.get('esConfig.ES_INDEX_V2'), + index: config.get('esConfig.ES_INDEX'), type: config.get('esConfig.ES_TYPE'), id: submission.id, body: { doc: {reviewSummation: submission.reviewSummation} } @@ -151,7 +150,7 @@ function * remove (message) { } yield client.delete({ - index: config.get('esConfig.ES_INDEX_V2'), + index: config.get('esConfig.ES_INDEX'), type: config.get('esConfig.ES_TYPE'), id: message.payload.id }) From 5e442787037db6f33043ded1cf0562ec294c3354 Mon Sep 17 00:00:00 2001 From: Mithun Kamath Date: Wed, 10 Feb 2021 19:23:32 +0530 Subject: [PATCH 8/8] https://github.com/topcoder-platform/submissions-api/issues/210 - Swap challenge id to correctly store details in elasticsearch --- src/services/ProcessorService.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/services/ProcessorService.js b/src/services/ProcessorService.js index 9dcf1fc..79527ff 100755 --- a/src/services/ProcessorService.js +++ b/src/services/ProcessorService.js @@ -28,6 +28,11 @@ function * getESData (id) { * @param {Object} message the message */ function * create (message) { + if (message.payload.resource === 'submission') { + message.payload.challengeId = message.payload.v5ChallengeId + delete message.payload.v5ChallengeId + } + yield client.create({ index: config.get('esConfig.ES_INDEX'), type: config.get('esConfig.ES_TYPE'), @@ -83,6 +88,13 @@ create.schema = { * @param {Object} message the message */ function * update (message) { + if (message.payload.resource === 'submission') { + const legacyChallengeId = message.payload.challengeId + message.payload.challengeId = message.payload.v5ChallengeId + message.payload.legacyChallengeId = legacyChallengeId + delete message.payload.v5ChallengeId + } + yield client.update({ index: config.get('esConfig.ES_INDEX'), type: config.get('esConfig.ES_TYPE'),