feat: improve request throttling [ZEND-4197] (#1255)

index.d.ts

@@ -9,6 +9,7 @@ export type RunMigrationConfig = {
   yes?: boolean
   retryLimit?: number
   requestBatchSize?: number
+  requestLimit?: number
   host?: string
 } & ({ filePath: string } | { migrationFunction: MigrationFunction })
 

package.json

@@ -65,6 +65,7 @@
     "kind-of": "^6.0.3",
     "listr2": "^3.13.5",
     "lodash": "^4.17.15",
+    "p-throttle": "^4.1.1",
     "uuid": "^9.0.0",
     "yargs": "^15.3.1"
   },
@@ -75,11 +76,13 @@
     "@types/chai": "^4.2.11",
     "@types/chai-as-promised": "^7.1.2",
     "@types/hoek": "^4.1.3",
+    "@types/inquirer": "^9.0.4",
     "@types/joi": "^17.2.3",
     "@types/lodash": "^4.14.149",
     "@types/mocha": "^10.0.1",
     "@types/node": "^17.0.14",
     "@types/sinon-chai": "^3.2.4",
+    "@types/yargs": "^17.0.28",
     "chai": "^4.2.0",
     "chai-as-promised": "^7.1.1",
     "commitizen": "^4.2.2",

src/bin/cli.ts

@@ -2,23 +2,27 @@ import * as path from 'path'
 import type { AxiosRequestConfig } from 'axios'
 
 import chalk from 'chalk'
-import * as inquirer from 'inquirer'
+import inquirer from 'inquirer'
 import { Listr } from 'listr2'
 import { createManagementClient } from './lib/contentful-client'
-const { version } = require('../../package.json')
-const { SpaceAccessError } = require('../lib/errors')
 import createMigrationParser, { ParseResult } from '../lib/migration-parser'
-import { renderPlan, renderValidationErrors, renderRuntimeErrors } from './lib/render-migration'
+import { renderPlan, renderRuntimeErrors, renderValidationErrors } from './lib/render-migration'
 import renderStepsErrors from './lib/steps-errors'
 import writeErrorsToLog from './lib/write-errors-to-log'
-import { RequestBatch } from '../lib/offline-api/index'
+import { RequestBatch } from '../lib/offline-api'
 import { getConfig } from './lib/config'
 import ValidationError from '../lib/interfaces/errors'
 import { PlainClientAPI } from 'contentful-management'
 import trim from 'lodash/trim'
 
+import { SpaceAccessError } from '../lib/errors'
+import pThrottle from 'p-throttle'
+
+const { version } = require('../../package.json')
+
 class ManyError extends Error {
   public errors: (Error | ValidationError)[]
+
   constructor(message: string, errors: (Error | ValidationError)[]) {
     super(message)
     this.errors = errors
@@ -28,6 +32,7 @@ class ManyError extends Error {
 class BatchError extends Error {
   public batch: RequestBatch
   public errors: Error[]
+
   constructor(message: string, batch: RequestBatch, errors: Error[]) {
     super(message)
     this.batch = batch
@@ -45,7 +50,16 @@ const makeTerminatingFunction =
     }
   }
 
-export const createMakeRequest = (client: PlainClientAPI, { spaceId, environmentId }) => {
+export const createMakeRequest = (
+  client: PlainClientAPI,
+  { spaceId, environmentId, limit = 10 }
+) => {
+  const throttle = pThrottle({
+    limit,
+    interval: 1000,
+    strict: false
+  })
+
   const makeBaseUrl = (url: string) => {
     const parts = ['spaces', spaceId, 'environments', environmentId, trim(url, '/')]
 
@@ -56,11 +70,14 @@ export const createMakeRequest = (client: PlainClientAPI, { spaceId, environment
     const { url, ...config } = requestConfig
     const fullUrl = makeBaseUrl(url)
 
-    return client.raw.http(fullUrl, config)
+    return throttle(() => client.raw.http(fullUrl, config))()
   }
 }
 
-const getMigrationFunctionFromFile = (filePath, terminate) => {
+const getMigrationFunctionFromFile = (
+  filePath: string,
+  terminate: ReturnType<typeof makeTerminatingFunction>
+) => {
   try {
     return require(filePath)
   } catch (e) {
@@ -90,7 +107,8 @@ const createRun = ({ shouldThrow }) =>
     const client = createManagementClient(clientConfig)
     const makeRequest = createMakeRequest(client, {
       spaceId: clientConfig.spaceId,
-      environmentId: clientConfig.environmentId
+      environmentId: clientConfig.environmentId,
+      limit: argv.requestLimit
     })
 
     const migrationParser = createMigrationParser(makeRequest, clientConfig)

src/bin/usage-params.ts

@@ -61,6 +61,13 @@ export default yargs
     type: 'number',
     default: 100
   })
+  .option('request-limit', {
+    alias: 'r',
+    boolean: false,
+    describe: 'Max requests per second',
+    type: 'number',
+    default: 10
+  })
   .option('header', {
     alias: 'H',
     type: 'string',