fix(s3): add maxConcurrentUploadFiles

Добавляем ограничение на параллельную загрузку файлов
VKCOM · Oct 28, 2024 · 0cfd414 · 0cfd414
1 parent a63a7b7
commit 0cfd414
Show file tree

Hide file tree

Showing 3 changed files with 28 additions and 5 deletions.
diff --git a/VKUI/s3/package.json b/VKUI/s3/package.json
@@ -5,13 +5,15 @@
   "license": "MIT",
   "private": true,
   "devDependencies": {
+    "@types/lodash": "^4.17.12",
     "@types/mime-types": "^2.1.4",
     "@types/node": "^22.7.7",
     "typescript": "^5.6.3"
   },
   "dependencies": {
     "@actions/core": "^1.11.1",
     "@aws-sdk/client-s3": "^3.675.0",
+    "lodash": "^4.17.21",
     "mime-types": "^2.1.35"
   },
   "scripts": {

diff --git a/VKUI/s3/src/main.ts b/VKUI/s3/src/main.ts
@@ -3,8 +3,11 @@ import path from 'path';
 
 import * as core from '@actions/core';
 import { S3, S3ClientConfig } from '@aws-sdk/client-s3';
+import lodash from 'lodash';
 import { lookup } from 'mime-types';
 
+const maxConcurrentUploadFiles = 10;
+
 const req = {
   required: true,
 };
@@ -61,7 +64,8 @@ class Action {
     const files = getFilesFromFolder(sourceDir);
 
     core.debug(`length ${files.length}`);
-    await Promise.all(
+
+    const uploadTasks = lodash.chunk(
       files.map((file) => {
         core.debug(`file: ${file}`);
         const fileStream = fs.createReadStream(file);
@@ -76,7 +80,12 @@ class Action {
           ContentType: lookup(file) || 'text/plain',
         });
       }),
+      maxConcurrentUploadFiles,
     );
+
+    for (const tasks of uploadTasks) {
+      await Promise.all(tasks);
+    }
   }
 
   private async delete(prefix: string) {

diff --git a/yarn.lock b/yarn.lock
@@ -1077,17 +1077,24 @@
   resolved "https://registry.yarnpkg.com/@esbuild/win32-x64/-/win32-x64-0.24.0.tgz#168ab1c7e1c318b922637fad8f339d48b01e1244"
   integrity sha512-7IAFPrjSQIJrGsK6flwg7NFmwBoSTyF3rl7If0hNUFQU4ilTsEPL6GuMuU9BfIWVVGuRnuIidkSMC+c0Otu8IA==
 
-"@eslint-community/eslint-utils@^4.2.0", "@eslint-community/eslint-utils@^4.4.0":
+"@eslint-community/eslint-utils@^4.2.0":
   version "4.4.0"
   resolved "https://registry.yarnpkg.com/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz#a23514e8fb9af1269d5f7788aa556798d61c6b59"
   integrity sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==
   dependencies:
     eslint-visitor-keys "^3.3.0"
 
+"@eslint-community/eslint-utils@^4.4.0":
+  version "4.4.1"
+  resolved "https://registry.yarnpkg.com/@eslint-community/eslint-utils/-/eslint-utils-4.4.1.tgz#d1145bf2c20132d6400495d6df4bf59362fd9d56"
+  integrity sha512-s3O3waFUrMV8P/XaF/+ZTp1X9XBZW1a4B97ZnjQF2KYWaFD2A8KyFBsrsfSjEmjn3RGWAIuvlneuZm3CUK3jbA==
+  dependencies:
+    eslint-visitor-keys "^3.4.3"
+
 "@eslint-community/regexpp@^4.10.0":
-  version "4.11.1"
-  resolved "https://registry.yarnpkg.com/@eslint-community/regexpp/-/regexpp-4.11.1.tgz#a547badfc719eb3e5f4b556325e542fbe9d7a18f"
-  integrity sha512-m4DVN9ZqskZoLU5GlWZadwDnYo3vAEydiUayB9widCl9ffWx2IvPnp6n3on5rJmziJSw9Bv+Z3ChDVdMwXCY8Q==
+  version "4.12.1"
+  resolved "https://registry.yarnpkg.com/@eslint-community/regexpp/-/regexpp-4.12.1.tgz#cfc6cffe39df390a3841cde2abccf92eaa7ae0e0"
+  integrity sha512-CCZCDJuduB9OUkFkY2IgppNZMi2lBQgD2qzwXkEia16cge2pijY/aXi96CJMquDMn3nJdlPV1A5KrJEXwfLNzQ==
 
 "@eslint-community/regexpp@^4.6.1":
   version "4.11.0"
@@ -2224,6 +2231,11 @@
   resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee"
   integrity sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==
 
+"@types/lodash@^4.17.12":
+  version "4.17.12"
+  resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.17.12.tgz#25d71312bf66512105d71e55d42e22c36bcfc689"
+  integrity sha512-sviUmCE8AYdaF/KIHLDJBQgeYzPBI0vf/17NaYehBJfYD1j6/L95Slh07NlyK2iNyBNaEkb3En2jRt+a8y3xZQ==
+
 "@types/mime-types@^2.1.4":
   version "2.1.4"
   resolved "https://registry.yarnpkg.com/@types/mime-types/-/mime-types-2.1.4.tgz#93a1933e24fed4fb9e4adc5963a63efcbb3317a2"