From 2f75c130ffe0092bcb1d8b9a14b60c8a029514d6 Mon Sep 17 00:00:00 2001 From: Romain Mercier Date: Wed, 15 Jan 2025 16:30:27 +0100 Subject: [PATCH 01/23] feat(api): #3121 scdl integration script --- packages/api/package.json | 3 +- packages/api/src/@enums/FileExtensionEnum.ts | 5 + .../api/src/@types/ScdlDataIntegration.ts | 11 + .../configurations/scdlIntegration.conf.ts | 2 + .../scripts/ScdlDataIntegration.node.test.ts | 290 ++++++++++++++++++ .../scripts/ScdlDataIntegration.node.ts | 109 +++++++ packages/api/tsconfig.json | 2 +- 7 files changed, 420 insertions(+), 2 deletions(-) create mode 100644 packages/api/src/@enums/FileExtensionEnum.ts create mode 100644 packages/api/src/@types/ScdlDataIntegration.ts create mode 100644 packages/api/src/configurations/scdlIntegration.conf.ts create mode 100644 packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts create mode 100644 packages/api/src/interfaces/scripts/ScdlDataIntegration.node.ts diff --git a/packages/api/package.json b/packages/api/package.json index f70df09df..373a49c1c 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -27,7 +27,8 @@ "lint:all": "eslint --fix --ext .js,.ts ./{migrations,src,tests,tools}/**/*.{js,ts} ./{migrations,src,tests,tools}/*.{js,ts}", "prettier": "prettier --write", "prettier:all": "prettier --write ./{migrations,src,tests,tools}/**/*.{js,ts}", - "lint-format": "lint-staged" + "lint-format": "lint-staged", + "scdl-integration": "ts-node src/interfaces/scripts/ScdlDataIntegration.node.ts" }, "lint-staged": { "./{migrations,src,tests,tools}/**/*.{js,ts}": [ diff --git a/packages/api/src/@enums/FileExtensionEnum.ts b/packages/api/src/@enums/FileExtensionEnum.ts new file mode 100644 index 000000000..cd8dce9af --- /dev/null +++ b/packages/api/src/@enums/FileExtensionEnum.ts @@ -0,0 +1,5 @@ +export enum FileExtensionEnum { + CSV = "csv", + XLS = "xls", + XLSX = "xlsx", +} diff --git a/packages/api/src/@types/ScdlDataIntegration.ts b/packages/api/src/@types/ScdlDataIntegration.ts new file mode 100644 index 000000000..aba1d90ea --- /dev/null +++ b/packages/api/src/@types/ScdlDataIntegration.ts @@ -0,0 +1,11 @@ +export interface ScdlFileProcessingConfig { + name: string; + parseParams: string[]; + addProducer: boolean; + producerName?: string; + producerSiret?: string; +} + +export interface ScdlFileProcessingConfigList { + files: ScdlFileProcessingConfig[]; +} diff --git a/packages/api/src/configurations/scdlIntegration.conf.ts b/packages/api/src/configurations/scdlIntegration.conf.ts new file mode 100644 index 000000000..72dd3d13f --- /dev/null +++ b/packages/api/src/configurations/scdlIntegration.conf.ts @@ -0,0 +1,2 @@ +export const SCDL_FILE_PROCESSING_PATH = process.env.SCDL_FILE_PROCESSING_PATH || "./data-integration"; +export const SCDL_FILE_PROCESSING_CONFIG_FILENAME = "scdl-file-processing-config.json"; diff --git a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts new file mode 100644 index 000000000..0f3bb605b --- /dev/null +++ b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts @@ -0,0 +1,290 @@ +import fs from "fs"; +import path from "path"; +import { main, loadConfig, processFile } from "./ScdlDataIntegration.node"; +import { + SCDL_FILE_PROCESSING_PATH, + SCDL_FILE_PROCESSING_CONFIG_FILENAME, +} from "../../configurations/scdlIntegration.conf"; +import ScdlCli from "../cli/Scdl.cli"; + +jest.mock("../../configurations/scdlIntegration.conf", () => ({ + SCDL_FILE_PROCESSING_PATH: path.resolve(__dirname, "test-integration"), + SCDL_FILE_PROCESSING_CONFIG_FILENAME: "test-scdl-file-processing-config.json", +})); +jest.mock("../cli/Scdl.cli"); + +const validConfigData = { + files: [ + { + name: "donnees-a-integrer1.csv", + parseParams: ["producerSlug1", "2025-01-13"], + addProducer: true, + producerName: "Test Producer 1", + producerSiret: "12345678901", + }, + { + name: "donnees-a-integrer2.csv", + parseParams: ["producerSlug2", "2025-01-14"], + addProducer: true, + producerName: "Test Producer 2", + producerSiret: "12345678902", + }, + { + name: "donnees-a-integrer3.csv", + parseParams: ["producerSlug2", "2025-01-15"], + addProducer: false, + }, + ], +}; + +describe("scdl-data-integration.node", () => { + const testFilePath = path.join(SCDL_FILE_PROCESSING_PATH, SCDL_FILE_PROCESSING_CONFIG_FILENAME); + + let addProducerMock: jest.SpyInstance, [slug: string, name: string, siret: string]>; + let parseMock: jest.SpyInstance< + Promise, + [ + file: string, + producerSlug: string, + exportDate: string, + delimiter?: string | undefined, + quote?: string | undefined, + ] + >; + let parseXlsMock: jest.SpyInstance< + Promise, + [ + file: string, + producerSlug: string, + exportDate: string, + pageName?: string | undefined, + rowOffsetStr?: string | number | undefined, + ] + >; + + beforeAll(() => { + if (!fs.existsSync(SCDL_FILE_PROCESSING_PATH)) { + fs.mkdirSync(SCDL_FILE_PROCESSING_PATH); + } + }); + + afterAll(() => { + if (fs.existsSync(SCDL_FILE_PROCESSING_PATH)) { + fs.rmSync(SCDL_FILE_PROCESSING_PATH, { recursive: true }); + } + }); + + beforeEach(() => { + jest.spyOn(process, "exit").mockImplementation((() => {}) as (code?: number) => never); + addProducerMock = jest.spyOn(ScdlCli.prototype, "addProducer").mockResolvedValue(); + parseMock = jest.spyOn(ScdlCli.prototype, "parse").mockResolvedValue(); + parseXlsMock = jest.spyOn(ScdlCli.prototype, "parseXls").mockResolvedValue(); + }); + + afterEach(() => { + jest.restoreAllMocks(); + if (fs.existsSync(testFilePath)) { + fs.unlinkSync(testFilePath); + } + }); + + describe("loadConfig method", () => { + it("should load config file successfully", () => { + fs.writeFileSync(testFilePath, JSON.stringify(validConfigData)); + const result = loadConfig(); + + expect(result).toEqual(validConfigData); + }); + + it("should throw an error when the JSON file is invalid", () => { + const invalidJsonData = "{ invalidJson: true "; + + fs.writeFileSync(testFilePath, invalidJsonData); + + expect(() => { + loadConfig(); + }).toThrowError(new Error("Unexpected token i in JSON at position 2")); + }); + + it("should throw an error when the JSON file does not exist", () => { + expect(() => { + loadConfig(); + }).toThrowError( + expect.objectContaining({ + message: expect.stringContaining("ENOENT: no such file or directory"), + }), + ); + }); + }); + + describe("Test for processFile method", () => { + const fileConfig = { + name: "donnees-a-integrer1.csv", + parseParams: ["producerSlug1", "2025-01-13"], + addProducer: true, + producerName: "Test Producer 1", + producerSiret: "12345678901", + }; + + const fileConfigWrongType = { + name: "donnees-a-integrer1.doc", + parseParams: ["producerSlug1", "2025-01-13"], + addProducer: true, + producerName: "Test Producer 1", + producerSiret: "12345678901", + }; + + it("should process file methods correctly", async () => { + await expect(processFile(fileConfig)).resolves.toBeUndefined(); + + expect(addProducerMock).toHaveBeenCalledTimes(1); + expect(addProducerMock).toHaveBeenCalledWith( + fileConfig.parseParams[0], + fileConfig.producerName, + fileConfig.producerSiret, + ); + + expect(parseMock).toHaveBeenCalledTimes(1); + expect(parseMock).toHaveBeenCalledWith( + expect.stringContaining(fileConfig.name), + fileConfig.parseParams[0], + fileConfig.parseParams[1], + undefined, + undefined, + ); + expect(parseXlsMock).not.toHaveBeenCalled(); + }); + + it("should catch error and not call parse method", async () => { + addProducerMock = jest + .spyOn(ScdlCli.prototype, "addProducer") + .mockRejectedValue(new Error("Mocked addProducer error")); + await expect(processFile(fileConfig)).resolves.toBeUndefined(); + expect(addProducerMock).toHaveBeenCalledTimes(1); + expect(parseMock).not.toHaveBeenCalled(); + expect(parseXlsMock).not.toHaveBeenCalled(); + }); + + it("should catch Unsupported file type error and not call parse method", async () => { + await expect(processFile(fileConfigWrongType)).resolves.toBeUndefined(); + expect(addProducerMock).toHaveBeenCalledTimes(1); + expect(parseMock).not.toHaveBeenCalled(); + expect(parseXlsMock).not.toHaveBeenCalled(); + }); + + it("should catch Producer already exists error and call parse method", async () => { + addProducerMock = jest + .spyOn(ScdlCli.prototype, "addProducer") + .mockRejectedValue(new Error("Producer already exists")); + + await expect(processFile(fileConfig)).resolves.toBeUndefined(); + expect(addProducerMock).toHaveBeenCalledTimes(1); + expect(parseMock).toHaveBeenCalledTimes(1); + expect(parseXlsMock).not.toHaveBeenCalled(); + }); + }); + + describe("Test for main script", () => { + it("should call ScdlCli methods with correct arguments", async () => { + fs.writeFileSync(testFilePath, JSON.stringify(validConfigData)); + + await expect(main()).resolves.toBeUndefined(); + + expect(addProducerMock).toHaveBeenCalledTimes(2); + expect(addProducerMock).toHaveBeenCalledWith("producerSlug1", "Test Producer 1", "12345678901"); + expect(addProducerMock).toHaveBeenCalledWith("producerSlug2", "Test Producer 2", "12345678902"); + + expect(parseMock).toHaveBeenCalledTimes(3); + expect(parseMock).toHaveBeenCalledWith( + path.join(SCDL_FILE_PROCESSING_PATH, "donnees-a-integrer1.csv"), + "producerSlug1", + "2025-01-13", + undefined, + undefined, + ); + expect(parseMock).toHaveBeenCalledWith( + path.join(SCDL_FILE_PROCESSING_PATH, "donnees-a-integrer2.csv"), + "producerSlug2", + "2025-01-14", + undefined, + undefined, + ); + expect(parseMock).toHaveBeenCalledWith( + path.join(SCDL_FILE_PROCESSING_PATH, "donnees-a-integrer3.csv"), + "producerSlug2", + "2025-01-15", + undefined, + undefined, + ); + expect(parseXlsMock).not.toHaveBeenCalled(); + }); + + it("should throw Unexpected token error", async () => { + const invalidJsonData = "{ invalidJson: true "; + + fs.writeFileSync(testFilePath, invalidJsonData); + await expect(main()).rejects.toThrow("Unexpected token i in JSON at position 2"); + expect(addProducerMock).not.toHaveBeenCalled(); + expect(parseMock).not.toHaveBeenCalled(); + expect(parseXlsMock).not.toHaveBeenCalled(); + }); + + it("should throw Invalid configuration file error", async () => { + const invalidConfigData = { + files: [ + { + name: "donnees-a-integrer1.csv", + addProducer: true, + producerName: "Test Producer 1", + producerSiret: "12345678901", + }, + ], + }; + fs.writeFileSync(testFilePath, JSON.stringify(invalidConfigData)); + + await expect(main()).rejects.toThrow( + "Invalid configuration file: The config does not match the expected structure.", + ); + expect(addProducerMock).not.toHaveBeenCalled(); + expect(parseMock).not.toHaveBeenCalled(); + expect(parseXlsMock).not.toHaveBeenCalled(); + }); + + it("when error with one file, should process continue for other files", async () => { + const configData = { + files: [ + { + name: "donnees-a-integrer1.xlsx", + parseParams: ["producerSlug1", "2025-01-13"], + addProducer: true, + producerName: "Test Producer 1", + producerSiret: "12345678901", + }, + { + name: "donnees-a-integrer2.csv", + parseParams: ["producerSlug2", "2025-01-14"], + addProducer: true, + producerName: "Test Producer 2", + producerSiret: "12345678902", + }, + { + name: "donnees-a-integrer3.csv", + parseParams: ["producerSlug3", "2025-01-15"], + addProducer: true, + producerName: "Test Producer 3", + producerSiret: "12345678903", + }, + ], + }; + fs.writeFileSync(testFilePath, JSON.stringify(configData)); + parseXlsMock = jest + .spyOn(ScdlCli.prototype, "parseXls") + .mockRejectedValue(new Error("Mocked addProducer error")); + + await expect(main()).resolves.toBeUndefined(); + expect(addProducerMock).toHaveBeenCalledTimes(3); + expect(parseMock).toHaveBeenCalledTimes(2); + expect(parseXlsMock).toHaveBeenCalledTimes(1); + }); + }); +}); diff --git a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.ts b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.ts new file mode 100644 index 000000000..e850b117d --- /dev/null +++ b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.ts @@ -0,0 +1,109 @@ +import fs from "fs"; +import path from "path"; +import ScdlCli from "../cli/Scdl.cli"; +import { + SCDL_FILE_PROCESSING_CONFIG_FILENAME, + SCDL_FILE_PROCESSING_PATH, +} from "../../configurations/scdlIntegration.conf"; +import { FileExtensionEnum } from "../../@enums/FileExtensionEnum"; +import { ScdlFileProcessingConfig, ScdlFileProcessingConfigList } from "../../@types/ScdlDataIntegration"; + +const scdlCli = new ScdlCli(); +const successList: string[] = []; +const errorList: string[] = []; + +const isConfig = (obj: any): obj is ScdlFileProcessingConfigList => { + return obj && Array.isArray(obj.files) && obj.files.every(file => isFileConfig(file)); +}; +const isFileConfig = (file: any): file is ScdlFileProcessingConfig => { + return ( + file && + typeof file.name === "string" && + Array.isArray(file.parseParams) && + typeof file.addProducer === "boolean" + ); +}; + +export const loadConfig = (): ScdlFileProcessingConfigList => { + const filePath = path.resolve(SCDL_FILE_PROCESSING_PATH, SCDL_FILE_PROCESSING_CONFIG_FILENAME); + const data = fs.readFileSync(filePath, "utf8"); + return JSON.parse(data) as ScdlFileProcessingConfigList; +}; + +export const processFile = async (fileInfo: ScdlFileProcessingConfig) => { + const { name, parseParams, addProducer, producerName, producerSiret } = fileInfo; + const dirPath = path.resolve(SCDL_FILE_PROCESSING_PATH); + const [producerSlug, exportDate, ...optionalParams] = parseParams; + + try { + if (addProducer && producerName && producerSiret) { + await scdlCli.addProducer(producerSlug, producerName, producerSiret); + successList.push(`added producer ${producerSlug}`); + } + } catch (e) { + if ((e as Error).message !== "Producer already exists") { + errorList.push( + `added producer ${producerSlug} for file ${name}, data parsing not performed : ${(e as Error).message}`, + ); + return; + } + } + + try { + const type = path.extname(name).slice(1).toLowerCase(); + const filePath = path.join(dirPath, name); + if (type === FileExtensionEnum.CSV) { + const [delimiter = undefined, quote = undefined] = optionalParams; + await scdlCli.parse(filePath, producerSlug, exportDate, delimiter, quote); + } else if (type === FileExtensionEnum.XLS || type === FileExtensionEnum.XLSX) { + const [pageName = undefined, rowOffsetStr = undefined] = optionalParams; + await scdlCli.parseXls(filePath, producerSlug, exportDate, pageName, rowOffsetStr); + } else { + console.error(`❌ Unsupported file type : ${name} (type: ${type})`); + throw new Error(`Unsupported file type : ${filePath}`); + } + successList.push(`parse data of ${producerSlug} for file ${name}`); + } catch (e) { + errorList.push(`parse data of ${producerSlug} for file ${name} : ${(e as Error).message}`); + } +}; + +/** + * main function to load the file listing .json and launch data integration + */ +export const main = async () => { + try { + const config: ScdlFileProcessingConfigList = loadConfig(); + + if (!isConfig(config)) { + throw new Error("Invalid configuration file: The config does not match the expected structure."); + } + const filePromises = config.files.map(processFile); + await Promise.all(filePromises); + } catch (e) { + console.log("❌ Global execution failure.", e); + errorList.push(`Global execution failure : ${(e as Error).message}`); + throw e; + } finally { + console.log("\n---------------Summary of Operations---------------"); + if (errorList.length === 0) { + { + console.log("🚀 All operations completed successfully! 🎯"); + } + } + if (successList.length > 0) { + console.log("✅ list of Success :"); + successList.forEach(desc => console.log(`➡️ ${desc}`)); + } + + if (errorList.length > 0) { + console.log("⚠️ List of Errors :"); + errorList.forEach(desc => console.log(`❌ ${desc}`)); + } + process.exit(0); + } +}; + +if (require.main === module) { + main(); +} diff --git a/packages/api/tsconfig.json b/packages/api/tsconfig.json index 3975755e6..ce9a8d665 100644 --- a/packages/api/tsconfig.json +++ b/packages/api/tsconfig.json @@ -2,7 +2,7 @@ "compilerOptions": { "target": "es2021", "module": "commonjs", - "types": ["reflect-metadata", "jest"], + "types": ["reflect-metadata", "jest", "node"], "paths": { "~": ["./src"] }, From 3944c13dad6dea0453ddb66a4115f708dcf0cdba Mon Sep 17 00:00:00 2001 From: Romain Mercier Date: Thu, 16 Jan 2025 11:56:03 +0100 Subject: [PATCH 02/23] feat(api): #3121 scdl integration script --- packages/api/tsconfig.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/tsconfig.json b/packages/api/tsconfig.json index ce9a8d665..3975755e6 100644 --- a/packages/api/tsconfig.json +++ b/packages/api/tsconfig.json @@ -2,7 +2,7 @@ "compilerOptions": { "target": "es2021", "module": "commonjs", - "types": ["reflect-metadata", "jest", "node"], + "types": ["reflect-metadata", "jest"], "paths": { "~": ["./src"] }, From 45ac075300ead2669cdd67486d1742c5a8b5d70d Mon Sep 17 00:00:00 2001 From: Romain Mercier Date: Thu, 16 Jan 2025 11:58:02 +0100 Subject: [PATCH 03/23] feat(api): #3121 scdl integration script - renamed script --- packages/api/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/package.json b/packages/api/package.json index 373a49c1c..923913abe 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -28,7 +28,7 @@ "prettier": "prettier --write", "prettier:all": "prettier --write ./{migrations,src,tests,tools}/**/*.{js,ts}", "lint-format": "lint-staged", - "scdl-integration": "ts-node src/interfaces/scripts/ScdlDataIntegration.node.ts" + "scdl:import": "ts-node src/interfaces/scripts/ScdlDataIntegration.node.ts" }, "lint-staged": { "./{migrations,src,tests,tools}/**/*.{js,ts}": [ From 068ab9113810b1f067aaea6a4f1e448b58006e55 Mon Sep 17 00:00:00 2001 From: Romain Mercier Date: Thu, 16 Jan 2025 12:11:04 +0100 Subject: [PATCH 04/23] feat(api): #3121 scdl integration script - added types for parseParams --- packages/api/src/@types/ScdlDataIntegration.ts | 11 ++++++++++- .../scripts/ScdlDataIntegration.node.test.ts | 11 ++++++----- .../interfaces/scripts/ScdlDataIntegration.node.ts | 3 ++- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/packages/api/src/@types/ScdlDataIntegration.ts b/packages/api/src/@types/ScdlDataIntegration.ts index aba1d90ea..f8a5a8d83 100644 --- a/packages/api/src/@types/ScdlDataIntegration.ts +++ b/packages/api/src/@types/ScdlDataIntegration.ts @@ -1,6 +1,15 @@ +export type ScdlParseArgs = [producerSlug: string, exportDate: string, delimiter?: string, quote?: string]; + +export type ScdlParseXlsArgs = [ + producerSlug: string, + exportDate: string, + pageName?: string, + rowOffsetStr?: number | string, +]; + export interface ScdlFileProcessingConfig { name: string; - parseParams: string[]; + parseParams: ScdlParseArgs | ScdlParseXlsArgs; addProducer: boolean; producerName?: string; producerSiret?: string; diff --git a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts index 0f3bb605b..6fbe3e4a9 100644 --- a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts +++ b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts @@ -6,6 +6,7 @@ import { SCDL_FILE_PROCESSING_CONFIG_FILENAME, } from "../../configurations/scdlIntegration.conf"; import ScdlCli from "../cli/Scdl.cli"; +import { ScdlParseArgs, ScdlParseXlsArgs } from "../../@types/ScdlDataIntegration"; jest.mock("../../configurations/scdlIntegration.conf", () => ({ SCDL_FILE_PROCESSING_PATH: path.resolve(__dirname, "test-integration"), @@ -120,7 +121,7 @@ describe("scdl-data-integration.node", () => { describe("Test for processFile method", () => { const fileConfig = { name: "donnees-a-integrer1.csv", - parseParams: ["producerSlug1", "2025-01-13"], + parseParams: ["producerSlug1", "2025-01-13"] as ScdlParseArgs, addProducer: true, producerName: "Test Producer 1", producerSiret: "12345678901", @@ -128,7 +129,7 @@ describe("scdl-data-integration.node", () => { const fileConfigWrongType = { name: "donnees-a-integrer1.doc", - parseParams: ["producerSlug1", "2025-01-13"], + parseParams: ["producerSlug1", "2025-01-13"] as ScdlParseXlsArgs, addProducer: true, producerName: "Test Producer 1", producerSiret: "12345678901", @@ -255,21 +256,21 @@ describe("scdl-data-integration.node", () => { files: [ { name: "donnees-a-integrer1.xlsx", - parseParams: ["producerSlug1", "2025-01-13"], + parseParams: ["producerSlug1", "2025-01-13"] as ScdlParseXlsArgs, addProducer: true, producerName: "Test Producer 1", producerSiret: "12345678901", }, { name: "donnees-a-integrer2.csv", - parseParams: ["producerSlug2", "2025-01-14"], + parseParams: ["producerSlug2", "2025-01-14"] as ScdlParseArgs, addProducer: true, producerName: "Test Producer 2", producerSiret: "12345678902", }, { name: "donnees-a-integrer3.csv", - parseParams: ["producerSlug3", "2025-01-15"], + parseParams: ["producerSlug3", "2025-01-15"] as ScdlParseArgs, addProducer: true, producerName: "Test Producer 3", producerSiret: "12345678903", diff --git a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.ts b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.ts index e850b117d..ce31390fa 100644 --- a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.ts +++ b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.ts @@ -53,7 +53,8 @@ export const processFile = async (fileInfo: ScdlFileProcessingConfig) => { const type = path.extname(name).slice(1).toLowerCase(); const filePath = path.join(dirPath, name); if (type === FileExtensionEnum.CSV) { - const [delimiter = undefined, quote = undefined] = optionalParams; + const delimiter = optionalParams[0]; + const quote = typeof optionalParams[1] === "string" ? optionalParams[1] : undefined; await scdlCli.parse(filePath, producerSlug, exportDate, delimiter, quote); } else if (type === FileExtensionEnum.XLS || type === FileExtensionEnum.XLSX) { const [pageName = undefined, rowOffsetStr = undefined] = optionalParams; From 744fb6659cf02a7717dbf22a8b1683cc25af80e2 Mon Sep 17 00:00:00 2001 From: Romain Mercier Date: Thu, 16 Jan 2025 14:48:49 +0100 Subject: [PATCH 05/23] feat(api): #3121 scdl integration script - rename variable --- packages/api/src/configurations/scdlIntegration.conf.ts | 2 +- .../api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/api/src/configurations/scdlIntegration.conf.ts b/packages/api/src/configurations/scdlIntegration.conf.ts index 72dd3d13f..9773ccf7b 100644 --- a/packages/api/src/configurations/scdlIntegration.conf.ts +++ b/packages/api/src/configurations/scdlIntegration.conf.ts @@ -1,2 +1,2 @@ export const SCDL_FILE_PROCESSING_PATH = process.env.SCDL_FILE_PROCESSING_PATH || "./data-integration"; -export const SCDL_FILE_PROCESSING_CONFIG_FILENAME = "scdl-file-processing-config.json"; +export const SCDL_FILE_PROCESSING_CONFIG_FILENAME = "scdl-file-processing.config.json"; diff --git a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts index 6fbe3e4a9..59c240117 100644 --- a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts +++ b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts @@ -10,7 +10,7 @@ import { ScdlParseArgs, ScdlParseXlsArgs } from "../../@types/ScdlDataIntegratio jest.mock("../../configurations/scdlIntegration.conf", () => ({ SCDL_FILE_PROCESSING_PATH: path.resolve(__dirname, "test-integration"), - SCDL_FILE_PROCESSING_CONFIG_FILENAME: "test-scdl-file-processing-config.json", + SCDL_FILE_PROCESSING_CONFIG_FILENAME: "test-scdl-file-processing.config.json", })); jest.mock("../cli/Scdl.cli"); From 208be3948dab500d6890b1ea85801a84573792c9 Mon Sep 17 00:00:00 2001 From: Romain Mercier Date: Thu, 16 Jan 2025 15:12:56 +0100 Subject: [PATCH 06/23] feat(api): #3121 scdl integration script - corrected type error --- .../api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts index 59c240117..cdb65e311 100644 --- a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts +++ b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts @@ -76,7 +76,7 @@ describe("scdl-data-integration.node", () => { }); beforeEach(() => { - jest.spyOn(process, "exit").mockImplementation((() => {}) as (code?: number) => never); + jest.spyOn(process, "exit").mockImplementation((() => {}) as (code?: any) => never); addProducerMock = jest.spyOn(ScdlCli.prototype, "addProducer").mockResolvedValue(); parseMock = jest.spyOn(ScdlCli.prototype, "parse").mockResolvedValue(); parseXlsMock = jest.spyOn(ScdlCli.prototype, "parseXls").mockResolvedValue(); From d54200c5f281dff9ad62f835cdaefc8c892d2af2 Mon Sep 17 00:00:00 2001 From: Romain Mercier Date: Tue, 28 Jan 2025 14:12:42 +0100 Subject: [PATCH 07/23] feat(api): #3121 scdl integration script - rename variable --- packages/api/src/@types/ScdlDataIntegration.ts | 2 +- packages/api/src/interfaces/cli/Scdl.cli.ts | 6 +++--- .../src/interfaces/scripts/ScdlDataIntegration.node.test.ts | 2 +- .../api/src/interfaces/scripts/ScdlDataIntegration.node.ts | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/api/src/@types/ScdlDataIntegration.ts b/packages/api/src/@types/ScdlDataIntegration.ts index f8a5a8d83..d341addd8 100644 --- a/packages/api/src/@types/ScdlDataIntegration.ts +++ b/packages/api/src/@types/ScdlDataIntegration.ts @@ -4,7 +4,7 @@ export type ScdlParseXlsArgs = [ producerSlug: string, exportDate: string, pageName?: string, - rowOffsetStr?: number | string, + rowOffset?: number | string, ]; export interface ScdlFileProcessingConfig { diff --git a/packages/api/src/interfaces/cli/Scdl.cli.ts b/packages/api/src/interfaces/cli/Scdl.cli.ts index fb9be8a77..4ee3e528e 100644 --- a/packages/api/src/interfaces/cli/Scdl.cli.ts +++ b/packages/api/src/interfaces/cli/Scdl.cli.ts @@ -32,12 +32,12 @@ export default class ScdlCli { producerSlug: string, exportDate: string, pageName?: string, - rowOffsetStr: number | string = 0, + rowOffset: number | string = 0, ) { await this.validateGenericInput(file, producerSlug, exportDate); - const rowOffset = typeof rowOffsetStr === "number" ? rowOffsetStr : parseInt(rowOffsetStr); + const parsedRowOffset = typeof rowOffset === "number" ? rowOffset : parseInt(rowOffset); const fileContent = fs.readFileSync(file); - const { entities, errors } = ScdlGrantParser.parseExcel(fileContent, pageName, rowOffset); + const { entities, errors } = ScdlGrantParser.parseExcel(fileContent, pageName, parsedRowOffset); await Promise.all([ this.persistEntities(entities, producerSlug, exportDate as string), this.exportErrors(errors, file), diff --git a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts index cdb65e311..1397ea6b1 100644 --- a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts +++ b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts @@ -59,7 +59,7 @@ describe("scdl-data-integration.node", () => { producerSlug: string, exportDate: string, pageName?: string | undefined, - rowOffsetStr?: string | number | undefined, + rowOffset?: string | number | undefined, ] >; diff --git a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.ts b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.ts index ce31390fa..ef84086e0 100644 --- a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.ts +++ b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.ts @@ -57,8 +57,8 @@ export const processFile = async (fileInfo: ScdlFileProcessingConfig) => { const quote = typeof optionalParams[1] === "string" ? optionalParams[1] : undefined; await scdlCli.parse(filePath, producerSlug, exportDate, delimiter, quote); } else if (type === FileExtensionEnum.XLS || type === FileExtensionEnum.XLSX) { - const [pageName = undefined, rowOffsetStr = undefined] = optionalParams; - await scdlCli.parseXls(filePath, producerSlug, exportDate, pageName, rowOffsetStr); + const [pageName = undefined, rowOffset = undefined] = optionalParams; + await scdlCli.parseXls(filePath, producerSlug, exportDate, pageName, rowOffset); } else { console.error(`❌ Unsupported file type : ${name} (type: ${type})`); throw new Error(`Unsupported file type : ${filePath}`); From 9581c11f9f5b86d8ccf992e01ca2c947b28b5b21 Mon Sep 17 00:00:00 2001 From: Romain Mercier Date: Tue, 28 Jan 2025 14:33:05 +0100 Subject: [PATCH 08/23] feat(api): #3121 scdl integration script - type factorization --- packages/api/src/@types/ScdlDataIntegration.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/packages/api/src/@types/ScdlDataIntegration.ts b/packages/api/src/@types/ScdlDataIntegration.ts index d341addd8..96bc7c108 100644 --- a/packages/api/src/@types/ScdlDataIntegration.ts +++ b/packages/api/src/@types/ScdlDataIntegration.ts @@ -1,11 +1,6 @@ -export type ScdlParseArgs = [producerSlug: string, exportDate: string, delimiter?: string, quote?: string]; - -export type ScdlParseXlsArgs = [ - producerSlug: string, - exportDate: string, - pageName?: string, - rowOffset?: number | string, -]; +type BaseScdlParseArgs = [producerSlug: string, exportDate: string]; +export type ScdlParseArgs = BaseScdlParseArgs & [delimiter?: string, quote?: string]; +export type ScdlParseXlsArgs = BaseScdlParseArgs & [pageName?: string, rowOffset?: number | string]; export interface ScdlFileProcessingConfig { name: string; From 304b1557dc8ec780567a05207f748e6d0c226d07 Mon Sep 17 00:00:00 2001 From: Romain Mercier Date: Tue, 28 Jan 2025 14:36:59 +0100 Subject: [PATCH 09/23] feat(api): #3121 scdl integration script - explicited type --- .../src/interfaces/scripts/ScdlDataIntegration.node.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts index 1397ea6b1..d6da9514e 100644 --- a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts +++ b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts @@ -6,7 +6,7 @@ import { SCDL_FILE_PROCESSING_CONFIG_FILENAME, } from "../../configurations/scdlIntegration.conf"; import ScdlCli from "../cli/Scdl.cli"; -import { ScdlParseArgs, ScdlParseXlsArgs } from "../../@types/ScdlDataIntegration"; +import { ScdlFileProcessingConfigList, ScdlParseArgs, ScdlParseXlsArgs } from "../../@types/ScdlDataIntegration"; jest.mock("../../configurations/scdlIntegration.conf", () => ({ SCDL_FILE_PROCESSING_PATH: path.resolve(__dirname, "test-integration"), @@ -14,7 +14,7 @@ jest.mock("../../configurations/scdlIntegration.conf", () => ({ })); jest.mock("../cli/Scdl.cli"); -const validConfigData = { +const validConfigData: ScdlFileProcessingConfigList = { files: [ { name: "donnees-a-integrer1.csv", From e22f88e8478d906a2bbcc8e01240edae90bd49c7 Mon Sep 17 00:00:00 2001 From: Romain Mercier Date: Tue, 28 Jan 2025 14:50:51 +0100 Subject: [PATCH 10/23] feat(api): #3121 scdl integration script - explicited describe name --- .../api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts index d6da9514e..a2f5deb27 100644 --- a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts +++ b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts @@ -38,7 +38,7 @@ const validConfigData: ScdlFileProcessingConfigList = { ], }; -describe("scdl-data-integration.node", () => { +describe("scdl data integration script", () => { const testFilePath = path.join(SCDL_FILE_PROCESSING_PATH, SCDL_FILE_PROCESSING_CONFIG_FILENAME); let addProducerMock: jest.SpyInstance, [slug: string, name: string, siret: string]>; From 238708488b5f03b3a1663ec3d18bdc927342f88b Mon Sep 17 00:00:00 2001 From: Romain Mercier Date: Tue, 28 Jan 2025 16:09:20 +0100 Subject: [PATCH 11/23] feat(api): #3121 scdl integration script - mocked fs --- .../scripts/ScdlDataIntegration.node.test.ts | 38 +++++++------------ 1 file changed, 14 insertions(+), 24 deletions(-) diff --git a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts index a2f5deb27..73e5e16d8 100644 --- a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts +++ b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts @@ -13,6 +13,7 @@ jest.mock("../../configurations/scdlIntegration.conf", () => ({ SCDL_FILE_PROCESSING_CONFIG_FILENAME: "test-scdl-file-processing.config.json", })); jest.mock("../cli/Scdl.cli"); +jest.mock("fs"); const validConfigData: ScdlFileProcessingConfigList = { files: [ @@ -63,18 +64,6 @@ describe("scdl data integration script", () => { ] >; - beforeAll(() => { - if (!fs.existsSync(SCDL_FILE_PROCESSING_PATH)) { - fs.mkdirSync(SCDL_FILE_PROCESSING_PATH); - } - }); - - afterAll(() => { - if (fs.existsSync(SCDL_FILE_PROCESSING_PATH)) { - fs.rmSync(SCDL_FILE_PROCESSING_PATH, { recursive: true }); - } - }); - beforeEach(() => { jest.spyOn(process, "exit").mockImplementation((() => {}) as (code?: any) => never); addProducerMock = jest.spyOn(ScdlCli.prototype, "addProducer").mockResolvedValue(); @@ -84,23 +73,20 @@ describe("scdl data integration script", () => { afterEach(() => { jest.restoreAllMocks(); - if (fs.existsSync(testFilePath)) { - fs.unlinkSync(testFilePath); - } }); describe("loadConfig method", () => { it("should load config file successfully", () => { - fs.writeFileSync(testFilePath, JSON.stringify(validConfigData)); + jest.spyOn(fs, "readFileSync").mockReturnValueOnce(JSON.stringify(validConfigData)); const result = loadConfig(); expect(result).toEqual(validConfigData); }); it("should throw an error when the JSON file is invalid", () => { - const invalidJsonData = "{ invalidJson: true "; - - fs.writeFileSync(testFilePath, invalidJsonData); + jest.spyOn(fs, "readFileSync").mockImplementationOnce(() => { + throw new Error("Unexpected token i in JSON at position 2"); + }); expect(() => { loadConfig(); @@ -108,6 +94,9 @@ describe("scdl data integration script", () => { }); it("should throw an error when the JSON file does not exist", () => { + jest.spyOn(fs, "readFileSync").mockImplementationOnce(() => { + throw new Error("ENOENT: no such file or directory"); + }); expect(() => { loadConfig(); }).toThrowError( @@ -187,7 +176,7 @@ describe("scdl data integration script", () => { describe("Test for main script", () => { it("should call ScdlCli methods with correct arguments", async () => { - fs.writeFileSync(testFilePath, JSON.stringify(validConfigData)); + jest.spyOn(fs, "readFileSync").mockReturnValueOnce(JSON.stringify(validConfigData)); await expect(main()).resolves.toBeUndefined(); @@ -221,9 +210,9 @@ describe("scdl data integration script", () => { }); it("should throw Unexpected token error", async () => { - const invalidJsonData = "{ invalidJson: true "; - - fs.writeFileSync(testFilePath, invalidJsonData); + jest.spyOn(fs, "readFileSync").mockImplementationOnce(() => { + throw new Error("Unexpected token i in JSON at position 2"); + }); await expect(main()).rejects.toThrow("Unexpected token i in JSON at position 2"); expect(addProducerMock).not.toHaveBeenCalled(); expect(parseMock).not.toHaveBeenCalled(); @@ -241,6 +230,7 @@ describe("scdl data integration script", () => { }, ], }; + jest.spyOn(fs, "readFileSync").mockReturnValueOnce(JSON.stringify(invalidConfigData)); fs.writeFileSync(testFilePath, JSON.stringify(invalidConfigData)); await expect(main()).rejects.toThrow( @@ -277,7 +267,7 @@ describe("scdl data integration script", () => { }, ], }; - fs.writeFileSync(testFilePath, JSON.stringify(configData)); + jest.spyOn(fs, "readFileSync").mockReturnValueOnce(JSON.stringify(configData)); parseXlsMock = jest .spyOn(ScdlCli.prototype, "parseXls") .mockRejectedValue(new Error("Mocked addProducer error")); From ecd6dfe819d7f4601c421a990dc0d4904422db5a Mon Sep 17 00:00:00 2001 From: Romain Mercier Date: Tue, 28 Jan 2025 16:13:33 +0100 Subject: [PATCH 12/23] feat(api): #3121 scdl integration script - del brackets --- .../src/interfaces/scripts/ScdlDataIntegration.node.test.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts index 73e5e16d8..1995207f3 100644 --- a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts +++ b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts @@ -88,9 +88,7 @@ describe("scdl data integration script", () => { throw new Error("Unexpected token i in JSON at position 2"); }); - expect(() => { - loadConfig(); - }).toThrowError(new Error("Unexpected token i in JSON at position 2")); + expect(() => loadConfig()).toThrowError(new Error("Unexpected token i in JSON at position 2")); }); it("should throw an error when the JSON file does not exist", () => { From 11101b04b23a1f0edf4c5ff7b62f80c0d20bdc57 Mon Sep 17 00:00:00 2001 From: Romain Mercier Date: Tue, 28 Jan 2025 16:47:29 +0100 Subject: [PATCH 13/23] feat(api): #3121 scdl integration script - refacto to test on thing in each test --- .../scripts/ScdlDataIntegration.node.test.ts | 60 +++++++++++-------- 1 file changed, 36 insertions(+), 24 deletions(-) diff --git a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts index 1995207f3..f10fe3ede 100644 --- a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts +++ b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts @@ -95,13 +95,7 @@ describe("scdl data integration script", () => { jest.spyOn(fs, "readFileSync").mockImplementationOnce(() => { throw new Error("ENOENT: no such file or directory"); }); - expect(() => { - loadConfig(); - }).toThrowError( - expect.objectContaining({ - message: expect.stringContaining("ENOENT: no such file or directory"), - }), - ); + expect(() => loadConfig()).toThrowError("ENOENT: no such file or directory"); }); }); @@ -122,17 +116,19 @@ describe("scdl data integration script", () => { producerSiret: "12345678901", }; - it("should process file methods correctly", async () => { + it("should process file methods correctly with addProducer Called with correct params", async () => { await expect(processFile(fileConfig)).resolves.toBeUndefined(); - expect(addProducerMock).toHaveBeenCalledTimes(1); expect(addProducerMock).toHaveBeenCalledWith( fileConfig.parseParams[0], fileConfig.producerName, fileConfig.producerSiret, ); + }); + + it("should process file methods correctly with parse method Called with correct params", async () => { + await expect(processFile(fileConfig)).resolves.toBeUndefined(); - expect(parseMock).toHaveBeenCalledTimes(1); expect(parseMock).toHaveBeenCalledWith( expect.stringContaining(fileConfig.name), fileConfig.parseParams[0], @@ -140,35 +136,57 @@ describe("scdl data integration script", () => { undefined, undefined, ); + }); + + it("should process file methods correctly with parseXls method not to have been Called", async () => { + await expect(processFile(fileConfig)).resolves.toBeUndefined(); expect(parseXlsMock).not.toHaveBeenCalled(); }); - it("should catch error and not call parse method", async () => { + it("should catch error", async () => { + addProducerMock = jest + .spyOn(ScdlCli.prototype, "addProducer") + .mockRejectedValue(new Error("Mocked addProducer error")); + await expect(processFile(fileConfig)).resolves.toBeUndefined(); + }); + + it("should not call parse method when addProducer error", async () => { addProducerMock = jest .spyOn(ScdlCli.prototype, "addProducer") .mockRejectedValue(new Error("Mocked addProducer error")); await expect(processFile(fileConfig)).resolves.toBeUndefined(); - expect(addProducerMock).toHaveBeenCalledTimes(1); expect(parseMock).not.toHaveBeenCalled(); - expect(parseXlsMock).not.toHaveBeenCalled(); }); - it("should catch Unsupported file type error and not call parse method", async () => { + it("should catch Unsupported file type error", async () => { + await expect(processFile(fileConfigWrongType)).resolves.toBeUndefined(); + }); + + it("should not call parse method when wrong file type", async () => { await expect(processFile(fileConfigWrongType)).resolves.toBeUndefined(); - expect(addProducerMock).toHaveBeenCalledTimes(1); expect(parseMock).not.toHaveBeenCalled(); + }); + + it("should not call parseXls method when wrong file type", async () => { + await expect(processFile(fileConfigWrongType)).resolves.toBeUndefined(); expect(parseXlsMock).not.toHaveBeenCalled(); }); - it("should catch Producer already exists error and call parse method", async () => { + it("should catch Producer already exists error", async () => { + addProducerMock = jest + .spyOn(ScdlCli.prototype, "addProducer") + .mockRejectedValue(new Error("Producer already exists")); + + await expect(processFile(fileConfig)).resolves.toBeUndefined(); + }); + + it("should call parse method when producer already exists", async () => { addProducerMock = jest .spyOn(ScdlCli.prototype, "addProducer") .mockRejectedValue(new Error("Producer already exists")); await expect(processFile(fileConfig)).resolves.toBeUndefined(); - expect(addProducerMock).toHaveBeenCalledTimes(1); expect(parseMock).toHaveBeenCalledTimes(1); - expect(parseXlsMock).not.toHaveBeenCalled(); }); }); @@ -212,9 +230,6 @@ describe("scdl data integration script", () => { throw new Error("Unexpected token i in JSON at position 2"); }); await expect(main()).rejects.toThrow("Unexpected token i in JSON at position 2"); - expect(addProducerMock).not.toHaveBeenCalled(); - expect(parseMock).not.toHaveBeenCalled(); - expect(parseXlsMock).not.toHaveBeenCalled(); }); it("should throw Invalid configuration file error", async () => { @@ -234,9 +249,6 @@ describe("scdl data integration script", () => { await expect(main()).rejects.toThrow( "Invalid configuration file: The config does not match the expected structure.", ); - expect(addProducerMock).not.toHaveBeenCalled(); - expect(parseMock).not.toHaveBeenCalled(); - expect(parseXlsMock).not.toHaveBeenCalled(); }); it("when error with one file, should process continue for other files", async () => { From 4ffaf5a581a7fbd9d9ee089c49fef6e3f3d716f9 Mon Sep 17 00:00:00 2001 From: Romain Mercier Date: Tue, 28 Jan 2025 16:50:04 +0100 Subject: [PATCH 14/23] feat(api): #3121 scdl integration script - renamed var --- .../src/interfaces/scripts/ScdlDataIntegration.node.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.ts b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.ts index ef84086e0..d1e703f24 100644 --- a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.ts +++ b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.ts @@ -50,17 +50,17 @@ export const processFile = async (fileInfo: ScdlFileProcessingConfig) => { } try { - const type = path.extname(name).slice(1).toLowerCase(); + const fileType = path.extname(name).slice(1).toLowerCase(); const filePath = path.join(dirPath, name); - if (type === FileExtensionEnum.CSV) { + if (fileType === FileExtensionEnum.CSV) { const delimiter = optionalParams[0]; const quote = typeof optionalParams[1] === "string" ? optionalParams[1] : undefined; await scdlCli.parse(filePath, producerSlug, exportDate, delimiter, quote); - } else if (type === FileExtensionEnum.XLS || type === FileExtensionEnum.XLSX) { + } else if (fileType === FileExtensionEnum.XLS || fileType === FileExtensionEnum.XLSX) { const [pageName = undefined, rowOffset = undefined] = optionalParams; await scdlCli.parseXls(filePath, producerSlug, exportDate, pageName, rowOffset); } else { - console.error(`❌ Unsupported file type : ${name} (type: ${type})`); + console.error(`❌ Unsupported file type : ${name} (type: ${fileType})`); throw new Error(`Unsupported file type : ${filePath}`); } successList.push(`parse data of ${producerSlug} for file ${name}`); From cf533e0645007e64df4bd0acf3f8893022fc27d9 Mon Sep 17 00:00:00 2001 From: Romain Mercier Date: Wed, 29 Jan 2025 12:47:56 +0100 Subject: [PATCH 15/23] feat(api): #3121 scdl integration script - refacto: change parseParams type, object instead of array --- .../api/src/@types/ScdlDataIntegration.ts | 17 +++++++++++--- .../scripts/ScdlDataIntegration.node.test.ts | 22 +++++++++---------- .../scripts/ScdlDataIntegration.node.ts | 19 +++++++++++----- 3 files changed, 38 insertions(+), 20 deletions(-) diff --git a/packages/api/src/@types/ScdlDataIntegration.ts b/packages/api/src/@types/ScdlDataIntegration.ts index 96bc7c108..64a1839bb 100644 --- a/packages/api/src/@types/ScdlDataIntegration.ts +++ b/packages/api/src/@types/ScdlDataIntegration.ts @@ -1,6 +1,17 @@ -type BaseScdlParseArgs = [producerSlug: string, exportDate: string]; -export type ScdlParseArgs = BaseScdlParseArgs & [delimiter?: string, quote?: string]; -export type ScdlParseXlsArgs = BaseScdlParseArgs & [pageName?: string, rowOffset?: number | string]; +export interface BaseScdlParseArgs { + producerSlug: string; + exportDate: string; +} + +export interface ScdlParseArgs extends BaseScdlParseArgs { + delimiter?: string; + quote?: string; +} + +export interface ScdlParseXlsArgs extends BaseScdlParseArgs { + pageName?: string; + rowOffset?: number | string; +} export interface ScdlFileProcessingConfig { name: string; diff --git a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts index f10fe3ede..97e26e69a 100644 --- a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts +++ b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts @@ -19,21 +19,21 @@ const validConfigData: ScdlFileProcessingConfigList = { files: [ { name: "donnees-a-integrer1.csv", - parseParams: ["producerSlug1", "2025-01-13"], + parseParams: { producerSlug: "producerSlug1", exportDate: "2025-01-13" }, addProducer: true, producerName: "Test Producer 1", producerSiret: "12345678901", }, { name: "donnees-a-integrer2.csv", - parseParams: ["producerSlug2", "2025-01-14"], + parseParams: { producerSlug: "producerSlug2", exportDate: "2025-01-14" }, addProducer: true, producerName: "Test Producer 2", producerSiret: "12345678902", }, { name: "donnees-a-integrer3.csv", - parseParams: ["producerSlug2", "2025-01-15"], + parseParams: { producerSlug: "producerSlug2", exportDate: "2025-01-15" }, addProducer: false, }, ], @@ -102,7 +102,7 @@ describe("scdl data integration script", () => { describe("Test for processFile method", () => { const fileConfig = { name: "donnees-a-integrer1.csv", - parseParams: ["producerSlug1", "2025-01-13"] as ScdlParseArgs, + parseParams: { producerSlug: "producerSlug1", exportDate: "2025-01-13" } as ScdlParseArgs, addProducer: true, producerName: "Test Producer 1", producerSiret: "12345678901", @@ -110,7 +110,7 @@ describe("scdl data integration script", () => { const fileConfigWrongType = { name: "donnees-a-integrer1.doc", - parseParams: ["producerSlug1", "2025-01-13"] as ScdlParseXlsArgs, + parseParams: { producerSlug: "producerSlug1", exportDate: "2025-01-13" } as ScdlParseXlsArgs, addProducer: true, producerName: "Test Producer 1", producerSiret: "12345678901", @@ -120,7 +120,7 @@ describe("scdl data integration script", () => { await expect(processFile(fileConfig)).resolves.toBeUndefined(); expect(addProducerMock).toHaveBeenCalledWith( - fileConfig.parseParams[0], + fileConfig.parseParams.producerSlug, fileConfig.producerName, fileConfig.producerSiret, ); @@ -131,8 +131,8 @@ describe("scdl data integration script", () => { expect(parseMock).toHaveBeenCalledWith( expect.stringContaining(fileConfig.name), - fileConfig.parseParams[0], - fileConfig.parseParams[1], + fileConfig.parseParams.producerSlug, + fileConfig.parseParams.exportDate, undefined, undefined, ); @@ -256,21 +256,21 @@ describe("scdl data integration script", () => { files: [ { name: "donnees-a-integrer1.xlsx", - parseParams: ["producerSlug1", "2025-01-13"] as ScdlParseXlsArgs, + parseParams: { producerSlug: "producerSlug1", exportDate: "2025-01-13" } as ScdlParseXlsArgs, addProducer: true, producerName: "Test Producer 1", producerSiret: "12345678901", }, { name: "donnees-a-integrer2.csv", - parseParams: ["producerSlug2", "2025-01-14"] as ScdlParseArgs, + parseParams: { producerSlug: "producerSlug2", exportDate: "2025-01-14" } as ScdlParseArgs, addProducer: true, producerName: "Test Producer 2", producerSiret: "12345678902", }, { name: "donnees-a-integrer3.csv", - parseParams: ["producerSlug3", "2025-01-15"] as ScdlParseArgs, + parseParams: { producerSlug: "producerSlug3", exportDate: "2025-01-15" } as ScdlParseArgs, addProducer: true, producerName: "Test Producer 3", producerSiret: "12345678903", diff --git a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.ts b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.ts index d1e703f24..3aeb77373 100644 --- a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.ts +++ b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.ts @@ -6,7 +6,12 @@ import { SCDL_FILE_PROCESSING_PATH, } from "../../configurations/scdlIntegration.conf"; import { FileExtensionEnum } from "../../@enums/FileExtensionEnum"; -import { ScdlFileProcessingConfig, ScdlFileProcessingConfigList } from "../../@types/ScdlDataIntegration"; +import { + ScdlFileProcessingConfig, + ScdlFileProcessingConfigList, + ScdlParseArgs, + ScdlParseXlsArgs, +} from "../../@types/ScdlDataIntegration"; const scdlCli = new ScdlCli(); const successList: string[] = []; @@ -19,7 +24,10 @@ const isFileConfig = (file: any): file is ScdlFileProcessingConfig => { return ( file && typeof file.name === "string" && - Array.isArray(file.parseParams) && + typeof file.parseParams == "object" && + file.parseParams !== null && + typeof file.parseParams.producerSlug === "string" && + typeof file.parseParams.exportDate === "string" && typeof file.addProducer === "boolean" ); }; @@ -33,7 +41,7 @@ export const loadConfig = (): ScdlFileProcessingConfigList => { export const processFile = async (fileInfo: ScdlFileProcessingConfig) => { const { name, parseParams, addProducer, producerName, producerSiret } = fileInfo; const dirPath = path.resolve(SCDL_FILE_PROCESSING_PATH); - const [producerSlug, exportDate, ...optionalParams] = parseParams; + const { producerSlug, exportDate, ...optionalParams } = parseParams; try { if (addProducer && producerName && producerSiret) { @@ -53,11 +61,10 @@ export const processFile = async (fileInfo: ScdlFileProcessingConfig) => { const fileType = path.extname(name).slice(1).toLowerCase(); const filePath = path.join(dirPath, name); if (fileType === FileExtensionEnum.CSV) { - const delimiter = optionalParams[0]; - const quote = typeof optionalParams[1] === "string" ? optionalParams[1] : undefined; + const { delimiter, quote } = optionalParams as ScdlParseArgs; await scdlCli.parse(filePath, producerSlug, exportDate, delimiter, quote); } else if (fileType === FileExtensionEnum.XLS || fileType === FileExtensionEnum.XLSX) { - const [pageName = undefined, rowOffset = undefined] = optionalParams; + const { pageName, rowOffset } = optionalParams as ScdlParseXlsArgs; await scdlCli.parseXls(filePath, producerSlug, exportDate, pageName, rowOffset); } else { console.error(`❌ Unsupported file type : ${name} (type: ${fileType})`); From 33ab37489fd07964f40795dc97916b27e31ca02c Mon Sep 17 00:00:00 2001 From: Romain Mercier Date: Wed, 29 Jan 2025 14:03:52 +0100 Subject: [PATCH 16/23] feat(api): #3121 scdl integration script - changed console.trace --- packages/api/src/interfaces/scripts/ScdlDataIntegration.node.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.ts b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.ts index 3aeb77373..c66749c08 100644 --- a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.ts +++ b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.ts @@ -89,7 +89,7 @@ export const main = async () => { const filePromises = config.files.map(processFile); await Promise.all(filePromises); } catch (e) { - console.log("❌ Global execution failure.", e); + console.trace("❌ Global execution failure.", e); errorList.push(`Global execution failure : ${(e as Error).message}`); throw e; } finally { From 6e9e94bbec14def0a6b09f1bb09a335e56d3d1e1 Mon Sep 17 00:00:00 2001 From: Romain Mercier Date: Wed, 29 Jan 2025 14:08:21 +0100 Subject: [PATCH 17/23] feat(api): #3121 scdl integration script --- packages/api/src/@types/ScdlDataIntegration.ts | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/packages/api/src/@types/ScdlDataIntegration.ts b/packages/api/src/@types/ScdlDataIntegration.ts index 64a1839bb..1ca4a412e 100644 --- a/packages/api/src/@types/ScdlDataIntegration.ts +++ b/packages/api/src/@types/ScdlDataIntegration.ts @@ -1,17 +1,6 @@ -export interface BaseScdlParseArgs { - producerSlug: string; - exportDate: string; -} - -export interface ScdlParseArgs extends BaseScdlParseArgs { - delimiter?: string; - quote?: string; -} - -export interface ScdlParseXlsArgs extends BaseScdlParseArgs { - pageName?: string; - rowOffset?: number | string; -} +type BaseScdlParseArgs = { producerSlug: string; exportDate: string }; +export type ScdlParseArgs = BaseScdlParseArgs & { delimiter?: string; quote?: string }; +export type ScdlParseXlsArgs = BaseScdlParseArgs & { pageName?: string; rowOffset?: number | string }; export interface ScdlFileProcessingConfig { name: string; From d9baa46d9dd4e41bbf09515b38cba74392cb0c08 Mon Sep 17 00:00:00 2001 From: Romain Mercier Date: Wed, 29 Jan 2025 14:12:56 +0100 Subject: [PATCH 18/23] feat(api): #3121 scdl integration script - del afterEach --- .../src/interfaces/scripts/ScdlDataIntegration.node.test.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts index 97e26e69a..0b11d81d3 100644 --- a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts +++ b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts @@ -71,10 +71,6 @@ describe("scdl data integration script", () => { parseXlsMock = jest.spyOn(ScdlCli.prototype, "parseXls").mockResolvedValue(); }); - afterEach(() => { - jest.restoreAllMocks(); - }); - describe("loadConfig method", () => { it("should load config file successfully", () => { jest.spyOn(fs, "readFileSync").mockReturnValueOnce(JSON.stringify(validConfigData)); From eee623ba4d943ea46bceb785bca0a7dd85d48881 Mon Sep 17 00:00:00 2001 From: Romain Mercier Date: Wed, 29 Jan 2025 14:18:09 +0100 Subject: [PATCH 19/23] feat(api): #3121 scdl integration script - del redundant test --- .../interfaces/scripts/ScdlDataIntegration.node.test.ts | 7 ------- 1 file changed, 7 deletions(-) diff --git a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts index 0b11d81d3..2f5b1b44e 100644 --- a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts +++ b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts @@ -86,13 +86,6 @@ describe("scdl data integration script", () => { expect(() => loadConfig()).toThrowError(new Error("Unexpected token i in JSON at position 2")); }); - - it("should throw an error when the JSON file does not exist", () => { - jest.spyOn(fs, "readFileSync").mockImplementationOnce(() => { - throw new Error("ENOENT: no such file or directory"); - }); - expect(() => loadConfig()).toThrowError("ENOENT: no such file or directory"); - }); }); describe("Test for processFile method", () => { From 8b4adb86b7c9fb79b4ffcd15385c41e89024fcb6 Mon Sep 17 00:00:00 2001 From: Romain Mercier Date: Wed, 29 Jan 2025 14:20:49 +0100 Subject: [PATCH 20/23] feat(api): #3121 scdl integration script - renamed libel --- .../interfaces/scripts/ScdlDataIntegration.node.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts index 2f5b1b44e..395eec2cc 100644 --- a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts +++ b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts @@ -105,7 +105,7 @@ describe("scdl data integration script", () => { producerSiret: "12345678901", }; - it("should process file methods correctly with addProducer Called with correct params", async () => { + it("should process file methods correctly with addProducer called with correct params", async () => { await expect(processFile(fileConfig)).resolves.toBeUndefined(); expect(addProducerMock).toHaveBeenCalledWith( @@ -115,7 +115,7 @@ describe("scdl data integration script", () => { ); }); - it("should process file methods correctly with parse method Called with correct params", async () => { + it("should process file methods correctly with parse method called with correct params", async () => { await expect(processFile(fileConfig)).resolves.toBeUndefined(); expect(parseMock).toHaveBeenCalledWith( @@ -127,7 +127,7 @@ describe("scdl data integration script", () => { ); }); - it("should process file methods correctly with parseXls method not to have been Called", async () => { + it("should process file methods correctly with parseXls method not to have been called", async () => { await expect(processFile(fileConfig)).resolves.toBeUndefined(); expect(parseXlsMock).not.toHaveBeenCalled(); }); @@ -179,7 +179,7 @@ describe("scdl data integration script", () => { }); }); - describe("Test for main script", () => { + describe("main", () => { it("should call ScdlCli methods with correct arguments", async () => { jest.spyOn(fs, "readFileSync").mockReturnValueOnce(JSON.stringify(validConfigData)); From 841bc3fbdda357540ba30d69faef974e4f17f254 Mon Sep 17 00:00:00 2001 From: Romain Mercier Date: Wed, 29 Jan 2025 17:20:44 +0100 Subject: [PATCH 21/23] feat(api): #3121 scdl integration script - refacto to use code with cli --- packages/api/package.json | 3 +- packages/api/src/cli.ts | 2 + .../ScdlBatch.cli.test.ts} | 42 +++--- .../api/src/interfaces/cli/ScdlBatch.cli.ts | 127 ++++++++++++++++++ .../scripts/ScdlDataIntegration.node.ts | 117 ---------------- 5 files changed, 154 insertions(+), 137 deletions(-) rename packages/api/src/interfaces/{scripts/ScdlDataIntegration.node.test.ts => cli/ScdlBatch.cli.test.ts} (86%) create mode 100644 packages/api/src/interfaces/cli/ScdlBatch.cli.ts delete mode 100644 packages/api/src/interfaces/scripts/ScdlDataIntegration.node.ts diff --git a/packages/api/package.json b/packages/api/package.json index 923913abe..f70df09df 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -27,8 +27,7 @@ "lint:all": "eslint --fix --ext .js,.ts ./{migrations,src,tests,tools}/**/*.{js,ts} ./{migrations,src,tests,tools}/*.{js,ts}", "prettier": "prettier --write", "prettier:all": "prettier --write ./{migrations,src,tests,tools}/**/*.{js,ts}", - "lint-format": "lint-staged", - "scdl:import": "ts-node src/interfaces/scripts/ScdlDataIntegration.node.ts" + "lint-format": "lint-staged" }, "lint-staged": { "./{migrations,src,tests,tools}/**/*.{js,ts}": [ diff --git a/packages/api/src/cli.ts b/packages/api/src/cli.ts index b76c27395..fd57635c6 100644 --- a/packages/api/src/cli.ts +++ b/packages/api/src/cli.ts @@ -25,6 +25,7 @@ import { initIndexes } from "./shared/MongoInit"; import GeoCli from "./interfaces/cli/Geo.cli"; import DataBretagneCli from "./interfaces/cli/DataBretagne.cli"; import PaymentFlatCli from "./interfaces/cli/PaymentFlat.cli"; +import ScdlBatchCli from "./interfaces/cli/ScdlBatch.cli"; async function main() { await connectDB(); await initIndexes(); @@ -52,6 +53,7 @@ async function main() { GeoCli, DataBretagneCli, PaymentFlatCli, + ScdlBatchCli, ]; const args = process.argv.slice(2); diff --git a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts b/packages/api/src/interfaces/cli/ScdlBatch.cli.test.ts similarity index 86% rename from packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts rename to packages/api/src/interfaces/cli/ScdlBatch.cli.test.ts index 395eec2cc..c301d4319 100644 --- a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts +++ b/packages/api/src/interfaces/cli/ScdlBatch.cli.test.ts @@ -1,6 +1,7 @@ -import fs from "fs"; import path from "path"; -import { main, loadConfig, processFile } from "./ScdlDataIntegration.node"; +import ScdlBatchCli from "./ScdlBatch.cli"; +import fs from "fs"; + import { SCDL_FILE_PROCESSING_PATH, SCDL_FILE_PROCESSING_CONFIG_FILENAME, @@ -40,6 +41,7 @@ const validConfigData: ScdlFileProcessingConfigList = { }; describe("scdl data integration script", () => { + let scdlBatchCli: ScdlBatchCli; const testFilePath = path.join(SCDL_FILE_PROCESSING_PATH, SCDL_FILE_PROCESSING_CONFIG_FILENAME); let addProducerMock: jest.SpyInstance, [slug: string, name: string, siret: string]>; @@ -64,6 +66,10 @@ describe("scdl data integration script", () => { ] >; + beforeAll(() => { + scdlBatchCli = new ScdlBatchCli(); + }); + beforeEach(() => { jest.spyOn(process, "exit").mockImplementation((() => {}) as (code?: any) => never); addProducerMock = jest.spyOn(ScdlCli.prototype, "addProducer").mockResolvedValue(); @@ -74,7 +80,7 @@ describe("scdl data integration script", () => { describe("loadConfig method", () => { it("should load config file successfully", () => { jest.spyOn(fs, "readFileSync").mockReturnValueOnce(JSON.stringify(validConfigData)); - const result = loadConfig(); + const result = scdlBatchCli.loadConfig(); expect(result).toEqual(validConfigData); }); @@ -84,7 +90,7 @@ describe("scdl data integration script", () => { throw new Error("Unexpected token i in JSON at position 2"); }); - expect(() => loadConfig()).toThrowError(new Error("Unexpected token i in JSON at position 2")); + expect(() => scdlBatchCli.loadConfig()).toThrowError(new Error("Unexpected token i in JSON at position 2")); }); }); @@ -106,7 +112,7 @@ describe("scdl data integration script", () => { }; it("should process file methods correctly with addProducer called with correct params", async () => { - await expect(processFile(fileConfig)).resolves.toBeUndefined(); + await expect(scdlBatchCli.processFile(fileConfig)).resolves.toBeUndefined(); expect(addProducerMock).toHaveBeenCalledWith( fileConfig.parseParams.producerSlug, @@ -116,7 +122,7 @@ describe("scdl data integration script", () => { }); it("should process file methods correctly with parse method called with correct params", async () => { - await expect(processFile(fileConfig)).resolves.toBeUndefined(); + await expect(scdlBatchCli.processFile(fileConfig)).resolves.toBeUndefined(); expect(parseMock).toHaveBeenCalledWith( expect.stringContaining(fileConfig.name), @@ -128,7 +134,7 @@ describe("scdl data integration script", () => { }); it("should process file methods correctly with parseXls method not to have been called", async () => { - await expect(processFile(fileConfig)).resolves.toBeUndefined(); + await expect(scdlBatchCli.processFile(fileConfig)).resolves.toBeUndefined(); expect(parseXlsMock).not.toHaveBeenCalled(); }); @@ -136,28 +142,28 @@ describe("scdl data integration script", () => { addProducerMock = jest .spyOn(ScdlCli.prototype, "addProducer") .mockRejectedValue(new Error("Mocked addProducer error")); - await expect(processFile(fileConfig)).resolves.toBeUndefined(); + await expect(scdlBatchCli.processFile(fileConfig)).resolves.toBeUndefined(); }); it("should not call parse method when addProducer error", async () => { addProducerMock = jest .spyOn(ScdlCli.prototype, "addProducer") .mockRejectedValue(new Error("Mocked addProducer error")); - await expect(processFile(fileConfig)).resolves.toBeUndefined(); + await expect(scdlBatchCli.processFile(fileConfig)).resolves.toBeUndefined(); expect(parseMock).not.toHaveBeenCalled(); }); it("should catch Unsupported file type error", async () => { - await expect(processFile(fileConfigWrongType)).resolves.toBeUndefined(); + await expect(scdlBatchCli.processFile(fileConfigWrongType)).resolves.toBeUndefined(); }); it("should not call parse method when wrong file type", async () => { - await expect(processFile(fileConfigWrongType)).resolves.toBeUndefined(); + await expect(scdlBatchCli.processFile(fileConfigWrongType)).resolves.toBeUndefined(); expect(parseMock).not.toHaveBeenCalled(); }); it("should not call parseXls method when wrong file type", async () => { - await expect(processFile(fileConfigWrongType)).resolves.toBeUndefined(); + await expect(scdlBatchCli.processFile(fileConfigWrongType)).resolves.toBeUndefined(); expect(parseXlsMock).not.toHaveBeenCalled(); }); @@ -166,7 +172,7 @@ describe("scdl data integration script", () => { .spyOn(ScdlCli.prototype, "addProducer") .mockRejectedValue(new Error("Producer already exists")); - await expect(processFile(fileConfig)).resolves.toBeUndefined(); + await expect(scdlBatchCli.processFile(fileConfig)).resolves.toBeUndefined(); }); it("should call parse method when producer already exists", async () => { @@ -174,7 +180,7 @@ describe("scdl data integration script", () => { .spyOn(ScdlCli.prototype, "addProducer") .mockRejectedValue(new Error("Producer already exists")); - await expect(processFile(fileConfig)).resolves.toBeUndefined(); + await expect(scdlBatchCli.processFile(fileConfig)).resolves.toBeUndefined(); expect(parseMock).toHaveBeenCalledTimes(1); }); }); @@ -183,7 +189,7 @@ describe("scdl data integration script", () => { it("should call ScdlCli methods with correct arguments", async () => { jest.spyOn(fs, "readFileSync").mockReturnValueOnce(JSON.stringify(validConfigData)); - await expect(main()).resolves.toBeUndefined(); + await expect(scdlBatchCli.main()).resolves.toBeUndefined(); expect(addProducerMock).toHaveBeenCalledTimes(2); expect(addProducerMock).toHaveBeenCalledWith("producerSlug1", "Test Producer 1", "12345678901"); @@ -218,7 +224,7 @@ describe("scdl data integration script", () => { jest.spyOn(fs, "readFileSync").mockImplementationOnce(() => { throw new Error("Unexpected token i in JSON at position 2"); }); - await expect(main()).rejects.toThrow("Unexpected token i in JSON at position 2"); + await expect(scdlBatchCli.main()).rejects.toThrow("Unexpected token i in JSON at position 2"); }); it("should throw Invalid configuration file error", async () => { @@ -235,7 +241,7 @@ describe("scdl data integration script", () => { jest.spyOn(fs, "readFileSync").mockReturnValueOnce(JSON.stringify(invalidConfigData)); fs.writeFileSync(testFilePath, JSON.stringify(invalidConfigData)); - await expect(main()).rejects.toThrow( + await expect(scdlBatchCli.main()).rejects.toThrow( "Invalid configuration file: The config does not match the expected structure.", ); }); @@ -271,7 +277,7 @@ describe("scdl data integration script", () => { .spyOn(ScdlCli.prototype, "parseXls") .mockRejectedValue(new Error("Mocked addProducer error")); - await expect(main()).resolves.toBeUndefined(); + await expect(scdlBatchCli.main()).resolves.toBeUndefined(); expect(addProducerMock).toHaveBeenCalledTimes(3); expect(parseMock).toHaveBeenCalledTimes(2); expect(parseXlsMock).toHaveBeenCalledTimes(1); diff --git a/packages/api/src/interfaces/cli/ScdlBatch.cli.ts b/packages/api/src/interfaces/cli/ScdlBatch.cli.ts new file mode 100644 index 000000000..0b0cb5307 --- /dev/null +++ b/packages/api/src/interfaces/cli/ScdlBatch.cli.ts @@ -0,0 +1,127 @@ +import path from "path"; +import fs from "fs"; +import { StaticImplements } from "../../decorators/staticImplements.decorator"; +import { CliStaticInterface } from "../../@types"; + +import { + ScdlFileProcessingConfig, + ScdlFileProcessingConfigList, + ScdlParseArgs, + ScdlParseXlsArgs, +} from "../../@types/ScdlDataIntegration"; +import { + SCDL_FILE_PROCESSING_CONFIG_FILENAME, + SCDL_FILE_PROCESSING_PATH, +} from "../../configurations/scdlIntegration.conf"; +import { FileExtensionEnum } from "../../@enums/FileExtensionEnum"; +import ScdlCli from "./Scdl.cli"; + +@StaticImplements() +export default class ScdlBatchCli { + static cmdName = "scdl-batch"; + + private scdlCli: ScdlCli; + protected successList: string[] = []; + protected errorList: string[] = []; + + constructor() { + this.scdlCli = new ScdlCli(); + } + + private isConfig(obj: any): obj is ScdlFileProcessingConfigList { + return obj && Array.isArray(obj.files) && obj.files.every((file: any) => this.isFileConfig(file)); + } + private isFileConfig(file: any): file is ScdlFileProcessingConfig { + return ( + file && + typeof file.name === "string" && + typeof file.parseParams == "object" && + file.parseParams !== null && + typeof file.parseParams.producerSlug === "string" && + typeof file.parseParams.exportDate === "string" && + typeof file.addProducer === "boolean" + ); + } + + public loadConfig(): ScdlFileProcessingConfigList { + const filePath = path.resolve(SCDL_FILE_PROCESSING_PATH, SCDL_FILE_PROCESSING_CONFIG_FILENAME); + const data = fs.readFileSync(filePath, "utf8"); + return JSON.parse(data) as ScdlFileProcessingConfigList; + } + + public async processFile(fileInfo: ScdlFileProcessingConfig): Promise { + const { name, parseParams, addProducer, producerName, producerSiret } = fileInfo; + const dirPath = path.resolve(SCDL_FILE_PROCESSING_PATH); + const { producerSlug, exportDate, ...optionalParams } = parseParams; + + try { + if (addProducer && producerName && producerSiret) { + await this.scdlCli.addProducer(producerSlug, producerName, producerSiret); + this.successList.push(`added producer ${producerSlug}`); + } + } catch (e) { + if ((e as Error).message !== "Producer already exists") { + this.errorList.push( + `added producer ${producerSlug} for file ${name}, data parsing not performed : ${ + (e as Error).message + }`, + ); + return; + } + } + + try { + const fileType = path.extname(name).slice(1).toLowerCase(); + const filePath = path.join(dirPath, name); + if (fileType === FileExtensionEnum.CSV) { + const { delimiter, quote } = optionalParams as ScdlParseArgs; + await this.scdlCli.parse(filePath, producerSlug, exportDate, delimiter, quote); + } else if (fileType === FileExtensionEnum.XLS || fileType === FileExtensionEnum.XLSX) { + const { pageName, rowOffset } = optionalParams as ScdlParseXlsArgs; + await this.scdlCli.parseXls(filePath, producerSlug, exportDate, pageName, rowOffset); + } else { + console.error(`❌ Unsupported file type : ${name} (type: ${fileType})`); + throw new Error(`Unsupported file type : ${filePath}`); + } + this.successList.push(`parse data of ${producerSlug} for file ${name}`); + } catch (e) { + this.errorList.push(`parse data of ${producerSlug} for file ${name} : ${(e as Error).message}`); + } + } + + /** + * main function to load the file listing .json and launch data integration + */ + public async main() { + try { + const config: ScdlFileProcessingConfigList = this.loadConfig(); + + if (!this.isConfig(config)) { + throw new Error("Invalid configuration file: The config does not match the expected structure."); + } + const filePromises = config.files.map(this.processFile.bind(this)); + await Promise.all(filePromises); + } catch (e) { + console.trace("❌ Global execution failure.", e); + this.errorList.push(`Global execution failure : ${(e as Error).message}`); + throw e; + } finally { + console.log("\n---------------Summary of Operations---------------"); + if (this.errorList.length === 0) { + { + console.log("🚀 All operations completed successfully! 🎯"); + } + } + if (this.successList.length > 0) { + console.log("✅ list of Success :"); + this.successList.forEach(desc => console.log(`➡️ ${desc}`)); + } + + if (this.errorList.length > 0) { + console.log("⚠️ List of Errors :"); + this.errorList.forEach(desc => console.log(`❌ ${desc}`)); + } + process.exit(0); + } + } +} diff --git a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.ts b/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.ts deleted file mode 100644 index c66749c08..000000000 --- a/packages/api/src/interfaces/scripts/ScdlDataIntegration.node.ts +++ /dev/null @@ -1,117 +0,0 @@ -import fs from "fs"; -import path from "path"; -import ScdlCli from "../cli/Scdl.cli"; -import { - SCDL_FILE_PROCESSING_CONFIG_FILENAME, - SCDL_FILE_PROCESSING_PATH, -} from "../../configurations/scdlIntegration.conf"; -import { FileExtensionEnum } from "../../@enums/FileExtensionEnum"; -import { - ScdlFileProcessingConfig, - ScdlFileProcessingConfigList, - ScdlParseArgs, - ScdlParseXlsArgs, -} from "../../@types/ScdlDataIntegration"; - -const scdlCli = new ScdlCli(); -const successList: string[] = []; -const errorList: string[] = []; - -const isConfig = (obj: any): obj is ScdlFileProcessingConfigList => { - return obj && Array.isArray(obj.files) && obj.files.every(file => isFileConfig(file)); -}; -const isFileConfig = (file: any): file is ScdlFileProcessingConfig => { - return ( - file && - typeof file.name === "string" && - typeof file.parseParams == "object" && - file.parseParams !== null && - typeof file.parseParams.producerSlug === "string" && - typeof file.parseParams.exportDate === "string" && - typeof file.addProducer === "boolean" - ); -}; - -export const loadConfig = (): ScdlFileProcessingConfigList => { - const filePath = path.resolve(SCDL_FILE_PROCESSING_PATH, SCDL_FILE_PROCESSING_CONFIG_FILENAME); - const data = fs.readFileSync(filePath, "utf8"); - return JSON.parse(data) as ScdlFileProcessingConfigList; -}; - -export const processFile = async (fileInfo: ScdlFileProcessingConfig) => { - const { name, parseParams, addProducer, producerName, producerSiret } = fileInfo; - const dirPath = path.resolve(SCDL_FILE_PROCESSING_PATH); - const { producerSlug, exportDate, ...optionalParams } = parseParams; - - try { - if (addProducer && producerName && producerSiret) { - await scdlCli.addProducer(producerSlug, producerName, producerSiret); - successList.push(`added producer ${producerSlug}`); - } - } catch (e) { - if ((e as Error).message !== "Producer already exists") { - errorList.push( - `added producer ${producerSlug} for file ${name}, data parsing not performed : ${(e as Error).message}`, - ); - return; - } - } - - try { - const fileType = path.extname(name).slice(1).toLowerCase(); - const filePath = path.join(dirPath, name); - if (fileType === FileExtensionEnum.CSV) { - const { delimiter, quote } = optionalParams as ScdlParseArgs; - await scdlCli.parse(filePath, producerSlug, exportDate, delimiter, quote); - } else if (fileType === FileExtensionEnum.XLS || fileType === FileExtensionEnum.XLSX) { - const { pageName, rowOffset } = optionalParams as ScdlParseXlsArgs; - await scdlCli.parseXls(filePath, producerSlug, exportDate, pageName, rowOffset); - } else { - console.error(`❌ Unsupported file type : ${name} (type: ${fileType})`); - throw new Error(`Unsupported file type : ${filePath}`); - } - successList.push(`parse data of ${producerSlug} for file ${name}`); - } catch (e) { - errorList.push(`parse data of ${producerSlug} for file ${name} : ${(e as Error).message}`); - } -}; - -/** - * main function to load the file listing .json and launch data integration - */ -export const main = async () => { - try { - const config: ScdlFileProcessingConfigList = loadConfig(); - - if (!isConfig(config)) { - throw new Error("Invalid configuration file: The config does not match the expected structure."); - } - const filePromises = config.files.map(processFile); - await Promise.all(filePromises); - } catch (e) { - console.trace("❌ Global execution failure.", e); - errorList.push(`Global execution failure : ${(e as Error).message}`); - throw e; - } finally { - console.log("\n---------------Summary of Operations---------------"); - if (errorList.length === 0) { - { - console.log("🚀 All operations completed successfully! 🎯"); - } - } - if (successList.length > 0) { - console.log("✅ list of Success :"); - successList.forEach(desc => console.log(`➡️ ${desc}`)); - } - - if (errorList.length > 0) { - console.log("⚠️ List of Errors :"); - errorList.forEach(desc => console.log(`❌ ${desc}`)); - } - process.exit(0); - } -}; - -if (require.main === module) { - main(); -} From e1c2b95b4a49035fc5fd1ad679448ae0e9dd1c92 Mon Sep 17 00:00:00 2001 From: Romain Mercier Date: Thu, 30 Jan 2025 11:50:30 +0100 Subject: [PATCH 22/23] feat(api): #3121 scdl integration script --- .../src/interfaces/cli/ScdlBatch.cli.test.ts | 23 +++++++++++++------ .../api/src/interfaces/cli/ScdlBatch.cli.ts | 7 +++--- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/packages/api/src/interfaces/cli/ScdlBatch.cli.test.ts b/packages/api/src/interfaces/cli/ScdlBatch.cli.test.ts index c301d4319..07890e688 100644 --- a/packages/api/src/interfaces/cli/ScdlBatch.cli.test.ts +++ b/packages/api/src/interfaces/cli/ScdlBatch.cli.test.ts @@ -80,6 +80,7 @@ describe("scdl data integration script", () => { describe("loadConfig method", () => { it("should load config file successfully", () => { jest.spyOn(fs, "readFileSync").mockReturnValueOnce(JSON.stringify(validConfigData)); + // @ts-expect-error: protected const result = scdlBatchCli.loadConfig(); expect(result).toEqual(validConfigData); @@ -89,7 +90,7 @@ describe("scdl data integration script", () => { jest.spyOn(fs, "readFileSync").mockImplementationOnce(() => { throw new Error("Unexpected token i in JSON at position 2"); }); - + // @ts-expect-error: protected expect(() => scdlBatchCli.loadConfig()).toThrowError(new Error("Unexpected token i in JSON at position 2")); }); }); @@ -112,6 +113,7 @@ describe("scdl data integration script", () => { }; it("should process file methods correctly with addProducer called with correct params", async () => { + // @ts-expect-error: protected await expect(scdlBatchCli.processFile(fileConfig)).resolves.toBeUndefined(); expect(addProducerMock).toHaveBeenCalledWith( @@ -122,6 +124,7 @@ describe("scdl data integration script", () => { }); it("should process file methods correctly with parse method called with correct params", async () => { + // @ts-expect-error: protected await expect(scdlBatchCli.processFile(fileConfig)).resolves.toBeUndefined(); expect(parseMock).toHaveBeenCalledWith( @@ -134,6 +137,7 @@ describe("scdl data integration script", () => { }); it("should process file methods correctly with parseXls method not to have been called", async () => { + // @ts-expect-error: protected await expect(scdlBatchCli.processFile(fileConfig)).resolves.toBeUndefined(); expect(parseXlsMock).not.toHaveBeenCalled(); }); @@ -142,6 +146,7 @@ describe("scdl data integration script", () => { addProducerMock = jest .spyOn(ScdlCli.prototype, "addProducer") .mockRejectedValue(new Error("Mocked addProducer error")); + // @ts-expect-error: protected await expect(scdlBatchCli.processFile(fileConfig)).resolves.toBeUndefined(); }); @@ -149,20 +154,24 @@ describe("scdl data integration script", () => { addProducerMock = jest .spyOn(ScdlCli.prototype, "addProducer") .mockRejectedValue(new Error("Mocked addProducer error")); + // @ts-expect-error: protected await expect(scdlBatchCli.processFile(fileConfig)).resolves.toBeUndefined(); expect(parseMock).not.toHaveBeenCalled(); }); it("should catch Unsupported file type error", async () => { + // @ts-expect-error: protected await expect(scdlBatchCli.processFile(fileConfigWrongType)).resolves.toBeUndefined(); }); it("should not call parse method when wrong file type", async () => { + // @ts-expect-error: protected await expect(scdlBatchCli.processFile(fileConfigWrongType)).resolves.toBeUndefined(); expect(parseMock).not.toHaveBeenCalled(); }); it("should not call parseXls method when wrong file type", async () => { + // @ts-expect-error: protected await expect(scdlBatchCli.processFile(fileConfigWrongType)).resolves.toBeUndefined(); expect(parseXlsMock).not.toHaveBeenCalled(); }); @@ -171,7 +180,7 @@ describe("scdl data integration script", () => { addProducerMock = jest .spyOn(ScdlCli.prototype, "addProducer") .mockRejectedValue(new Error("Producer already exists")); - + // @ts-expect-error: protected await expect(scdlBatchCli.processFile(fileConfig)).resolves.toBeUndefined(); }); @@ -179,7 +188,7 @@ describe("scdl data integration script", () => { addProducerMock = jest .spyOn(ScdlCli.prototype, "addProducer") .mockRejectedValue(new Error("Producer already exists")); - + // @ts-expect-error: protected await expect(scdlBatchCli.processFile(fileConfig)).resolves.toBeUndefined(); expect(parseMock).toHaveBeenCalledTimes(1); }); @@ -189,7 +198,7 @@ describe("scdl data integration script", () => { it("should call ScdlCli methods with correct arguments", async () => { jest.spyOn(fs, "readFileSync").mockReturnValueOnce(JSON.stringify(validConfigData)); - await expect(scdlBatchCli.main()).resolves.toBeUndefined(); + await expect(scdlBatchCli.import()).resolves.toBeUndefined(); expect(addProducerMock).toHaveBeenCalledTimes(2); expect(addProducerMock).toHaveBeenCalledWith("producerSlug1", "Test Producer 1", "12345678901"); @@ -224,7 +233,7 @@ describe("scdl data integration script", () => { jest.spyOn(fs, "readFileSync").mockImplementationOnce(() => { throw new Error("Unexpected token i in JSON at position 2"); }); - await expect(scdlBatchCli.main()).rejects.toThrow("Unexpected token i in JSON at position 2"); + await expect(scdlBatchCli.import()).rejects.toThrow("Unexpected token i in JSON at position 2"); }); it("should throw Invalid configuration file error", async () => { @@ -241,7 +250,7 @@ describe("scdl data integration script", () => { jest.spyOn(fs, "readFileSync").mockReturnValueOnce(JSON.stringify(invalidConfigData)); fs.writeFileSync(testFilePath, JSON.stringify(invalidConfigData)); - await expect(scdlBatchCli.main()).rejects.toThrow( + await expect(scdlBatchCli.import()).rejects.toThrow( "Invalid configuration file: The config does not match the expected structure.", ); }); @@ -277,7 +286,7 @@ describe("scdl data integration script", () => { .spyOn(ScdlCli.prototype, "parseXls") .mockRejectedValue(new Error("Mocked addProducer error")); - await expect(scdlBatchCli.main()).resolves.toBeUndefined(); + await expect(scdlBatchCli.import()).resolves.toBeUndefined(); expect(addProducerMock).toHaveBeenCalledTimes(3); expect(parseMock).toHaveBeenCalledTimes(2); expect(parseXlsMock).toHaveBeenCalledTimes(1); diff --git a/packages/api/src/interfaces/cli/ScdlBatch.cli.ts b/packages/api/src/interfaces/cli/ScdlBatch.cli.ts index 0b0cb5307..a24556ba0 100644 --- a/packages/api/src/interfaces/cli/ScdlBatch.cli.ts +++ b/packages/api/src/interfaces/cli/ScdlBatch.cli.ts @@ -43,13 +43,13 @@ export default class ScdlBatchCli { ); } - public loadConfig(): ScdlFileProcessingConfigList { + private loadConfig(): ScdlFileProcessingConfigList { const filePath = path.resolve(SCDL_FILE_PROCESSING_PATH, SCDL_FILE_PROCESSING_CONFIG_FILENAME); const data = fs.readFileSync(filePath, "utf8"); return JSON.parse(data) as ScdlFileProcessingConfigList; } - public async processFile(fileInfo: ScdlFileProcessingConfig): Promise { + protected async processFile(fileInfo: ScdlFileProcessingConfig): Promise { const { name, parseParams, addProducer, producerName, producerSiret } = fileInfo; const dirPath = path.resolve(SCDL_FILE_PROCESSING_PATH); const { producerSlug, exportDate, ...optionalParams } = parseParams; @@ -92,7 +92,7 @@ export default class ScdlBatchCli { /** * main function to load the file listing .json and launch data integration */ - public async main() { + public async import() { try { const config: ScdlFileProcessingConfigList = this.loadConfig(); @@ -121,7 +121,6 @@ export default class ScdlBatchCli { console.log("⚠️ List of Errors :"); this.errorList.forEach(desc => console.log(`❌ ${desc}`)); } - process.exit(0); } } } From 9788a9604def6f79f07e955aca3ef3eb03a4928c Mon Sep 17 00:00:00 2001 From: "alice.telescoop" Date: Mon, 3 Feb 2025 11:50:25 +0100 Subject: [PATCH 23/23] Merge branch 'develop' into 3121-scdl-integration-script --- .../src/dataProviders/db/providers/osiris/osiris.action.port.ts | 2 +- .../dataProviders/db/providers/osiris/osiris.request.port.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/api/src/dataProviders/db/providers/osiris/osiris.action.port.ts b/packages/api/src/dataProviders/db/providers/osiris/osiris.action.port.ts index 296378324..7ca5e3d61 100644 --- a/packages/api/src/dataProviders/db/providers/osiris/osiris.action.port.ts +++ b/packages/api/src/dataProviders/db/providers/osiris/osiris.action.port.ts @@ -25,13 +25,13 @@ export class OsirisActionPort extends MongoPort { const options: FindOneAndUpdateOptions = { returnDocument: "after", includeResultMetadata: true }; const { _id, ...actionWithoutId } = OsirisActionAdapter.toDbo(osirisAction); const dbo = - //@ts-expect-error -- mongo typing expects no metadata ( await this.collection.findOneAndUpdate( { "indexedInformations.osirisActionId": osirisAction.indexedInformations.osirisActionId }, { $set: actionWithoutId }, options, ) + //@ts-expect-error -- mongo typing expects no metadata )?.value; if (!dbo) throw new MongoCnxError(); return OsirisActionAdapter.toEntity(dbo); diff --git a/packages/api/src/dataProviders/db/providers/osiris/osiris.request.port.ts b/packages/api/src/dataProviders/db/providers/osiris/osiris.request.port.ts index 2506cd903..292e67591 100644 --- a/packages/api/src/dataProviders/db/providers/osiris/osiris.request.port.ts +++ b/packages/api/src/dataProviders/db/providers/osiris/osiris.request.port.ts @@ -22,13 +22,13 @@ export class OsirisRequestPort extends MongoPort { const options = { returnDocument: "after", includeResultMetadata: true } as FindOneAndUpdateOptions; const { _id, ...requestWithoutId } = osirisRequest; return ( - //@ts-expect-error -- mongo typing expects no metadata ( await this.collection.findOneAndUpdate( { "providerInformations.osirisId": osirisRequest.providerInformations.osirisId }, { $set: requestWithoutId }, options, ) + //@ts-expect-error -- mongo typing expects no metadata )?.value as OsirisRequestEntity ); }