feat(api): #3121 scdl integration script

packages/api/package.json

@@ -27,7 +27,8 @@
         "lint:all": "eslint --fix --ext .js,.ts ./{migrations,src,tests,tools}/**/*.{js,ts} ./{migrations,src,tests,tools}/*.{js,ts}",
         "prettier": "prettier --write",
         "prettier:all": "prettier --write ./{migrations,src,tests,tools}/**/*.{js,ts}",
-        "lint-format": "lint-staged"
+        "lint-format": "lint-staged",
+        "scdl:import": "ts-node src/interfaces/scripts/ScdlDataIntegration.node.ts"
     },
     "lint-staged": {
         "./{migrations,src,tests,tools}/**/*.{js,ts}": [

packages/api/src/@enums/FileExtensionEnum.ts

@@ -0,0 +1,5 @@
+export enum FileExtensionEnum {
+    CSV = "csv",
+    XLS = "xls",
+    XLSX = "xlsx",
+}

packages/api/src/@types/ScdlDataIntegration.ts

@@ -0,0 +1,15 @@
+type BaseScdlParseArgs = [producerSlug: string, exportDate: string];
+export type ScdlParseArgs = BaseScdlParseArgs & [delimiter?: string, quote?: string];
+export type ScdlParseXlsArgs = BaseScdlParseArgs & [pageName?: string, rowOffset?: number | string];
+
+export interface ScdlFileProcessingConfig {
+    name: string;
+    parseParams: ScdlParseArgs | ScdlParseXlsArgs;
+    addProducer: boolean;
+    producerName?: string;
+    producerSiret?: string;
+}
+
+export interface ScdlFileProcessingConfigList {
+    files: ScdlFileProcessingConfig[];
+}

packages/api/src/configurations/scdlIntegration.conf.ts

@@ -0,0 +1,2 @@
+export const SCDL_FILE_PROCESSING_PATH = process.env.SCDL_FILE_PROCESSING_PATH || "./data-integration";
+export const SCDL_FILE_PROCESSING_CONFIG_FILENAME = "scdl-file-processing.config.json";

packages/api/src/interfaces/cli/Scdl.cli.ts

@@ -32,12 +32,12 @@ export default class ScdlCli {
         producerSlug: string,
         exportDate: string,
         pageName?: string,
-        rowOffsetStr: number | string = 0,
+        rowOffset: number | string = 0,
     ) {
         await this.validateGenericInput(file, producerSlug, exportDate);
-        const rowOffset = typeof rowOffsetStr === "number" ? rowOffsetStr : parseInt(rowOffsetStr);
+        const parsedRowOffset = typeof rowOffset === "number" ? rowOffset : parseInt(rowOffset);
         const fileContent = fs.readFileSync(file);
-        const { entities, errors } = ScdlGrantParser.parseExcel(fileContent, pageName, rowOffset);
+        const { entities, errors } = ScdlGrantParser.parseExcel(fileContent, pageName, parsedRowOffset);
         await Promise.all([
             this.persistEntities(entities, producerSlug, exportDate as string),
             this.exportErrors(errors, file),

packages/api/src/interfaces/scripts/ScdlDataIntegration.node.test.ts

@@ -0,0 +1,291 @@
+import fs from "fs";
+import path from "path";
+import { main, loadConfig, processFile } from "./ScdlDataIntegration.node";
+import {
+    SCDL_FILE_PROCESSING_PATH,
+    SCDL_FILE_PROCESSING_CONFIG_FILENAME,
+} from "../../configurations/scdlIntegration.conf";
+import ScdlCli from "../cli/Scdl.cli";
+import { ScdlFileProcessingConfigList, ScdlParseArgs, ScdlParseXlsArgs } from "../../@types/ScdlDataIntegration";
+
+jest.mock("../../configurations/scdlIntegration.conf", () => ({
+    SCDL_FILE_PROCESSING_PATH: path.resolve(__dirname, "test-integration"),
+    SCDL_FILE_PROCESSING_CONFIG_FILENAME: "test-scdl-file-processing.config.json",
+}));
+jest.mock("../cli/Scdl.cli");
+jest.mock("fs");
+
+const validConfigData: ScdlFileProcessingConfigList = {
+    files: [
+        {
+            name: "donnees-a-integrer1.csv",
+            parseParams: ["producerSlug1", "2025-01-13"],
+            addProducer: true,
+            producerName: "Test Producer 1",
+            producerSiret: "12345678901",
+        },
+        {
+            name: "donnees-a-integrer2.csv",
+            parseParams: ["producerSlug2", "2025-01-14"],
+            addProducer: true,
+            producerName: "Test Producer 2",
+            producerSiret: "12345678902",
+        },
+        {
+            name: "donnees-a-integrer3.csv",
+            parseParams: ["producerSlug2", "2025-01-15"],
+            addProducer: false,
+        },
+    ],
+};
+
+describe("scdl data integration script", () => {
+    const testFilePath = path.join(SCDL_FILE_PROCESSING_PATH, SCDL_FILE_PROCESSING_CONFIG_FILENAME);
+
+    let addProducerMock: jest.SpyInstance<Promise<void>, [slug: string, name: string, siret: string]>;
+    let parseMock: jest.SpyInstance<
+        Promise<void>,
+        [
+            file: string,
+            producerSlug: string,
+            exportDate: string,
+            delimiter?: string | undefined,
+            quote?: string | undefined,
+        ]
+    >;
+    let parseXlsMock: jest.SpyInstance<
+        Promise<void>,
+        [
+            file: string,
+            producerSlug: string,
+            exportDate: string,
+            pageName?: string | undefined,
+            rowOffset?: string | number | undefined,
+        ]
+    >;
+
+    beforeEach(() => {
+        jest.spyOn(process, "exit").mockImplementation((() => {}) as (code?: any) => never);
+        addProducerMock = jest.spyOn(ScdlCli.prototype, "addProducer").mockResolvedValue();
+        parseMock = jest.spyOn(ScdlCli.prototype, "parse").mockResolvedValue();
+        parseXlsMock = jest.spyOn(ScdlCli.prototype, "parseXls").mockResolvedValue();
+    });
+
+    afterEach(() => {
+        jest.restoreAllMocks();
+    });
+
+    describe("loadConfig method", () => {
+        it("should load config file successfully", () => {
+            jest.spyOn(fs, "readFileSync").mockReturnValueOnce(JSON.stringify(validConfigData));
+            const result = loadConfig();
+
+            expect(result).toEqual(validConfigData);
+        });
+
+        it("should throw an error when the JSON file is invalid", () => {
+            jest.spyOn(fs, "readFileSync").mockImplementationOnce(() => {
+                throw new Error("Unexpected token i in JSON at position 2");
+            });
+
+            expect(() => loadConfig()).toThrowError(new Error("Unexpected token i in JSON at position 2"));
+        });
+
+        it("should throw an error when the JSON file does not exist", () => {
+            jest.spyOn(fs, "readFileSync").mockImplementationOnce(() => {
+                throw new Error("ENOENT: no such file or directory");
+            });
+            expect(() => loadConfig()).toThrowError("ENOENT: no such file or directory");
+        });
+    });
+
+    describe("Test for processFile method", () => {
+        const fileConfig = {
+            name: "donnees-a-integrer1.csv",
+            parseParams: ["producerSlug1", "2025-01-13"] as ScdlParseArgs,
+            addProducer: true,
+            producerName: "Test Producer 1",
+            producerSiret: "12345678901",
+        };
+
+        const fileConfigWrongType = {
+            name: "donnees-a-integrer1.doc",
+            parseParams: ["producerSlug1", "2025-01-13"] as ScdlParseXlsArgs,
+            addProducer: true,
+            producerName: "Test Producer 1",
+            producerSiret: "12345678901",
+        };
+
+        it("should process file methods correctly with addProducer Called with correct params", async () => {
+            await expect(processFile(fileConfig)).resolves.toBeUndefined();
+
+            expect(addProducerMock).toHaveBeenCalledWith(
+                fileConfig.parseParams[0],
+                fileConfig.producerName,
+                fileConfig.producerSiret,
+            );
+        });
+
+        it("should process file methods correctly with parse method Called with correct params", async () => {
+            await expect(processFile(fileConfig)).resolves.toBeUndefined();
+
+            expect(parseMock).toHaveBeenCalledWith(
+                expect.stringContaining(fileConfig.name),
+                fileConfig.parseParams[0],
+                fileConfig.parseParams[1],
+                undefined,
+                undefined,
+            );
+        });
+
+        it("should process file methods correctly with parseXls method not to have been Called", async () => {
+            await expect(processFile(fileConfig)).resolves.toBeUndefined();
+            expect(parseXlsMock).not.toHaveBeenCalled();
+        });
+
+        it("should catch error", async () => {
+            addProducerMock = jest
+                .spyOn(ScdlCli.prototype, "addProducer")
+                .mockRejectedValue(new Error("Mocked addProducer error"));
+            await expect(processFile(fileConfig)).resolves.toBeUndefined();
+        });
+
+        it("should not call parse method when addProducer error", async () => {
+            addProducerMock = jest
+                .spyOn(ScdlCli.prototype, "addProducer")
+                .mockRejectedValue(new Error("Mocked addProducer error"));
+            await expect(processFile(fileConfig)).resolves.toBeUndefined();
+            expect(parseMock).not.toHaveBeenCalled();
+        });
+
+        it("should catch Unsupported file type error", async () => {
+            await expect(processFile(fileConfigWrongType)).resolves.toBeUndefined();
+        });
+
+        it("should not call parse method when wrong file type", async () => {
+            await expect(processFile(fileConfigWrongType)).resolves.toBeUndefined();
+            expect(parseMock).not.toHaveBeenCalled();
+        });
+
+        it("should not call parseXls method when wrong file type", async () => {
+            await expect(processFile(fileConfigWrongType)).resolves.toBeUndefined();
+            expect(parseXlsMock).not.toHaveBeenCalled();
+        });
+
+        it("should catch Producer already exists error", async () => {
+            addProducerMock = jest
+                .spyOn(ScdlCli.prototype, "addProducer")
+                .mockRejectedValue(new Error("Producer already exists"));
+
+            await expect(processFile(fileConfig)).resolves.toBeUndefined();
+        });
+
+        it("should call parse method when producer already exists", async () => {
+            addProducerMock = jest
+                .spyOn(ScdlCli.prototype, "addProducer")
+                .mockRejectedValue(new Error("Producer already exists"));
+
+            await expect(processFile(fileConfig)).resolves.toBeUndefined();
+            expect(parseMock).toHaveBeenCalledTimes(1);
+        });
+    });
+
+    describe("Test for main script", () => {
+        it("should call ScdlCli methods with correct arguments", async () => {
+            jest.spyOn(fs, "readFileSync").mockReturnValueOnce(JSON.stringify(validConfigData));
+
+            await expect(main()).resolves.toBeUndefined();
+
+            expect(addProducerMock).toHaveBeenCalledTimes(2);
+            expect(addProducerMock).toHaveBeenCalledWith("producerSlug1", "Test Producer 1", "12345678901");
+            expect(addProducerMock).toHaveBeenCalledWith("producerSlug2", "Test Producer 2", "12345678902");
+
+            expect(parseMock).toHaveBeenCalledTimes(3);
+            expect(parseMock).toHaveBeenCalledWith(
+                path.join(SCDL_FILE_PROCESSING_PATH, "donnees-a-integrer1.csv"),
+                "producerSlug1",
+                "2025-01-13",
+                undefined,
+                undefined,
+            );
+            expect(parseMock).toHaveBeenCalledWith(
+                path.join(SCDL_FILE_PROCESSING_PATH, "donnees-a-integrer2.csv"),
+                "producerSlug2",
+                "2025-01-14",
+                undefined,
+                undefined,
+            );
+            expect(parseMock).toHaveBeenCalledWith(
+                path.join(SCDL_FILE_PROCESSING_PATH, "donnees-a-integrer3.csv"),
+                "producerSlug2",
+                "2025-01-15",
+                undefined,
+                undefined,
+            );
+            expect(parseXlsMock).not.toHaveBeenCalled();
+        });
+
+        it("should throw Unexpected token error", async () => {
+            jest.spyOn(fs, "readFileSync").mockImplementationOnce(() => {
+                throw new Error("Unexpected token i in JSON at position 2");
+            });
+            await expect(main()).rejects.toThrow("Unexpected token i in JSON at position 2");
+        });
+
+        it("should throw Invalid configuration file error", async () => {
+            const invalidConfigData = {
+                files: [
+                    {
+                        name: "donnees-a-integrer1.csv",
+                        addProducer: true,
+                        producerName: "Test Producer 1",
+                        producerSiret: "12345678901",
+                    },
+                ],
+            };
+            jest.spyOn(fs, "readFileSync").mockReturnValueOnce(JSON.stringify(invalidConfigData));
+            fs.writeFileSync(testFilePath, JSON.stringify(invalidConfigData));
+
+            await expect(main()).rejects.toThrow(
+                "Invalid configuration file: The config does not match the expected structure.",
+            );
+        });
+
+        it("when error with one file, should process continue for other files", async () => {
+            const configData = {
+                files: [
+                    {
+                        name: "donnees-a-integrer1.xlsx",
+                        parseParams: ["producerSlug1", "2025-01-13"] as ScdlParseXlsArgs,
+                        addProducer: true,
+                        producerName: "Test Producer 1",
+                        producerSiret: "12345678901",
+                    },
+                    {
+                        name: "donnees-a-integrer2.csv",
+                        parseParams: ["producerSlug2", "2025-01-14"] as ScdlParseArgs,
+                        addProducer: true,
+                        producerName: "Test Producer 2",
+                        producerSiret: "12345678902",
+                    },
+                    {
+                        name: "donnees-a-integrer3.csv",
+                        parseParams: ["producerSlug3", "2025-01-15"] as ScdlParseArgs,
+                        addProducer: true,
+                        producerName: "Test Producer 3",
+                        producerSiret: "12345678903",
+                    },
+                ],
+            };
+            jest.spyOn(fs, "readFileSync").mockReturnValueOnce(JSON.stringify(configData));
+            parseXlsMock = jest
+                .spyOn(ScdlCli.prototype, "parseXls")
+                .mockRejectedValue(new Error("Mocked addProducer error"));
+
+            await expect(main()).resolves.toBeUndefined();
+            expect(addProducerMock).toHaveBeenCalledTimes(3);
+            expect(parseMock).toHaveBeenCalledTimes(2);
+            expect(parseXlsMock).toHaveBeenCalledTimes(1);
+        });
+    });
+});