feat(api): 40 implement referentiel import functionality with region academique support

.vscode/tasks.json

@@ -118,7 +118,7 @@
     },
     {
       "label": "Start Api-v1 API",
-      "command": "npm run dev:api",
+      "command": "source ~/.zshrc && nvm use && npm run dev:api",
       "type": "shell",
       "problemMatcher": [],
       "options": {
@@ -127,7 +127,7 @@
     },
     {
       "label": "Start Api-v1 TASKS",
-      "command": "npm run dev:tasks",
+      "command": "source ~/.zshrc && nvm use && npm run dev:tasks",
       "type": "shell",
       "problemMatcher": [],
       "options": {
@@ -150,7 +150,16 @@
     },
     {
       "label": "Start Api-v2 TASKS",
-      "command": "npm run dev:tasks",
+      "command": "source ~/.zshrc && nvm use && npm run dev:tasks",
+      "type": "shell",
+      "problemMatcher": [],
+      "options": {
+        "cwd": "./apiv2"
+      }
+    },
+    {
+      "label": "Start Apiv2",
+      "command": "npm run dev:app apiv2",
       "type": "shell",
       "problemMatcher": [],
       "options": {

apiv2/.gitignore

@@ -6,3 +6,4 @@ dist/**/*
 **.js
 .env
 **.xlsx*
+!test/**/**/fixtures/*.xlsx

apiv2/src/admin/Admin.module.ts

@@ -42,9 +42,7 @@ import { sejourMongoProviders } from "./infra/sejours/phase1/sejour/provider/Sej
 import { sessionMongoProviders } from "./infra/sejours/phase1/session/provider/SessionMongo.provider";
 import { FileGateway } from "@shared/core/File.gateway";
 import { FileProvider } from "@shared/infra/File.provider";
-import { useCaseProvider as referentielUseCaseProvider } from "./infra/referentiel/initProvider/useCase";
-import { ImportReferentielController } from "./infra/referentiel/api/ImportReferentiel.controller";
-import { ReferentielRoutesService } from "./core/referentiel/routes/ReferentielRoutes.service";
+import { referentielUseCaseProviders } from "./infra/referentiel/initProvider/useCase";
 import { HistoryController } from "./infra/history/api/History.controller";
 import { historyProvider } from "./infra/history/historyProvider";
 import { serviceProvider } from "./infra/iam/service/serviceProvider";
@@ -56,7 +54,7 @@ import { ClsPluginTransactional } from "@nestjs-cls/transactional";
 
 import { DATABASE_CONNECTION } from "@infra/Database.provider";
 import { TransactionalAdapterMongoose } from "@infra/TransactionalAdatpterMongoose";
-import { referentielServiceProvider } from "./infra/referentiel/initProvider/service";
+import { ReferentielModule } from "./infra/referentiel/ReferentielModule";
 import { segmentDeLigneMongoProviders } from "./infra/sejours/phase1/segmentDeLigne/provider/SegmentDeLigneMongo.provider";
 import { demandeModificationLigneDeBusMongoProviders } from "./infra/sejours/phase1/demandeModificationLigneDeBus/provider/DemandeModificationLigneDeBusMongo.provider";
 
@@ -78,12 +76,12 @@ import { demandeModificationLigneDeBusMongoProviders } from "./infra/sejours/pha
         NotificationModule,
         QueueModule,
         TaskModule,
+        ReferentielModule,
     ],
     controllers: [
         ClasseController,
         AffectationController,
         Phase1Controller,
-        ImportReferentielController,
         AuthController,
         AdminTaskController,
         HistoryController,
@@ -93,7 +91,6 @@ import { demandeModificationLigneDeBusMongoProviders } from "./infra/sejours/pha
         ClasseService,
         AffectationService,
         SimulationAffectationHTSService,
-        ReferentielRoutesService,
         { provide: AuthProvider, useClass: JwtTokenService },
         ...classeMongoProviders,
         ...referentMongoProviders,
@@ -121,9 +118,8 @@ import { demandeModificationLigneDeBusMongoProviders } from "./infra/sejours/pha
         ...cleGatewayProviders,
         ...phase1GatewayProviders,
         ...jeuneGatewayProviders,
-        ...referentielUseCaseProvider,
+        ...referentielUseCaseProviders,
         ...serviceProvider,
-        ...referentielServiceProvider,
     ],
 })
 export class AdminModule {

apiv2/src/admin/AdminJob.module.ts

@@ -20,7 +20,7 @@ import { gatewayProviders as jeuneGatewayProviders } from "./infra/sejours/jeune
 import { FileProvider } from "@shared/infra/File.provider";
 import { FileGateway } from "@shared/core/File.gateway";
 import { TaskGateway } from "@task/core/Task.gateway";
-import { useCaseProvider as referentielUseCaseProvider } from "./infra/referentiel/initProvider/useCase";
+import { referentielUseCaseProviders } from "./infra/referentiel/initProvider/useCase";
 import { AffectationService } from "./core/sejours/phase1/affectation/Affectation.service";
 import { ValiderAffectationHTS } from "./core/sejours/phase1/affectation/ValiderAffectationHTS";
 import { planDeTransportMongoProviders } from "./infra/sejours/phase1/planDeTransport/provider/PlanDeTransportMongo.provider";
@@ -38,6 +38,7 @@ import { ClockProvider } from "@shared/infra/Clock.provider";
 import { NotificationGateway } from "@notification/core/Notification.gateway";
 import { NotificationProducer } from "@notification/infra/Notification.producer";
 import { referentielServiceProvider } from "./infra/referentiel/initProvider/service";
+import { ReferentielModule } from "./infra/referentiel/ReferentielModule";
 import { segmentDeLigneMongoProviders } from "./infra/sejours/phase1/segmentDeLigne/provider/SegmentDeLigneMongo.provider";
 import { demandeModificationLigneDeBusMongoProviders } from "./infra/sejours/phase1/demandeModificationLigneDeBus/provider/DemandeModificationLigneDeBusMongo.provider";
 
@@ -56,6 +57,7 @@ import { demandeModificationLigneDeBusMongoProviders } from "./infra/sejours/pha
         ConfigModule,
         TaskModule,
         DatabaseModule,
+        ReferentielModule,
     ],
     providers: [
         Logger,
@@ -85,7 +87,7 @@ import { demandeModificationLigneDeBusMongoProviders } from "./infra/sejours/pha
         SimulationAffectationHTSService,
         SimulationAffectationHTS,
         ValiderAffectationHTS,
-        ...referentielUseCaseProvider,
+        ...referentielUseCaseProviders,
         ...referentielServiceProvider,
         AdminTaskImportReferentielSelectorService,
     ],

apiv2/src/admin/core/referentiel/ReferentielImportTask.model.ts

@@ -0,0 +1,27 @@
+import { TaskModel } from "@task/core/Task.model";
+import { ReferentielTaskType } from "snu-lib";
+
+export interface ReferentielImportTaskAuthor {
+    id?: string;
+    prenom?: string;
+    nom?: string;
+    role?: string;
+    sousRole?: string;
+    email?: string;
+}
+
+export interface ReferentielImportTaskParameters {
+    type: ReferentielTaskType;
+    fileName: string;
+    fileKey: string;
+    fileLineCount: number;
+    auteur: ReferentielImportTaskAuthor;
+    folderPath: string;
+}
+
+export type ReferentielImportTaskResult = {
+    rapportKey: string;
+};
+
+export type ReferentielImportTaskModel = TaskModel<ReferentielImportTaskParameters, ReferentielImportTaskResult>;
+export type CreateReferentielImportTaskModel = Omit<ReferentielImportTaskModel, "id" | "createdAt" | "updatedAt">;

apiv2/src/admin/core/referentiel/ReferentielImportTask.service.spec.ts

@@ -0,0 +1,163 @@
+import { Test, TestingModule } from "@nestjs/testing";
+import { Logger } from "@nestjs/common";
+
+import { FileGateway } from "@shared/core/File.gateway";
+import { TaskGateway } from "@task/core/Task.gateway";
+import { FunctionalException, FunctionalExceptionCode } from "@shared/core/FunctionalException";
+import { ReferentielTaskType, TaskName, TaskStatus } from "snu-lib";
+import { ReferentielImportTaskService } from "./ReferentielImportTask.service";
+import { REGION_ACADEMIQUE_COLUMN_NAMES } from "./ReferentielTaskTypeColumns";
+
+describe("ReferentielImportTaskService", () => {
+    let referentielImportTaskService: ReferentielImportTaskService;
+    let fileGateway: FileGateway;
+    let taskGateway: TaskGateway;
+
+    beforeEach(async () => {
+        const module: TestingModule = await Test.createTestingModule({
+            providers: [
+                ReferentielImportTaskService,
+                Logger,
+                {
+                    provide: FileGateway,
+                    useValue: {
+                        parseXLS: jest.fn().mockResolvedValue([{}]),
+                        uploadFile: jest.fn().mockResolvedValue({ Key: "test-key" }),
+                    },
+                },
+                { provide: TaskGateway, useValue: { create: jest.fn() } },
+            ],
+        }).compile();
+
+        fileGateway = module.get<FileGateway>(FileGateway);
+        taskGateway = module.get<TaskGateway>(TaskGateway);
+        referentielImportTaskService = module.get<ReferentielImportTaskService>(ReferentielImportTaskService);
+    });
+
+    const regionAcademiqueRecord = {
+        [REGION_ACADEMIQUE_COLUMN_NAMES.code]: "BRE",
+        [REGION_ACADEMIQUE_COLUMN_NAMES.libelle]: "BRETAGNE",
+        [REGION_ACADEMIQUE_COLUMN_NAMES.zone]: "A",
+        [REGION_ACADEMIQUE_COLUMN_NAMES.date_derniere_modification_si]: "31/07/2024"
+    };
+
+
+    describe("import", () => {
+        const mockAuteur = {
+            id: "id",
+            prenom: "prenom",
+            nom: "nom",
+            role: "role",
+            sousRole: "sousRole",
+        };
+
+        const mockImportParams = {
+            importType: ReferentielTaskType.IMPORT_REGIONS_ACADEMIQUES,
+            fileName: "fileName",
+            buffer: Buffer.from("test"),
+            mimetype: "mimetype",
+            auteur: mockAuteur,
+        };
+
+        it("should not import empty file", async () => {
+            jest.spyOn(fileGateway, "parseXLS").mockResolvedValue([]);
+            await expect(
+                referentielImportTaskService.import({
+                    importType: ReferentielTaskType.IMPORT_REGIONS_ACADEMIQUES,
+                    fileName: "fileName",
+                    buffer: Buffer.from(""),
+                    mimetype: "mimetype",
+                    auteur: mockAuteur,
+                }),
+            ).rejects.toThrow(new FunctionalException(FunctionalExceptionCode.IMPORT_EMPTY_FILE));
+        });
+
+        it("should not import file without valid column", async () => {
+            jest.spyOn(fileGateway, "parseXLS").mockResolvedValue([
+                {
+                    "Session formule": "",
+                    "Code court de Route": "",
+                    "Commentaire interne sur l'enregistrement": "",
+                    "Session : Code de la session": "",
+                    "Session : Désignation de la session": "",
+                    "Session : Date de début de la session": "",
+                    "Session : Date de fin de la session": "",
+                   // Route: "",
+                    "Code point de rassemblement initial": "",
+                    "Point de rassemblement initial": "",
+                }
+            ]);
+            await expect(
+                referentielImportTaskService.import({
+                    importType: ReferentielTaskType.IMPORT_ROUTES,
+                    fileName: "fileName",
+                    buffer: Buffer.from("test"),
+                    mimetype: "mimetype",
+                    auteur: {
+                        id: "id",
+                        prenom: "prenom",
+                        nom: "nom",
+                        role: "role",
+                        sousRole: "sousRole",
+                    },
+                }),
+            ).rejects.toThrow(new FunctionalException(FunctionalExceptionCode.IMPORT_MISSING_COLUMN));
+        });
+
+
+
+        it("should import file with valid columns", async () => {
+            jest.spyOn(fileGateway, "parseXLS").mockResolvedValue([regionAcademiqueRecord]);
+
+            await referentielImportTaskService.import({
+                importType: ReferentielTaskType.IMPORT_REGIONS_ACADEMIQUES,
+                fileName: "fileName",
+                buffer: Buffer.from("test"),
+                mimetype: "mimetype",
+                auteur: {
+                    id: "id",
+                    prenom: "prenom",
+                    nom: "nom",
+                    role: "role",
+                    sousRole: "sousRole",
+                },
+            });
+
+            expect(taskGateway.create).toHaveBeenCalledWith({
+                name: TaskName.REFERENTIEL_IMPORT,
+                status: TaskStatus.PENDING,
+                metadata: {
+                    parameters: {
+                        type: ReferentielTaskType.IMPORT_REGIONS_ACADEMIQUES,
+                        fileKey: "test-key",
+                        fileLineCount: 1,
+                        fileName: "fileName",
+                        auteur: {
+                            id: "id",
+                            prenom: "prenom",
+                            nom: "nom",
+                            role: "role",
+                            sousRole: "sousRole",
+                        },
+                    },
+                },
+            });
+        });
+
+        it("devrait vérifier la présence de toutes les colonnes requises", async () => {
+            const partialData = {
+                "Code région académique": "RA01",
+                "Région académique : Libellé région académique long": "Test Region",
+                // Colonne manquante: "Zone région académique édition"
+                "Région académique : Date de création": "2023-01-01",
+                "Région académique : Date de dernière modification": "2023-01-02",
+            };
+
+            jest.spyOn(fileGateway, "parseXLS").mockResolvedValue([partialData]);
+
+            await expect(referentielImportTaskService.import(mockImportParams))
+                .rejects
+                .toThrow(new FunctionalException(FunctionalExceptionCode.IMPORT_MISSING_COLUMN, "Zone région académique édition"));
+        });
+    });
+});

apiv2/src/admin/core/referentiel/ReferentielImportTask.service.ts

@@ -0,0 +1,71 @@
+import { Inject, Injectable } from "@nestjs/common";
+import { FileGateway } from "@shared/core/File.gateway";
+import { FunctionalException, FunctionalExceptionCode } from "@shared/core/FunctionalException";
+import { TaskGateway } from "@task/core/Task.gateway";
+
+import { ReferentielTaskType, TaskName, TaskStatus } from "snu-lib";
+import { TaskModel } from "@task/core/Task.model";
+import { ReferentielImportTaskAuthor } from "./ReferentielImportTask.model";
+import { REGION_ACADEMIQUE_COLUMN_NAMES, ROUTE_COLUMN_NAMES } from "@admin/core/referentiel/ReferentielTaskTypeColumns";
+
+export const REQUIRED_COLUMN_NAMES = {
+    [ReferentielTaskType.IMPORT_REGIONS_ACADEMIQUES]: Object.values(REGION_ACADEMIQUE_COLUMN_NAMES),
+    [ReferentielTaskType.IMPORT_ROUTES]: Object.values(ROUTE_COLUMN_NAMES),
+};
+
+@Injectable()
+export class ReferentielImportTaskService {
+    constructor(
+        @Inject(TaskGateway) private readonly taskGateway: TaskGateway,
+        @Inject(FileGateway) private readonly fileGateway: FileGateway,
+    ) {}
+
+    async import({
+        importType,
+        fileName,
+        buffer,
+        mimetype,
+        auteur,
+    }: {
+        importType: typeof ReferentielTaskType[keyof typeof ReferentielTaskType];
+        fileName: string;
+        buffer: Buffer;
+        mimetype: string;
+        auteur: ReferentielImportTaskAuthor;
+    }): Promise<TaskModel> {
+        const dataToImport = await this.fileGateway.parseXLS<Record<string, string>>(buffer, {
+            defval: "",
+        });
+
+        if (dataToImport.length === 0) {
+            throw new FunctionalException(FunctionalExceptionCode.IMPORT_EMPTY_FILE);
+        }
+        for (const column of REQUIRED_COLUMN_NAMES[importType]) {
+            if (!dataToImport[0].hasOwnProperty(column)) {
+                throw new FunctionalException(FunctionalExceptionCode.IMPORT_MISSING_COLUMN, column);
+            }
+        }
+
+        const timestamp = `${new Date().toISOString()?.replaceAll(":", "-")?.replace(".", "-")}`;
+        const s3File = await this.fileGateway.uploadFile(`file/admin/referentiel/${importType}/${timestamp}_${fileName}`, {
+            data: buffer,
+            mimetype,
+        });
+
+        const task = await this.taskGateway.create({
+            name: TaskName.REFERENTIEL_IMPORT,
+            status: TaskStatus.PENDING,
+            metadata: {
+                parameters: {
+                    type: importType,
+                    fileName,
+                    fileKey: s3File.Key,
+                    fileLineCount: dataToImport.length,
+                    auteur,
+                },
+            },  
+        });
+
+        return task;
+    }
+}