Fix/160 invalid date format on import csv (#7525)

* add failing test covering the bug * fix date format on import csv * update test title * improve testing * Testing the queue.v2 * fix import entities with relationships * fix eslint issues * Revert "Testing the queue.v2" This reverts commit efc5f1c. --------- Co-authored-by: Daneryl <[email protected]>
huridocs · Dec 16, 2024 · aa81851 · aa81851
1 parent 88e79e5
commit aa81851
Show file tree

Hide file tree

Showing 6 changed files with 112 additions and 30 deletions.
diff --git a/app/api/csv/csvLoader.ts b/app/api/csv/csvLoader.ts
@@ -132,7 +132,8 @@ export class CSVLoader extends EventEmitter {
             template,
             file,
             propNameToThesauriId,
-            indexedTranslations
+            indexedTranslations,
+            dateFormat
           );
           this.emit('entityLoaded', entity);
         }

diff --git a/app/api/csv/importEntity.ts b/app/api/csv/importEntity.ts
@@ -2,7 +2,6 @@
 import { createReadStream } from 'fs';
 import entities from 'api/entities';
 import { search } from 'api/search';
-import entitiesModel from 'api/entities/entitiesModel';
 import { processDocument } from 'api/files/processDocument';
 import { RawEntity } from 'api/csv/entityRow';
 import { TemplateSchema } from 'shared/types/templateType';
@@ -174,34 +173,36 @@ const translateEntity = async (
   template: TemplateSchema,
   importFile: ImportFile,
   propNameToThesauriId: Record<string, string>,
-  indexedTranslations: FullyIndexedTranslations
+  indexedTranslations: FullyIndexedTranslations,
+  dateFormat?: string
 ) => {
-  await entitiesModel.saveMultiple(
-    await Promise.all(
-      translations.map(async translatedEntity => {
-        const translatedEntityObject = await entityObject(
-          {
-            ...translatedEntity,
-            propertiesFromColumns: {
-              ...translatedEntity.propertiesFromColumns,
-              id: ensure(entity.sharedId),
-            },
-          },
-          template,
-          {
-            language: translatedEntity.language,
-          }
-        );
+  await translations.reduce(async (prevPromise, translation) => {
+    await prevPromise;
+
+    const entityParsed = await entityObject(
+      {
+        ...translation,
+        propertiesFromColumns: {
+          ...translation.propertiesFromColumns,
+          id: ensure(entity.sharedId),
+        },
+      },
+      template,
+      {
+        language: translation.language,
+        dateFormat,
+      }
+    );
 
-        return translateSelectLabels(
-          translatedEntityObject,
-          translatedEntity.language,
-          indexedTranslations,
-          propNameToThesauriId
-        );
-      })
-    )
-  );
+    const toSave = translateSelectLabels(
+      entityParsed,
+      translation.language,
+      indexedTranslations,
+      propNameToThesauriId
+    );
+
+    await entities.save(toSave, { language: translation.language, user: {} });
+  }, Promise.resolve());
 
   await Promise.all(
     translations.map(async translatedEntity => {

diff --git a/app/api/csv/specs/csvLoader.spec.js b/app/api/csv/specs/csvLoader.spec.js
@@ -1,13 +1,15 @@
+/* eslint-disable max-statements */
 /* eslint-disable max-lines */
 import path from 'path';
 
 import { CSVLoader } from 'api/csv';
-import { templateWithGeneratedTitle } from 'api/csv/specs/csvLoaderFixtures';
+import { simpleTemplateId, templateWithGeneratedTitle } from 'api/csv/specs/csvLoaderFixtures';
 import entities from 'api/entities';
 import translations from 'api/i18n';
 import { search } from 'api/search';
 import settings from 'api/settings';
 import db from 'api/utils/testing_db';
+import moment from 'moment';
 import typeParsers from '../typeParsers';
 import fixtures, { template1Id } from './csvLoaderFixtures';
 import { mockCsvFileReadStream } from './helpers';
@@ -400,4 +402,59 @@ describe('csvLoader', () => {
       });
     });
   });
+
+  describe('should parse date respecting the dateFormat on settings collection ', () => {
+    beforeEach(() => jest.restoreAllMocks());
+
+    const setDateFormat = async dateFormat => {
+      const _fixtures = { ...fixtures };
+      _fixtures.settings = [
+        {
+          ..._fixtures.settings[0],
+          languages: [
+            { key: 'en', label: 'English', default: true },
+            { key: 'es', label: 'Spanish' },
+          ],
+          dateFormat,
+        },
+      ];
+      await db.setupFixturesAndContext(_fixtures);
+    };
+
+    it('should correctly parse MM/dd/yyyy', async () => {
+      const dateFormat = 'MM/dd/yyyy';
+      await setDateFormat(dateFormat);
+
+      const dateOnCSV = '12/31/2024';
+      const csv = path.join(__dirname, '/simple_template.csv');
+      const selectedLanguageOnUserInterface = 'es';
+      const expectedDate = moment.utc(dateOnCSV, [dateFormat.toUpperCase()]).unix();
+
+      await loader.load(csv, simpleTemplateId, { language: selectedLanguageOnUserInterface });
+
+      const [englishEntity] = await entities.get({ language: 'en' });
+      const [spanishEntity] = await entities.get({ language: 'es' });
+
+      expect(spanishEntity.metadata.date_field).toEqual([{ value: expectedDate }]);
+      expect(englishEntity.metadata.date_field).toEqual([{ value: expectedDate }]);
+    });
+
+    it('should correctly parse yyyy/MM/dd', async () => {
+      const dateFormat = 'yyyy/MM/dd';
+      await setDateFormat(dateFormat);
+
+      const dateOnCSV = '2024/12/31';
+      const csv = path.join(__dirname, '/simple_template_2.csv');
+      const selectedLanguageOnUserInterface = 'es';
+      const expectedDate = moment.utc(dateOnCSV, [dateFormat.toUpperCase()]).unix();
+
+      await loader.load(csv, simpleTemplateId, { language: selectedLanguageOnUserInterface });
+
+      const [englishEntity] = await entities.get({ language: 'en' });
+      const [spanishEntity] = await entities.get({ language: 'es' });
+
+      expect(spanishEntity.metadata.date_field).toEqual([{ value: expectedDate }]);
+      expect(englishEntity.metadata.date_field).toEqual([{ value: expectedDate }]);
+    });
+  });
 });
diff --git a/app/api/csv/specs/csvLoaderFixtures.ts b/app/api/csv/specs/csvLoaderFixtures.ts
@@ -6,6 +6,7 @@ import { TranslationDBO } from 'api/i18n.v2/schemas/TranslationDBO';
 import { getFixturesFactory } from 'api/utils/fixturesFactory';
 
 const template1Id = db.id();
+const simpleTemplateId = db.id();
 const multiSelectThesaurusId = db.id();
 const thesauri1Id = db.id();
 const templateToRelateId = db.id();
@@ -48,6 +49,24 @@ const commonTranslationsV2 = (language: LanguageISO6391): TranslationDBO[] => [
 
 export default {
   templates: [
+    {
+      _id: simpleTemplateId,
+      name: 'Simple template',
+      properties: [
+        {
+          _id: db.id(),
+          type: propertyTypes.text,
+          label: 'simple text field',
+          name: templateUtils.safeName('simple text field'),
+        },
+        {
+          _id: db.id(),
+          type: propertyTypes.date,
+          label: 'Date field',
+          name: templateUtils.safeName('Date field'),
+        },
+      ],
+    },
     {
       _id: templateToRelateId,
       name: 'template to relate',
@@ -180,4 +199,4 @@ export default {
   ],
 };
 
-export { template1Id, templateWithGeneratedTitle, thesauri1Id };
+export { template1Id, templateWithGeneratedTitle, thesauri1Id, simpleTemplateId };
diff --git a/app/api/csv/specs/simple_template.csv b/app/api/csv/specs/simple_template.csv
@@ -0,0 +1,2 @@
+Title,Date field,simple text field__en,simple text field__es
+Simple template title,12/31/2024,English,Spanish
diff --git a/app/api/csv/specs/simple_template_2.csv b/app/api/csv/specs/simple_template_2.csv
@@ -0,0 +1,2 @@
+Title,Date field,simple text field__en,simple text field__es
+Simple template title,2024/12/31,English,Spanish
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		Title,Date field,simple text field__en,simple text field__es
		Simple template title,12/31/2024,English,Spanish
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		Title,Date field,simple text field__en,simple text field__es
		Simple template title,2024/12/31,English,Spanish