diff --git a/app/api/config.ts b/app/api/config.ts index 45da19bbc2..65b6b71d7f 100644 --- a/app/api/config.ts +++ b/app/api/config.ts @@ -25,7 +25,7 @@ const filesRootPath = FILES_ROOT_PATH || rootPath; // when using multiple node processes const CLUSTER_MODE = process.env.CLUSTER_MODE || false; -const onlyDBHOST = () => (DBHOST ? `mongodb://${DBHOST}/` : 'mongodb://localhost/'); +const onlyDBHOST = () => (DBHOST ? `mongodb://${DBHOST}/` : 'mongodb://127.0.0.1/'); export const config = { VERSION: ENVIRONMENT ? version : `development-${version}`, diff --git a/app/api/csv/entityRow.ts b/app/api/csv/entityRow.ts index eb1b3688ff..a92e762427 100644 --- a/app/api/csv/entityRow.ts +++ b/app/api/csv/entityRow.ts @@ -5,7 +5,7 @@ type Languages = string[]; export type RawEntity = { language: string; - [k: string]: string; + propertiesFromColumns: CSVRow; }; const toSafeName = (row: CSVRow, newNameGeneration: boolean = false): CSVRow => @@ -57,16 +57,16 @@ const extractEntity = ( const propName = key.split(`__${languageCode}`)[0]; const selectedKey = propName in propNameToThesauriId ? `${propName}__${defaultLanguage}` : key; - entity[propName] = safeNamed[selectedKey]; //eslint-disable-line no-param-reassign + entity.propertiesFromColumns[propName] = safeNamed[selectedKey]; //eslint-disable-line no-param-reassign return entity; }, - { ...baseEntity, language: languageCode } + { propertiesFromColumns: { ...baseEntity }, language: languageCode } ) ); return { - rawEntity: rawEntities.find((e: CSVRow) => e.language === currentLanguage), - rawTranslations: rawEntities.filter((e: CSVRow) => e.language !== currentLanguage), + rawEntity: rawEntities.find((e: RawEntity) => e.language === currentLanguage), + rawTranslations: rawEntities.filter((e: RawEntity) => e.language !== currentLanguage), }; }; diff --git a/app/api/csv/importEntity.ts b/app/api/csv/importEntity.ts index 30d2f7c34d..483ea24338 100644 --- a/app/api/csv/importEntity.ts +++ b/app/api/csv/importEntity.ts @@ -22,7 +22,9 @@ const parse = async (toImportEntity: RawEntity, prop: PropertySchema, dateFormat : typeParsers.text(toImportEntity, prop); const hasValidValue = (prop: PropertySchema, toImportEntity: RawEntity) => - prop.name ? toImportEntity[prop.name] || prop.type === propertyTypes.generatedid : false; + prop.name + ? toImportEntity.propertiesFromColumns[prop.name] || prop.type === propertyTypes.generatedid + : false; const toMetadata = async ( template: TemplateSchema, @@ -40,17 +42,18 @@ const toMetadata = async ( Promise.resolve({}) ); -const currentEntityIdentifiers = async (sharedId: string, language: string) => +const currentEntityIdentifiers = async (sharedId: string | undefined, language: string) => sharedId ? entities.get({ sharedId, language }, '_id sharedId').then(([e]) => e) : {}; const titleByTemplate = (template: TemplateSchema, entity: RawEntity) => { + const { propertiesFromColumns: data } = entity; const generatedTitle = - !entity.title && + !data.title && template.commonProperties?.find(property => property.name === 'title' && property.generatedId); if (generatedTitle) { return generateID(3, 4, 4); } - return entity.title; + return data.title; }; const entityObject = async ( @@ -61,7 +64,7 @@ const entityObject = async ( title: titleByTemplate(template, toImportEntity), template: template._id, metadata: await toMetadata(template, toImportEntity, dateFormat), - ...(await currentEntityIdentifiers(toImportEntity.id, language)), + ...(await currentEntityIdentifiers(toImportEntity.propertiesFromColumns.id, language)), }); type Options = { @@ -76,8 +79,9 @@ const importEntity = async ( importFile: ImportFile, { user = {}, language, dateFormat }: Options ) => { - const { attachments } = toImportEntity; - delete toImportEntity.attachments; + const { propertiesFromColumns } = toImportEntity; + const { attachments } = propertiesFromColumns; + delete propertiesFromColumns.attachments; const eo = await entityObject(toImportEntity, template, { language, dateFormat }); const entity = await entities.save( eo, @@ -85,8 +89,8 @@ const importEntity = async ( { updateRelationships: true, index: false } ); - if (toImportEntity.file && entity.sharedId) { - const file = await importFile.extractFile(toImportEntity.file); + if (propertiesFromColumns.file && entity.sharedId) { + const file = await importFile.extractFile(propertiesFromColumns.file); await processDocument(entity.sharedId, file); await storage.storeFile(file.filename, createReadStream(file.path), 'document'); } @@ -148,7 +152,13 @@ const translateEntity = async ( await Promise.all( translations.map(async translatedEntity => { const translatedEntityObject = await entityObject( - { ...translatedEntity, id: ensure(entity.sharedId) }, + { + ...translatedEntity, + propertiesFromColumns: { + ...translatedEntity.propertiesFromColumns, + id: ensure(entity.sharedId), + }, + }, template, { language: translatedEntity.language, @@ -167,8 +177,8 @@ const translateEntity = async ( await Promise.all( translations.map(async translatedEntity => { - if (translatedEntity.file) { - const file = await importFile.extractFile(translatedEntity.file); + if (translatedEntity.propertiesFromColumns.file) { + const file = await importFile.extractFile(translatedEntity.propertiesFromColumns.file); await processDocument(ensure(entity.sharedId), file); } }) diff --git a/app/api/csv/specs/__snapshots__/importFile.spec.ts.snap b/app/api/csv/specs/__snapshots__/importFile.spec.ts.snap index add4054262..43784c62a2 100644 --- a/app/api/csv/specs/__snapshots__/importFile.spec.ts.snap +++ b/app/api/csv/specs/__snapshots__/importFile.spec.ts.snap @@ -1,11 +1,11 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP exports[`importFile readStream should return a readable stream for the csv file 1`] = ` -"Title , text label , numeric label, non configured, select_label, not defined type, geolocation_geolocation,auto id, additional tag(s), Multi Select Label, Date label +"Title , text label , numeric label, non configured, select_label, not defined type, geolocation_geolocation,auto id, additional tag(s), Multi Select Label, Date label, Language -title1, text value 1, 1977, ______________, thesauri1 , notType1 , 1|1,,tag1, multivalue1,03/01/2022 -title2, text value 2, 2019, ______________, thesauri2 , notType2 , ,,tag2, multivalue2,01/03/2022 -title3, text value 3, 2020, ______________, thesauri2 , notType3 , 0|0,,tag3, multivalue1|multivalue3,01/01/2022 +title1, text value 1, 1977, ______________, thesauri1 , notType1 , 1|1,,tag1, multivalue1,03/01/2022,English +title2, text value 2, 2019, ______________, thesauri2 , notType2 , ,,tag2, multivalue2,01/03/2022,Spanish +title3, text value 3, 2020, ______________, thesauri2 , notType3 , 0|0,,tag3, multivalue1|multivalue3,01/01/2022,AnyStringIsGood " `; diff --git a/app/api/csv/specs/csvLoader.spec.js b/app/api/csv/specs/csvLoader.spec.js index ed57cea824..69f4d68a70 100644 --- a/app/api/csv/specs/csvLoader.spec.js +++ b/app/api/csv/specs/csvLoader.spec.js @@ -168,6 +168,7 @@ describe('csvLoader', () => { 'additional_tag(s)', 'multi_select_label', 'date_label', + 'language', ]); }); @@ -177,6 +178,11 @@ describe('csvLoader', () => { expect(textValues.length).toEqual(0); }); + it('should import properties named "Language" properly', () => { + const textValues = imported.map(i => i.metadata.language[0].value); + expect(textValues).toEqual(['English', 'Spanish', 'AnyStringIsGood']); + }); + describe('metadata parsing', () => { it('should parse metadata properties by type using typeParsers', () => { const textValues = imported.map(i => i.metadata.text_label[0].value); @@ -287,8 +293,8 @@ describe('csvLoader', () => { it('should fail when parsing throws an error', async () => { jest.spyOn(entities, 'save').mockImplementation(() => Promise.resolve({})); jest.spyOn(typeParsers, 'text').mockImplementation(entity => { - if (entity.title === 'title2') { - throw new Error(`error-${entity.title}`); + if (entity.propertiesFromColumns.title === 'title2') { + throw new Error(`error-${entity.propertiesFromColumns.title}`); } }); diff --git a/app/api/csv/specs/csvLoaderFixtures.ts b/app/api/csv/specs/csvLoaderFixtures.ts index f4e7240d89..42242b8922 100644 --- a/app/api/csv/specs/csvLoaderFixtures.ts +++ b/app/api/csv/specs/csvLoaderFixtures.ts @@ -110,6 +110,11 @@ export default { label: 'Date label', name: templateUtils.safeName('Date label'), }, + { + type: propertyTypes.text, + label: 'Language', + name: templateUtils.safeName('Language'), + }, ], }, { diff --git a/app/api/csv/specs/entityRow.spec.js b/app/api/csv/specs/entityRow.spec.js index 3f3975d5df..108eeba061 100644 --- a/app/api/csv/specs/entityRow.spec.js +++ b/app/api/csv/specs/entityRow.spec.js @@ -19,8 +19,10 @@ describe('entityRow', () => { ); expect(rawEntity).toEqual({ - title: 'test_en', - not_portuguese_ptescaped_property: 'not portuguese', + propertiesFromColumns: { + title: 'test_en', + not_portuguese_ptescaped_property: 'not portuguese', + }, language: 'en', }); }); @@ -37,7 +39,9 @@ describe('entityRow', () => { {} ); - expect(rawTranslations).toEqual([{ title: 'test_es', language: 'es' }]); + expect(rawTranslations).toEqual([ + { propertiesFromColumns: { title: 'test_es' }, language: 'es' }, + ]); }); it('should return translations for languages that have values not blank', () => { @@ -53,7 +57,9 @@ describe('entityRow', () => { {} ); - expect(rawTranslations).toEqual([{ title: 'test_es', language: 'es' }]); + expect(rawTranslations).toEqual([ + { propertiesFromColumns: { title: 'test_es' }, language: 'es' }, + ]); }); it('should return all translations when everything is translated', () => { @@ -70,8 +76,8 @@ describe('entityRow', () => { ); expect(rawTranslations).toEqual([ - { title: 'test_es', language: 'es' }, - { text: 'text_pt', language: 'pt' }, + { propertiesFromColumns: { title: 'test_es' }, language: 'es' }, + { propertiesFromColumns: { text: 'text_pt' }, language: 'pt' }, ]); }); @@ -90,17 +96,21 @@ describe('entityRow', () => { ); expect(rawEntity).toEqual({ - title: 'test_es', - some__entirely_new_property: 'has __en in name, but is not english', - a__pt_property: 'not portugese', + propertiesFromColumns: { + title: 'test_es', + some__entirely_new_property: 'has __en in name, but is not english', + a__pt_property: 'not portugese', + }, language: 'es', }); expect(rawTranslations).toEqual([ { - title: 'test_en', - some__entirely_new_property: 'has __en in name, but is not english', - a__pt_property: 'not portugese', + propertiesFromColumns: { + title: 'test_en', + some__entirely_new_property: 'has __en in name, but is not english', + a__pt_property: 'not portugese', + }, language: 'en', }, ]); diff --git a/app/api/csv/specs/test.csv b/app/api/csv/specs/test.csv index 5e6a6f6474..59765b22f6 100644 --- a/app/api/csv/specs/test.csv +++ b/app/api/csv/specs/test.csv @@ -1,5 +1,5 @@ -Title , text label , numeric label, non configured, select_label, not defined type, geolocation_geolocation,auto id, additional tag(s), Multi Select Label, Date label +Title , text label , numeric label, non configured, select_label, not defined type, geolocation_geolocation,auto id, additional tag(s), Multi Select Label, Date label, Language -title1, text value 1, 1977, ______________, thesauri1 , notType1 , 1|1,,tag1, multivalue1,03/01/2022 -title2, text value 2, 2019, ______________, thesauri2 , notType2 , ,,tag2, multivalue2,01/03/2022 -title3, text value 3, 2020, ______________, thesauri2 , notType3 , 0|0,,tag3, multivalue1|multivalue3,01/01/2022 +title1, text value 1, 1977, ______________, thesauri1 , notType1 , 1|1,,tag1, multivalue1,03/01/2022,English +title2, text value 2, 2019, ______________, thesauri2 , notType2 , ,,tag2, multivalue2,01/03/2022,Spanish +title3, text value 3, 2020, ______________, thesauri2 , notType3 , 0|0,,tag3, multivalue1|multivalue3,01/01/2022,AnyStringIsGood diff --git a/app/api/csv/specs/typeParsers.spec.js b/app/api/csv/specs/typeParsers.spec.js index 977fee16dd..0799f1e50e 100644 --- a/app/api/csv/specs/typeParsers.spec.js +++ b/app/api/csv/specs/typeParsers.spec.js @@ -1,11 +1,15 @@ import moment from 'moment'; import typeParsers from '../typeParsers'; +const rawEntityWithProps = props => ({ + propertiesFromColumns: props, +}); + describe('csvLoader typeParsers', () => { describe('text', () => { it('should return the value', async () => { const templateProp = { name: 'text_prop' }; - const rawEntity = { text_prop: 'text' }; + const rawEntity = rawEntityWithProps({ text_prop: 'text' }); expect(await typeParsers.text(rawEntity, templateProp)).toEqual([{ value: 'text' }]); }); @@ -14,14 +18,14 @@ describe('csvLoader typeParsers', () => { describe('numeric', () => { it('should return numeric value', async () => { const templateProp = { name: 'numeric_prop' }; - const rawEntity = { numeric_prop: '2019' }; + const rawEntity = rawEntityWithProps({ numeric_prop: '2019' }); expect(await typeParsers.numeric(rawEntity, templateProp)).toEqual([{ value: 2019 }]); }); it('should return original value if value is NaN (will be catched by the entitiy validator)', async () => { const templateProp = { name: 'numeric_prop' }; - const rawEntity = { numeric_prop: 'Not a number' }; + const rawEntity = rawEntityWithProps({ numeric_prop: 'Not a number' }); expect(await typeParsers.numeric(rawEntity, templateProp)).toEqual([ { value: 'Not a number' }, @@ -32,7 +36,7 @@ describe('csvLoader typeParsers', () => { describe('link', () => { it('should use the text as url and label', async () => { const templateProp = { name: 'link_prop' }; - const rawEntity = { link_prop: 'http://www.url.com' }; + const rawEntity = rawEntityWithProps({ link_prop: 'http://www.url.com' }); expect(await typeParsers.link(rawEntity, templateProp)).toEqual([ { @@ -46,14 +50,14 @@ describe('csvLoader typeParsers', () => { it('should return null if url is not valid', async () => { const templateProp = { name: 'link_prop' }; - const rawEntity = { link_prop: 'url' }; + const rawEntity = rawEntityWithProps({ link_prop: 'url' }); expect(await typeParsers.link(rawEntity, templateProp)).toBe(null); }); it('should use "|" as separator for label and url', async () => { const templateProp = { name: 'link_prop' }; - const rawEntity = { link_prop: 'label|http://www.url.com' }; + const rawEntity = rawEntityWithProps({ link_prop: 'label|http://www.url.com' }); expect(await typeParsers.link(rawEntity, templateProp)).toEqual([ { @@ -104,7 +108,11 @@ describe('csvLoader typeParsers', () => { async ({ dateProp, dateFormat, expectedDate }) => { const templateProp = { name: 'date_prop' }; - const expected = await typeParsers.date({ date_prop: dateProp }, templateProp, dateFormat); + const expected = await typeParsers.date( + rawEntityWithProps({ date_prop: dateProp }), + templateProp, + dateFormat + ); expect(moment.utc(expected[0].value, 'X').format('DD-MM-YYYY')).toEqual(expectedDate); } @@ -129,7 +137,7 @@ describe('csvLoader typeParsers', () => { const templateProp = { name: 'date_prop' }; const expected = await typeParsers.multidate( - { date_prop: dateProp }, + rawEntityWithProps({ date_prop: dateProp }), templateProp, dateFormat ); @@ -160,7 +168,11 @@ describe('csvLoader typeParsers', () => { { value: { from, to }, }, - ] = await typeParsers.daterange({ date_prop: dateProp }, templateProp, dateFormat); + ] = await typeParsers.daterange( + rawEntityWithProps({ date_prop: dateProp }), + templateProp, + dateFormat + ); expect({ from: from && moment.utc(from, 'X').format('DD-MM-YYYY'), @@ -249,7 +261,7 @@ describe('csvLoader typeParsers', () => { const templateProp = { name: 'date_prop' }; const expected = await typeParsers.multidaterange( - { date_prop: dateProp }, + rawEntityWithProps({ date_prop: dateProp }), templateProp, dateFormat ); diff --git a/app/api/csv/typeParsers.ts b/app/api/csv/typeParsers.ts index 38429f89a2..0e0ea6866e 100644 --- a/app/api/csv/typeParsers.ts +++ b/app/api/csv/typeParsers.ts @@ -14,7 +14,9 @@ import relationship from './typeParsers/relationship'; const defaultParser = async ( entityToImport: RawEntity, property: PropertySchema -): Promise => [{ value: entityToImport[ensure(property.name)] }]; +): Promise => [ + { value: entityToImport.propertiesFromColumns[ensure(property.name)] }, +]; const parseDateValue = (dateValue: string, dateFormat: string) => { const allowedFormats = [ @@ -64,7 +66,7 @@ export default { entityToImport: RawEntity, property: PropertySchema ): Promise { - const value = entityToImport[ensure(property.name)]; + const value = entityToImport.propertiesFromColumns[ensure(property.name)]; return Number.isNaN(Number(value)) ? [{ value }] : [{ value: Number(value) }]; }, @@ -73,7 +75,7 @@ export default { property: PropertySchema, dateFormat: string ): Promise { - const date = entityToImport[ensure(property.name)]; + const date = entityToImport.propertiesFromColumns[ensure(property.name)]; return [parseDate(date, dateFormat)]; }, @@ -82,7 +84,9 @@ export default { property: PropertySchema, dateFormat: string ): Promise { - const dates = parseMultiValue(entityToImport[ensure(property.name)]); + const dates = parseMultiValue( + entityToImport.propertiesFromColumns[ensure(property.name)] + ); return dates.map(date => parseDate(date, dateFormat)); }, @@ -91,7 +95,7 @@ export default { property: PropertySchema, dateFormat: string ): Promise { - const range = entityToImport[ensure(property.name)]; + const range = entityToImport.propertiesFromColumns[ensure(property.name)]; return [parseDateRange(range, dateFormat)]; }, @@ -100,7 +104,9 @@ export default { property: PropertySchema, dateFormat: string ): Promise { - const ranges = parseMultiValue(entityToImport[ensure(property.name)]); + const ranges = parseMultiValue( + entityToImport.propertiesFromColumns[ensure(property.name)] + ); return ranges.map(range => parseDateRange(range, dateFormat)); }, @@ -108,10 +114,11 @@ export default { entityToImport: RawEntity, property: PropertySchema ): Promise { - let [label, linkUrl] = entityToImport[ensure(property.name)].split('|'); + let [label, linkUrl] = + entityToImport.propertiesFromColumns[ensure(property.name)].split('|'); if (!linkUrl) { - linkUrl = entityToImport[ensure(property.name)]; + linkUrl = entityToImport.propertiesFromColumns[ensure(property.name)]; label = linkUrl; } diff --git a/app/api/csv/typeParsers/generatedid.ts b/app/api/csv/typeParsers/generatedid.ts index a872baf226..208796303f 100644 --- a/app/api/csv/typeParsers/generatedid.ts +++ b/app/api/csv/typeParsers/generatedid.ts @@ -7,7 +7,8 @@ const generatedid = async ( entityToImport: RawEntity, property: PropertySchema ): Promise => { - const value = entityToImport[ensure(property.name)] || generateID(3, 4, 4); + const value = + entityToImport.propertiesFromColumns[ensure(property.name)] || generateID(3, 4, 4); return [{ value }]; }; diff --git a/app/api/csv/typeParsers/geolocation.ts b/app/api/csv/typeParsers/geolocation.ts index f4cdb3ed1e..464e7f2cc9 100644 --- a/app/api/csv/typeParsers/geolocation.ts +++ b/app/api/csv/typeParsers/geolocation.ts @@ -6,7 +6,7 @@ const geolocation = async ( entityToImport: RawEntity, property: PropertySchema ): Promise => { - const [lat, lon] = entityToImport[ensure(property.name)].split('|'); + const [lat, lon] = entityToImport.propertiesFromColumns[ensure(property.name)].split('|'); if (lat && lon) { return [{ value: { lat: Number(lat), lon: Number(lon), label: '' } }]; } diff --git a/app/api/csv/typeParsers/multiselect.ts b/app/api/csv/typeParsers/multiselect.ts index d0b12f24ae..315c8f4e5a 100644 --- a/app/api/csv/typeParsers/multiselect.ts +++ b/app/api/csv/typeParsers/multiselect.ts @@ -41,7 +41,7 @@ const multiselect = async ( const currentThesauri = (await thesauri.getById(property.content)) || ({} as ThesaurusSchema); const values = splitMultiselectLabels( - entityToImport[ensure(property.name)] + entityToImport.propertiesFromColumns[ensure(property.name)] ).normalizedLabelToLabel; const thesaurusValues = flatThesaurusValues(currentThesauri); diff --git a/app/api/csv/typeParsers/relationship.ts b/app/api/csv/typeParsers/relationship.ts index 12c19a6898..2d74b4dbae 100644 --- a/app/api/csv/typeParsers/relationship.ts +++ b/app/api/csv/typeParsers/relationship.ts @@ -6,7 +6,7 @@ import { PropertySchema } from 'shared/types/commonTypes'; import { EntityWithFilesSchema } from 'shared/types/entityType'; const relationship = async (entityToImport: RawEntity, property: PropertySchema) => { - const values = entityToImport[ensure(property.name)] + const values = entityToImport.propertiesFromColumns[ensure(property.name)] .split('|') .filter(emptyString) .filter(unique); diff --git a/app/api/csv/typeParsers/select.ts b/app/api/csv/typeParsers/select.ts index 728b8b72e2..aa410b13b5 100644 --- a/app/api/csv/typeParsers/select.ts +++ b/app/api/csv/typeParsers/select.ts @@ -20,7 +20,7 @@ const select = async ( property: PropertySchema ): Promise => { const currentThesauri = (await thesauri.getById(property.content)) || ({} as ThesaurusSchema); - const propValue = entityToImport[ensure(property.name)]; + const propValue = entityToImport.propertiesFromColumns[ensure(property.name)]; const normalizedPropValue = normalizeThesaurusLabel(propValue); if (!normalizedPropValue) { diff --git a/app/api/csv/typeParsers/specs/generatedid.spec.ts b/app/api/csv/typeParsers/specs/generatedid.spec.ts index 2497dde0e6..57c7a33729 100644 --- a/app/api/csv/typeParsers/specs/generatedid.spec.ts +++ b/app/api/csv/typeParsers/specs/generatedid.spec.ts @@ -5,12 +5,12 @@ describe('generatedid parser', () => { const templateProp = { name: 'id', label: 'id', type: propertyTypes.generatedid }; it('should return the value if the property has one', async () => { - const rawEntity = { id: 'XYZ123', language: 'en' }; + const rawEntity = { propertiesFromColumns: { id: 'XYZ123' }, language: 'en' }; expect(await typeParsers.generatedid(rawEntity, templateProp)).toEqual([{ value: 'XYZ123' }]); }); it('should return a generated id if the property is empty', async () => { - const rawEntity = { id: '', language: 'en' }; + const rawEntity = { propertiesFromColumns: { id: '' }, language: 'en' }; const [propertyValue] = await typeParsers.generatedid(rawEntity, templateProp); expect(propertyValue.value).toEqual(expect.stringMatching(/^[a-zA-Z0-9-]{12}$/)); }); diff --git a/app/api/csv/typeParsers/specs/geolocation.spec.js b/app/api/csv/typeParsers/specs/geolocation.spec.js index 2753c1481e..5dd1f4318c 100644 --- a/app/api/csv/typeParsers/specs/geolocation.spec.js +++ b/app/api/csv/typeParsers/specs/geolocation.spec.js @@ -1,9 +1,15 @@ import typeParsers from '../../typeParsers'; +const rawEntityWithGeoValue = stringVal => ({ + propertiesFromColumns: { + geolocation_prop: stringVal, + }, +}); + describe('geolocation parser', () => { it('should build a geolocation type object', async () => { const templateProp = { name: 'geolocation_prop' }; - const rawEntity = { geolocation_prop: '1.5|45.65' }; + const rawEntity = rawEntityWithGeoValue('1.5|45.65'); expect(await typeParsers.geolocation(rawEntity, templateProp)).toEqual([ { value: { lat: 1.5, lon: 45.65, label: '' } }, @@ -12,7 +18,7 @@ describe('geolocation parser', () => { it('should work on 0 values', async () => { const templateProp = { name: 'geolocation_prop' }; - const rawEntity = { geolocation_prop: '0|0' }; + const rawEntity = rawEntityWithGeoValue('0|0'); expect(await typeParsers.geolocation(rawEntity, templateProp)).toEqual([ { value: { lat: 0, lon: 0, label: '' } }, @@ -22,7 +28,7 @@ describe('geolocation parser', () => { describe('when there is only one value', () => { it('should return empty array', async () => { const templateProp = { name: 'geolocation_prop' }; - const rawEntity = { geolocation_prop: 'oneValue' }; + const rawEntity = rawEntityWithGeoValue('oneValue'); expect(await typeParsers.geolocation(rawEntity, templateProp)).toEqual([]); }); diff --git a/app/api/csv/typeParsers/specs/multiselect.spec.js b/app/api/csv/typeParsers/specs/multiselect.spec.js index 208ff404b4..8f51be9b31 100644 --- a/app/api/csv/typeParsers/specs/multiselect.spec.js +++ b/app/api/csv/typeParsers/specs/multiselect.spec.js @@ -6,6 +6,12 @@ import db from 'api/utils/testing_db'; import { fixtures, thesauri1Id } from '../../specs/fixtures'; import typeParsers from '../../typeParsers'; +const rawEntityWithMultiselectValue = val => ({ + propertiesFromColumns: { + multiselect_prop: val, + }, +}); + describe('multiselect', () => { let value1; let value2; @@ -18,26 +24,26 @@ describe('multiselect', () => { afterAll(async () => db.disconnect()); beforeAll(async () => { await db.clearAllAndLoad(fixtures); - value1 = await typeParsers.multiselect({ multiselect_prop: 'value4' }, templateProp); + value1 = await typeParsers.multiselect(rawEntityWithMultiselectValue('value4'), templateProp); value2 = await typeParsers.multiselect( - { multiselect_prop: 'Value1|value3| value3' }, + rawEntityWithMultiselectValue('Value1|value3| value3'), templateProp ); value3 = await typeParsers.multiselect( - { multiselect_prop: 'value1| value2 | Value3' }, + rawEntityWithMultiselectValue('value1| value2 | Value3'), templateProp ); value4 = await typeParsers.multiselect( - { multiselect_prop: 'value1|value2|VALUE4' }, + rawEntityWithMultiselectValue('value1|value2|VALUE4'), templateProp ); - await typeParsers.multiselect({ multiselect_prop: '' }, templateProp); + await typeParsers.multiselect(rawEntityWithMultiselectValue(''), templateProp); - await typeParsers.multiselect({ multiselect_prop: '|' }, templateProp); + await typeParsers.multiselect(rawEntityWithMultiselectValue('|'), templateProp); thesauri1 = await thesauri.getById(thesauri1Id); }); diff --git a/app/api/csv/typeParsers/specs/relationship.spec.js b/app/api/csv/typeParsers/specs/relationship.spec.js index 9897d94d76..a98a8240a0 100644 --- a/app/api/csv/typeParsers/specs/relationship.spec.js +++ b/app/api/csv/typeParsers/specs/relationship.spec.js @@ -5,6 +5,13 @@ import db from 'api/utils/testing_db'; import { fixtures, templateToRelateId } from '../../specs/fixtures'; import typeParsers from '../../typeParsers'; +const rawEntityWithRelationshipValue = (val, language, propname = 'relationship_prop') => ({ + propertiesFromColumns: { + [propname]: val, + }, + language, +}); + describe('relationship', () => { let value1; let value2; @@ -43,22 +50,25 @@ describe('relationship', () => { const runScenarios = async () => { value1 = await typeParsers.relationship( - { relationship_prop: 'value1|value3|value3', language: 'en' }, + rawEntityWithRelationshipValue('value1|value3|value3', 'en'), templateProp ); value2 = await typeParsers.relationship( - { relationship_prop: 'value1|value2', language: 'en' }, + rawEntityWithRelationshipValue('value1|value2', 'en'), templateProp ); value3 = await typeParsers.relationship( - { relationship_prop: 'value1|value2', language: 'en' }, + rawEntityWithRelationshipValue('value1|value2', 'en'), templateProp ); - await typeParsers.relationship({ relationship_prop: '' }, templateProp); - await typeParsers.relationship({ relationship_prop: '|' }, templateProp); - await typeParsers.relationship({ relationship_no_content: 'newValue' }, noContentTemplateProp); + await typeParsers.relationship(rawEntityWithRelationshipValue(''), templateProp); + await typeParsers.relationship(rawEntityWithRelationshipValue('|'), templateProp); + await typeParsers.relationship( + rawEntityWithRelationshipValue('newValue', undefined, 'relationship_no_content'), + noContentTemplateProp + ); }; beforeAll(async () => { diff --git a/app/api/csv/typeParsers/specs/select.spec.js b/app/api/csv/typeParsers/specs/select.spec.js index 8eccf11f9f..5907fdda4f 100644 --- a/app/api/csv/typeParsers/specs/select.spec.js +++ b/app/api/csv/typeParsers/specs/select.spec.js @@ -6,6 +6,12 @@ import db from 'api/utils/testing_db'; import { fixtures, thesauri1Id } from '../../specs/fixtures'; import typeParsers from '../../typeParsers'; +const rawEntityWithSelectValue = val => ({ + propertiesFromColumns: { + select_prop: val, + }, +}); + describe('select', () => { beforeEach(async () => db.clearAllAndLoad(fixtures)); afterAll(async () => db.disconnect()); @@ -13,8 +19,8 @@ describe('select', () => { it('should find thesauri value and return the id and value', async () => { const templateProp = { name: 'select_prop', content: thesauri1Id }; - const value1 = await typeParsers.select({ select_prop: 'value1' }, templateProp); - const value2 = await typeParsers.select({ select_prop: 'vAlUe2' }, templateProp); + const value1 = await typeParsers.select(rawEntityWithSelectValue('value1'), templateProp); + const value2 = await typeParsers.select(rawEntityWithSelectValue('vAlUe2'), templateProp); const thesauri1 = await thesauri.getById(thesauri1Id); expect(value1).toEqual([{ value: thesauri1.values[0].id, label: 'value1' }]); @@ -23,7 +29,7 @@ describe('select', () => { it('should return null on blank values', async () => { const templateProp = { name: 'select_prop', content: thesauri1Id }; - const rawEntity = { select_prop: ' ' }; + const rawEntity = rawEntityWithSelectValue(''); const value = await typeParsers.select(rawEntity, templateProp); diff --git a/app/api/files/specs/fixtures.ts b/app/api/files/specs/fixtures.ts index 11e60ffe78..89099bd8ec 100644 --- a/app/api/files/specs/fixtures.ts +++ b/app/api/files/specs/fixtures.ts @@ -166,7 +166,6 @@ const fixtures: DBFixture = { propertyName: 'property 1', extractorId: fixturesFactory.id('property_1_extractor'), date: 1654002449676, - state: 'Empty / Label', segment: '', suggestedValue: '', }, @@ -179,7 +178,6 @@ const fixtures: DBFixture = { propertyName: 'property 2', extractorId: fixturesFactory.id('property_2_extractor'), date: 1654002449676, - state: 'Empty / Label', segment: '', suggestedValue: '', }, @@ -192,7 +190,6 @@ const fixtures: DBFixture = { propertyName: 'property 1', extractorId: fixturesFactory.id('property_1_extractor'), date: 1654002449676, - state: 'Empty / Label', segment: '', suggestedValue: '', }, @@ -205,7 +202,6 @@ const fixtures: DBFixture = { propertyName: 'property 2', extractorId: fixturesFactory.id('property_2_extractor'), date: 1654002449676, - state: 'Empty / Label', segment: '', suggestedValue: '', }, diff --git a/app/api/migrations/migrations/146-remove_obsolete_mongo_index/index.js b/app/api/migrations/migrations/146-remove_obsolete_mongo_index/index.js new file mode 100644 index 0000000000..4cde56b82a --- /dev/null +++ b/app/api/migrations/migrations/146-remove_obsolete_mongo_index/index.js @@ -0,0 +1,37 @@ +const INDICES_TO_REMOVE = { + ixsuggestions: ['extractorId_1_date_1_state_-1', 'extractorId_1_entityTemplate_1_state_1'], +}; + +const handleCollection = async (collection, indexNames) => { + for (let j = 0; j < indexNames.length; j += 1) { + const indexName = indexNames[j]; + // eslint-disable-next-line no-await-in-loop + if (await collection.indexExists(indexName)) await collection.dropIndex(indexName); + } +}; + +export default { + delta: 146, + + name: 'remove_obsolete_mongo_index', + + description: 'Removes one or more obsolete indices from mongodb.', + + reindex: false, + + async up(db) { + process.stdout.write(`${this.name}...\r\n`); + + const existingCollections = new Set( + (await db.collections()).map(collection => collection.collectionName) + ); + const indices = Object.entries(INDICES_TO_REMOVE).filter(pair => + existingCollections.has(pair[0]) + ); + + for (let i = 0; i < indices.length; i += 1) { + // eslint-disable-next-line no-await-in-loop + await handleCollection(await db.collection(indices[i][0]), indices[i][1]); + } + }, +}; diff --git a/app/api/migrations/migrations/146-remove_obsolete_mongo_index/specs/146-remove_obsolete_mongo_index.spec.js b/app/api/migrations/migrations/146-remove_obsolete_mongo_index/specs/146-remove_obsolete_mongo_index.spec.js new file mode 100644 index 0000000000..6ff488671c --- /dev/null +++ b/app/api/migrations/migrations/146-remove_obsolete_mongo_index/specs/146-remove_obsolete_mongo_index.spec.js @@ -0,0 +1,57 @@ +import testingDB from 'api/utils/testing_db'; +import migration from '../index.js'; +import { fixtures } from './fixtures.js'; + +describe('migration remove_obsolete_mongo_index', () => { + let suggestionsIndexInfo; + + const createIndexes = async db => { + await db.collection('ixsuggestions').createIndex({ entityId: 1 }); + await db.collection('ixsuggestions').createIndex({ fileId: 1 }); + await db.collection('ixsuggestions').createIndex({ extractorId: 1, entityId: 1, fileId: 1 }); + await db.collection('ixsuggestions').createIndex({ extractorId: 1, date: 1, state: -1 }); + await db + .collection('ixsuggestions') + .createIndex({ extractorId: 1, entityTemplate: 1, state: 1 }); + }; + + const getIndexInfo = async db => { + suggestionsIndexInfo = await db.collection('ixsuggestions').indexInformation(); + }; + + beforeAll(async () => { + jest.spyOn(process.stdout, 'write').mockImplementation(() => {}); + await testingDB.setupFixturesAndContext(fixtures); + const db = testingDB.mongodb; + await createIndexes(db); + await migration.up(db); + await getIndexInfo(db); + }); + + afterAll(async () => { + await testingDB.disconnect(); + }); + + it('should have a delta number', () => { + expect(migration.delta).toBe(146); + }); + + it('should remove the targeted indices', async () => { + expect(suggestionsIndexInfo['extractorId_1_date_1_state_-1']).toBe(undefined); + expect(suggestionsIndexInfo.extractorId_1_entityTemplate_1_state_1).toBe(undefined); + }); + + it('should leave the other indices intact', async () => { + expect(suggestionsIndexInfo.entityId_1).toEqual([['entityId', 1]]); + expect(suggestionsIndexInfo.fileId_1).toEqual([['fileId', 1]]); + expect(suggestionsIndexInfo.extractorId_1_entityId_1_fileId_1).toEqual([ + ['extractorId', 1], + ['entityId', 1], + ['fileId', 1], + ]); + }); + + it('should check if a reindex is needed', async () => { + expect(migration.reindex).toBe(false); + }); +}); diff --git a/app/api/migrations/migrations/146-remove_obsolete_mongo_index/specs/fixtures.js b/app/api/migrations/migrations/146-remove_obsolete_mongo_index/specs/fixtures.js new file mode 100644 index 0000000000..cbf2144390 --- /dev/null +++ b/app/api/migrations/migrations/146-remove_obsolete_mongo_index/specs/fixtures.js @@ -0,0 +1,4 @@ +export const fixtures = { + entities: [{ title: 'test_doc' }], + ixsuggestions: [{ propertyName: 'some name', state: 'Obsolete' }], +}; diff --git a/app/api/migrations/migrations/147-update_translations/index.js b/app/api/migrations/migrations/147-update_translations/index.js new file mode 100644 index 0000000000..98b3dcded8 --- /dev/null +++ b/app/api/migrations/migrations/147-update_translations/index.js @@ -0,0 +1,108 @@ +const newKeys = [ + { key: 'Suggestion accepted.' }, + { key: 'Showing' }, + { key: 'Accept suggestion' }, + { key: 'Stats & Filters' }, + { key: 'Labeled' }, + { key: 'Non-labeled' }, + { key: 'Pending' }, + { key: 'Clear all' }, + { key: 'Apply' }, + { key: 'Current value:' }, + { key: 'Suggestion:' }, + { key: 'Current Value/Suggestion' }, + { key: 'No context' }, +]; + +const deletedKeys = [ + { key: 'Reviewing' }, + { key: 'Confirm suggestion acceptance' }, + { key: 'Apply to all languages' }, + { key: 'Back to dashboard' }, + { key: 'Match / Label' }, + { key: 'Mismatch / Label' }, + { key: 'Match / Value' }, + { key: 'Mismatch / Value' }, + { key: 'Empty / Label' }, + { key: 'Empty / Value' }, + { key: 'State Legend' }, + { key: 'labelMatchDesc' }, + { key: 'labelMismatchDesc' }, + { key: 'labelEmptyDesc' }, + { key: 'valueMatchDesc' }, + { key: 'valueMismatchDesc' }, + { key: 'valueEmptyDesc' }, + { key: 'obsoleteDesc' }, + { key: 'emptyDesc' }, + { key: 'This will update the entity across all languages' }, + { key: 'Mismatch / Empty' }, + { key: 'Empty / Empty' }, + { key: 'emptyMismatchDesc' }, + { key: 'Non-matching' }, + { key: 'Empty / Obsolete' }, + { key: 'This will cancel the finding suggestion process' }, + { key: 'Add properties' }, + { key: 'Show Filters' }, +]; +const updateTranslation = (currentTranslation, keysToUpdate, loc) => { + const translation = { ...currentTranslation }; + const newTranslation = keysToUpdate.find(row => row.key === currentTranslation.key); + if (newTranslation) { + translation.key = newTranslation.newKey; + if (loc === 'en' || currentTranslation.value === newTranslation.oldValue) { + translation.value = newTranslation.newValue; + } + } + return translation; +}; + +export default { + delta: 147, + + reindex: false, + + name: 'update_translations', + + description: 'Updates some translations for new User/Groups UI in settings', + + async up(db) { + const keysToInsert = newKeys; + const keysToDelete = deletedKeys; + const translations = await db.collection('translations').find().toArray(); + const locToSystemContext = {}; + translations.forEach(tr => { + locToSystemContext[tr.locale] = tr.contexts.find(c => c.id === 'System'); + }); + + const alreadyInDB = []; + Object.entries(locToSystemContext).forEach(([loc, context]) => { + const contextValues = context.values.reduce((newValues, currentTranslation) => { + const deleted = keysToDelete.find( + deletedTranslation => deletedTranslation.key === currentTranslation.key + ); + if (!deleted) { + const translation = updateTranslation(currentTranslation, [], loc); + newValues.push(translation); + } + keysToInsert.forEach(newEntry => { + if (newEntry.key === currentTranslation.key) { + alreadyInDB.push(currentTranslation.key); + } + }); + return newValues; + }, []); + keysToInsert + .filter(k => !alreadyInDB.includes(k.key)) + .forEach(newEntry => { + contextValues.push({ key: newEntry.key, value: newEntry.key }); + }); + context.values = contextValues; + }); + + await Promise.all( + translations.map(tr => db.collection('translations').replaceOne({ _id: tr._id }, tr)) + ); + + process.stdout.write(`${this.name}...\r\n`); + }, +}; diff --git a/app/api/migrations/migrations/147-update_translations/specs/147-update_translations.spec.js b/app/api/migrations/migrations/147-update_translations/specs/147-update_translations.spec.js new file mode 100644 index 0000000000..8c347dea29 --- /dev/null +++ b/app/api/migrations/migrations/147-update_translations/specs/147-update_translations.spec.js @@ -0,0 +1,75 @@ +import testingDB from 'api/utils/testing_db'; +import migration from '../index.js'; +import fixtures, { templateContext } from './fixtures.js'; + +describe('migration update translations of settings new Users/Groups UI', () => { + beforeEach(async () => { + jest.spyOn(process.stdout, 'write').mockImplementation(() => {}); + await testingDB.setupFixturesAndContext(fixtures); + }); + + afterAll(async () => { + await testingDB.disconnect(); + }); + + it('should have a delta number', () => { + expect(migration.delta).toBe(147); + }); + + it('should update the keys that have changed', async () => { + await migration.up(testingDB.mongodb); + const allTranslations = await testingDB.mongodb.collection('translations').find().toArray(); + + const uwaziUI = allTranslations.filter(tr => + tr.contexts.filter(ctx => ctx.type === 'Uwazi UI') + ); + + const previousSystemValues = { + key: 'existing-key-in-system', + value: 'existing-key-in-system', + }; + + const addedKeys = [ + expect.objectContaining({ + key: 'Suggestion accepted.', + value: 'Suggestion accepted.', + }), + expect.objectContaining({ + key: 'Accept suggestion', + value: 'Accept suggestion', + }), + expect.objectContaining({ + key: 'Showing', + value: 'Showing', + }), + expect.objectContaining({ + key: 'Stats & Filters', + value: 'Stats & Filters', + }), + ]; + const defaultContextContent = expect.objectContaining({ + type: 'Uwazi UI', + values: expect.arrayContaining([previousSystemValues, ...addedKeys]), + }); + expect(uwaziUI).toMatchObject([ + expect.objectContaining({ + locale: 'en', + contexts: [defaultContextContent, templateContext], + }), + expect.objectContaining({ + locale: 'es', + contexts: [ + expect.objectContaining({ + type: 'Uwazi UI', + values: expect.arrayContaining([previousSystemValues, ...addedKeys]), + }), + templateContext, + ], + }), + expect.objectContaining({ + locale: 'pt', + contexts: [defaultContextContent, templateContext], + }), + ]); + }); +}); diff --git a/app/api/migrations/migrations/147-update_translations/specs/fixtures.js b/app/api/migrations/migrations/147-update_translations/specs/fixtures.js new file mode 100644 index 0000000000..f9a29257a5 --- /dev/null +++ b/app/api/migrations/migrations/147-update_translations/specs/fixtures.js @@ -0,0 +1,115 @@ +import db from 'api/utils/testing_db'; + +const templateContext = { + id: db.id(), + label: 'default template', + type: 'Entity', + values: [ + { + key: 'default template', + value: 'default template', + }, + { + key: 'Title', + value: 'Title', + }, + ], +}; + +const fixturesDB = { + translations: [ + { + _id: db.id(), + locale: 'en', + contexts: [ + { + _id: db.id(), + type: 'Uwazi UI', + label: 'User Interface', + id: 'System', + values: [ + { + key: 'existing-key-in-system', + value: 'existing-key-in-system', + }, + { + _id: db.id(), + key: 'Can not delete template:', + value: 'Can not delete template: changed', + }, + { + _id: db.id(), + key: '- Site page:', + value: '- Site page:', + }, + { _id: db.id(), key: 'Document OCR trigger', value: 'Document OCR trigger' }, + ], + }, + templateContext, + ], + }, + { + _id: db.id(), + locale: 'es', + contexts: [ + { + _id: db.id(), + type: 'Uwazi UI', + label: 'User Interface', + id: 'System', + values: [ + { + key: 'existing-key-in-system', + value: 'existing-key-in-system', + }, + { + _id: db.id(), + key: 'Confirm delete relationship type:', + value: 'Confirmar eliminación de tipo de relación:', + }, + { + _id: db.id(), + key: '- Site page:', + value: '- Sito:', + }, + { _id: db.id(), key: 'Document OCR trigger', value: 'Document OCR trigger' }, + ], + }, + templateContext, + ], + }, + { + _id: db.id(), + locale: 'pt', + contexts: [ + { + _id: db.id(), + type: 'Uwazi UI', + label: 'User Interface', + id: 'System', + values: [ + { + key: 'existing-key-in-system', + value: 'existing-key-in-system', + }, + { + _id: db.id(), + key: 'Can not delete template:', + value: 'Can not delete template:', + }, + { + _id: db.id(), + key: '- Site page:', + value: '- Site page:', + }, + { _id: db.id(), key: 'Document OCR trigger', value: 'Document OCR trigger' }, + ], + }, + templateContext, + ], + }, + ], +}; + +export { templateContext }; +export default fixturesDB; diff --git a/app/api/odm/specs/DB.spec.ts b/app/api/odm/specs/DB.spec.ts index ca5c1fb608..85ac82fd1d 100644 --- a/app/api/odm/specs/DB.spec.ts +++ b/app/api/odm/specs/DB.spec.ts @@ -4,6 +4,7 @@ import waitForExpect from 'wait-for-expect'; import { tenants } from 'api/tenants/tenantContext'; import { testingTenants } from 'api/utils/testingTenants'; +import { config } from 'api/config'; import { DB } from '../DB'; import { instanceModel } from '../model'; @@ -24,7 +25,7 @@ describe('DB', () => { let db2: Db; beforeEach(async () => { - const uri = 'mongodb://localhost/'; + const uri = config.DBHOST; await DB.connect(`${uri}_DB_spec_ts`); db1 = DB.getConnection().useDb('db1').db; db2 = DB.getConnection().useDb('db2').db; diff --git a/app/api/services/informationextraction/InformationExtraction.ts b/app/api/services/informationextraction/InformationExtraction.ts index fa68352937..9403417b57 100644 --- a/app/api/services/informationextraction/InformationExtraction.ts +++ b/app/api/services/informationextraction/InformationExtraction.ts @@ -370,7 +370,7 @@ class InformationExtraction { return { status: 'ready', message: 'Ready' }; } - return { status: 'error', message: '' }; + return { status: 'error', message: 'No model found' }; }; materialsForModel = async (extractor: IXExtractorType, serviceUrl: string) => { diff --git a/app/api/services/informationextraction/getFiles.ts b/app/api/services/informationextraction/getFiles.ts index 1664d904c0..c4500dec24 100644 --- a/app/api/services/informationextraction/getFiles.ts +++ b/app/api/services/informationextraction/getFiles.ts @@ -14,7 +14,6 @@ import settings from 'api/settings/settings'; import templatesModel from 'api/templates/templates'; import { propertyTypes } from 'shared/propertyTypes'; import languages from 'shared/languages'; -import { SuggestionState } from 'shared/types/suggestionSchema'; const BATCH_SIZE = 50; const MAX_TRAINING_FILES_NUMBER = 500; @@ -130,7 +129,7 @@ async function getFilesForSuggestions(extractorId: ObjectIdSchema) { { extractorId, date: { $lt: currentModel.creationDate }, - state: { $ne: SuggestionState.error }, + 'state.error': { $ne: true }, }, 'fileId', { limit: BATCH_SIZE } diff --git a/app/api/services/informationextraction/specs/InformationExtraction.spec.ts b/app/api/services/informationextraction/specs/InformationExtraction.spec.ts index 7a20ee654e..8df449e769 100644 --- a/app/api/services/informationextraction/specs/InformationExtraction.spec.ts +++ b/app/api/services/informationextraction/specs/InformationExtraction.spec.ts @@ -1,12 +1,13 @@ /* eslint-disable max-lines */ +// eslint-disable-next-line node/no-restricted-import +import fs from 'fs/promises'; + import { testingEnvironment } from 'api/utils/testingEnvironment'; import { testingTenants } from 'api/utils/testingTenants'; import { IXSuggestionsModel } from 'api/suggestions/IXSuggestionsModel'; -import { SuggestionState } from 'shared/types/suggestionSchema'; import { ResultsMessage } from 'api/services/tasksmanager/TaskManager'; import * as setupSockets from 'api/socketio/setupSockets'; -// eslint-disable-next-line node/no-restricted-import -import fs from 'fs/promises'; + import { factory, fixtures } from './fixtures'; import { InformationExtraction } from '../InformationExtraction'; import { ExternalDummyService } from '../../tasksmanager/specs/ExternalDummyService'; @@ -301,7 +302,16 @@ describe('InformationExtraction', () => { expect.objectContaining({ entityId: 'A1', status: 'processing', - state: SuggestionState.processing, + state: { + labeled: true, + withValue: true, + withSuggestion: true, + match: false, + hasContext: true, + processing: true, + obsolete: false, + error: false, + }, }) ); }); @@ -379,7 +389,16 @@ describe('InformationExtraction', () => { suggestedValue: 'suggestion_text_1', segment: 'segment_text_1', status: 'ready', - state: SuggestionState.labelMismatch, + state: { + labeled: true, + withValue: true, + withSuggestion: true, + match: false, + hasContext: true, + processing: false, + obsolete: false, + error: false, + }, }) ); }); @@ -444,7 +463,16 @@ describe('InformationExtraction', () => { propertyName: 'property1', status: 'ready', suggestedValue: 'text_in_other_language', - state: SuggestionState.labelMismatch, + state: { + labeled: true, + withValue: true, + withSuggestion: true, + match: false, + hasContext: true, + processing: false, + obsolete: false, + error: false, + }, }) ); @@ -454,7 +482,16 @@ describe('InformationExtraction', () => { propertyName: 'property1', status: 'ready', suggestedValue: 'text_in_eng_language', - state: SuggestionState.valueMismatch, + state: { + labeled: false, + withValue: true, + withSuggestion: true, + match: false, + hasContext: true, + processing: false, + obsolete: false, + error: false, + }, }) ); }); @@ -491,7 +528,16 @@ describe('InformationExtraction', () => { segment: '', status: 'failed', error: 'Issue calculation suggestion', - state: SuggestionState.error, + state: { + labeled: true, + withValue: true, + withSuggestion: false, + match: false, + hasContext: false, + processing: false, + obsolete: false, + error: true, + }, }) ); }); diff --git a/app/api/services/informationextraction/specs/fixtures.ts b/app/api/services/informationextraction/specs/fixtures.ts index 7b40c53432..0155e1542e 100644 --- a/app/api/services/informationextraction/specs/fixtures.ts +++ b/app/api/services/informationextraction/specs/fixtures.ts @@ -289,7 +289,16 @@ const fixtures: DBFixture = { suggestedValue: '', segment: '', status: 'ready', - state: 'Obsolete', + state: { + labeled: false, + withValue: false, + withSuggestion: false, + match: false, + hasContext: false, + obsolete: true, + processing: false, + error: false, + }, date: 100, }, { @@ -302,7 +311,16 @@ const fixtures: DBFixture = { suggestedValue: '', segment: '', status: 'ready', - state: 'Error', + state: { + labeled: false, + withValue: false, + withSuggestion: false, + match: false, + hasContext: false, + obsolete: true, + processing: false, + error: true, + }, date: 100, }, ], diff --git a/app/api/services/informationextraction/specs/ixextractors.spec.ts b/app/api/services/informationextraction/specs/ixextractors.spec.ts index a9a2826f2f..99075f6a98 100644 --- a/app/api/services/informationextraction/specs/ixextractors.spec.ts +++ b/app/api/services/informationextraction/specs/ixextractors.spec.ts @@ -4,7 +4,7 @@ import { Suggestions } from 'api/suggestions/suggestions'; import { getFixturesFactory } from 'api/utils/fixturesFactory'; import { testingEnvironment } from 'api/utils/testingEnvironment'; import db, { DBFixture, testingDB } from 'api/utils/testing_db'; -import { SuggestionState } from 'shared/types/suggestionSchema'; +import { IXSuggestionStateType } from 'shared/types/suggestionType'; import { Extractors } from '../ixextractors'; const fixtureFactory = getFixturesFactory(); @@ -125,6 +125,28 @@ const fixtures: DBFixture = { ], }; +const emptyState: IXSuggestionStateType = { + labeled: false, + withValue: false, + withSuggestion: false, + match: false, + hasContext: false, + obsolete: false, + processing: false, + error: false, +}; + +const expectedStates: Record = { + onlyLabeled: { + ...emptyState, + labeled: true, + }, + onlyValue: { + ...emptyState, + withValue: true, + }, +}; + describe('ixextractors', () => { beforeEach(async () => { await testingEnvironment.setUp(fixtures); @@ -161,7 +183,7 @@ describe('ixextractors', () => { error: '', segment: '', suggestedValue: '', - state: SuggestionState.labelEmpty, + state: expectedStates.onlyLabeled, entityTemplate: fixtureFactory.id('personTemplate').toString(), }, { @@ -173,7 +195,7 @@ describe('ixextractors', () => { error: '', segment: '', suggestedValue: '', - state: SuggestionState.labelEmpty, + state: expectedStates.onlyLabeled, entityTemplate: fixtureFactory.id('personTemplate').toString(), }, ], @@ -196,7 +218,7 @@ describe('ixextractors', () => { error: '', segment: '', suggestedValue: '', - state: SuggestionState.valueEmpty, + state: expectedStates.onlyValue, entityTemplate: fixtureFactory.id('animalTemplate').toString(), }, { @@ -208,7 +230,7 @@ describe('ixextractors', () => { error: '', segment: '', suggestedValue: '', - state: SuggestionState.valueEmpty, + state: expectedStates.onlyValue, entityTemplate: fixtureFactory.id('animalTemplate').toString(), }, { @@ -220,7 +242,7 @@ describe('ixextractors', () => { error: '', segment: '', suggestedValue: '', - state: SuggestionState.valueEmpty, + state: expectedStates.onlyValue, entityTemplate: fixtureFactory.id('personTemplate').toString(), }, { @@ -232,7 +254,7 @@ describe('ixextractors', () => { error: '', segment: '', suggestedValue: '', - state: SuggestionState.valueEmpty, + state: expectedStates.onlyValue, entityTemplate: fixtureFactory.id('personTemplate').toString(), }, ], diff --git a/app/api/services/pdfsegmentation/specs/PDFSegmentation.spec.ts b/app/api/services/pdfsegmentation/specs/PDFSegmentation.spec.ts index 529bf73b35..745b7dd882 100644 --- a/app/api/services/pdfsegmentation/specs/PDFSegmentation.spec.ts +++ b/app/api/services/pdfsegmentation/specs/PDFSegmentation.spec.ts @@ -20,11 +20,12 @@ import { DB } from 'api/odm'; import { Db } from 'mongodb'; import request from 'shared/JSONRequest'; +// eslint-disable-next-line node/no-restricted-import +import fs from 'fs/promises'; +import { config } from 'api/config'; import { PDFSegmentation } from '../PDFSegmentation'; import { SegmentationModel } from '../segmentationModel'; import { ExternalDummyService } from '../../tasksmanager/specs/ExternalDummyService'; -// eslint-disable-next-line node/no-restricted-import -import fs from 'fs/promises'; jest.mock('api/services/tasksmanager/TaskManager.ts'); @@ -72,7 +73,7 @@ describe('PDFSegmentation', () => { }); beforeAll(async () => { - const uri = 'mongodb://localhost/'; + const uri = config.DBHOST; await DB.connect(`${uri}PDFSegmentation_spec`); }); diff --git a/app/api/suggestions/IXSuggestionsModel.ts b/app/api/suggestions/IXSuggestionsModel.ts index b5c295b66b..070e0aceb2 100644 --- a/app/api/suggestions/IXSuggestionsModel.ts +++ b/app/api/suggestions/IXSuggestionsModel.ts @@ -15,8 +15,11 @@ const mongoSchema = new mongoose.Schema(props, { mongoSchema.index({ entityId: 1 }); mongoSchema.index({ fileId: 1 }); mongoSchema.index({ extractorId: 1, entityId: 1, fileId: 1 }); -mongoSchema.index({ extractorId: 1, date: 1, state: -1 }); -mongoSchema.index({ extractorId: 1, entityTemplate: 1, state: 1 }); +mongoSchema.index({ extractorId: 1, 'state.labeled': 1, 'state.match': 1 }); +mongoSchema.index({ extractorId: 1, 'tate.labeled': 1, 'state.withSuggestion': 1 }); +mongoSchema.index({ extractorId: 1, 'state.labeled': 1, 'state.hasContext': 1 }); +mongoSchema.index({ extractorId: 1, 'state.labeled': 1, 'state.obsolete': 1 }); +mongoSchema.index({ extractorId: 1, 'state.labeled': 1, 'state.error': 1 }); const IXSuggestionsModel = instanceModel('ixsuggestions', mongoSchema); diff --git a/app/api/suggestions/pipelineStages.ts b/app/api/suggestions/pipelineStages.ts index 695c62ffbf..3e065c1e74 100644 --- a/app/api/suggestions/pipelineStages.ts +++ b/app/api/suggestions/pipelineStages.ts @@ -1,16 +1,79 @@ import { ObjectId } from 'mongodb'; import { FilterQuery } from 'mongoose'; import { LanguagesListSchema } from 'shared/types/commonTypes'; -import { IXSuggestionType } from 'shared/types/suggestionType'; +import { IXSuggestionType, SuggestionCustomFilter } from 'shared/types/suggestionType'; -export const getMatchStage = (filters: FilterQuery) => [ - { - $match: { - ...filters, - status: { $ne: 'processing' }, - }, +export const baseQueryFragment = (extractorId: ObjectId, ignoreProcessing = true) => { + const query: FilterQuery = { + extractorId, + }; + if (ignoreProcessing) { + query.status = { $ne: 'processing' }; + } + return query; +}; + +export const filterFragments = { + labeled: { + _fragment: { 'state.labeled': true }, + match: { 'state.labeled': true, 'state.match': true }, + mismatch: { 'state.labeled': true, 'state.match': false }, }, -]; + nonLabeled: { + _fragment: { 'state.labeled': false }, + noSuggestion: { 'state.labeled': false, 'state.withSuggestion': false }, + noContext: { 'state.labeled': false, 'state.hasContext': false }, + obsolete: { 'state.labeled': false, 'state.obsolete': true }, + others: { 'state.labeled': false, 'state.error': true }, + }, +}; + +export const translateCustomFilter = (customFilter: SuggestionCustomFilter) => { + const orFilters = []; + if (customFilter.labeled.match) { + orFilters.push(filterFragments.labeled.match); + } + if (customFilter.labeled.mismatch) { + orFilters.push(filterFragments.labeled.mismatch); + } + + if (customFilter.nonLabeled.noSuggestion) { + orFilters.push(filterFragments.nonLabeled.noSuggestion); + } + if (customFilter.nonLabeled.noContext) { + orFilters.push(filterFragments.nonLabeled.noContext); + } + if (customFilter.nonLabeled.obsolete) { + orFilters.push(filterFragments.nonLabeled.obsolete); + } + if (customFilter.nonLabeled.others) { + orFilters.push(filterFragments.nonLabeled.others); + } + return orFilters; +}; + +export const getMatchStage = ( + extractorId: ObjectId, + customFilter: SuggestionCustomFilter | undefined, + countOnly = false +) => { + const matchQuery: FilterQuery = baseQueryFragment(extractorId); + if (customFilter) { + const orFilters = translateCustomFilter(customFilter); + if (orFilters.length > 0) matchQuery.$or = orFilters; + } + + const countExpression = countOnly ? [{ $count: 'count' }] : []; + + const matchStage = [ + { + $match: matchQuery, + }, + ...countExpression, + ]; + + return matchStage; +}; export const getEntityStage = (languages: LanguagesListSchema) => { const defaultLanguage = languages.find(l => l.default)?.key; @@ -152,12 +215,11 @@ export const getEntityTemplateFilterStage = (entityTemplates: string[] | undefin ] : []; -export const groupByAndSort = (field: string) => [ +export const groupByAndCount = (field: string) => [ { $group: { _id: field, count: { $sum: 1 }, }, }, - { $sort: { _id: 1 } }, ]; diff --git a/app/api/suggestions/routes.ts b/app/api/suggestions/routes.ts index f59f8aeb6f..01bc8d1679 100644 --- a/app/api/suggestions/routes.ts +++ b/app/api/suggestions/routes.ts @@ -1,5 +1,5 @@ /* eslint-disable max-lines */ -import { Application, NextFunction, Request, Response } from 'express'; +import { Application, Request, Response } from 'express'; import { ObjectId } from 'mongodb'; import { Suggestions } from 'api/suggestions/suggestions'; @@ -7,14 +7,15 @@ import { InformationExtraction } from 'api/services/informationextraction/Inform import { validateAndCoerceRequest } from 'api/utils/validateRequest'; import { needsAuthorization } from 'api/auth'; import { parseQuery } from 'api/utils/parseQueryMiddleware'; -import { - IXSuggestionsStatsQuerySchema, - SuggestionsQueryFilterSchema, -} from 'shared/types/suggestionSchema'; +import { ObjectIdSchema } from 'shared/types/commonTypes'; +import { SuggestionsQueryFilterSchema } from 'shared/types/suggestionSchema'; import { objectIdSchema } from 'shared/types/commonSchemas'; -import { IXSuggestionsFilter, IXSuggestionsStatsQuery } from 'shared/types/suggestionType'; +import { + IXAggregationQuery, + IXSuggestionAggregation, + IXSuggestionsQuery, +} from 'shared/types/suggestionType'; import { serviceMiddleware } from './serviceMiddleware'; -import { ObjectIdSchema } from 'shared/types/commonTypes'; const IX = new InformationExtraction(); @@ -72,42 +73,59 @@ export const suggestionsRoutes = (app: Application) => { size: { type: 'number', minimum: 1, maximum: 500 }, }, }, + sort: { + type: 'object', + properties: { + property: { type: 'string' }, + order: { type: 'string' }, + }, + }, }, }, }, }), async ( req: Request & { - query: { filter: IXSuggestionsFilter; page: { number: number; size: number } }; + query: IXSuggestionsQuery; }, - res: Response, - _next: NextFunction + res: Response ) => { - const suggestionsList = await Suggestions.get( - { language: req.language, ...req.query.filter }, - { page: req.query.page } - ); + const suggestionsList = await Suggestions.get(req.query.filter, { + page: req.query.page, + sort: req.query.sort, + }); res.json(suggestionsList); } ); app.get( - '/api/suggestions/stats', + '/api/suggestions/aggregation', serviceMiddleware, needsAuthorization(['admin']), parseQuery, validateAndCoerceRequest({ + type: 'object', + definitions: { objectIdSchema }, properties: { - query: IXSuggestionsStatsQuerySchema, + query: { + type: 'object', + additionalProperties: false, + required: ['extractorId'], + properties: { + extractorId: objectIdSchema, + }, + }, }, }), async ( - req: Request & { query: IXSuggestionsStatsQuery }, - res: Response, - _next: NextFunction + req: Request & { + query: IXAggregationQuery; + }, + res: Response ) => { - const stats = await Suggestions.getStats(req.query.extractorId); - res.json(stats); + const { extractorId } = req.query; + const aggregation = await Suggestions.aggregate(extractorId); + res.json(aggregation); } ); @@ -151,26 +169,28 @@ export const suggestionsRoutes = (app: Application) => { body: { type: 'object', additionalProperties: false, - required: ['suggestion', 'allLanguages'], + required: ['suggestions'], properties: { - suggestion: { - type: 'object', - additionalProperties: false, - required: ['_id', 'sharedId', 'entityId'], - properties: { - _id: objectIdSchema, - sharedId: { type: 'string' }, - entityId: { type: 'string' }, + suggestions: { + type: 'array', + items: { + type: 'object', + additionalProperties: false, + required: ['_id', 'sharedId', 'entityId'], + properties: { + _id: objectIdSchema, + sharedId: { type: 'string' }, + entityId: { type: 'string' }, + }, }, }, - allLanguages: { type: 'boolean' }, }, }, }, }), async (req, res, _next) => { - const { suggestion, allLanguages } = req.body; - await Suggestions.accept(suggestion, allLanguages); + const { suggestions } = req.body; + await Suggestions.accept(suggestions); res.json({ success: true }); } ); diff --git a/app/api/suggestions/specs/customFilters.spec.ts b/app/api/suggestions/specs/customFilters.spec.ts new file mode 100644 index 0000000000..1ecab21715 --- /dev/null +++ b/app/api/suggestions/specs/customFilters.spec.ts @@ -0,0 +1,223 @@ +import db from 'api/utils/testing_db'; +import { SuggestionCustomFilter } from 'shared/types/suggestionType'; +import { factory, stateFilterFixtures } from './fixtures'; +import { Suggestions } from '../suggestions'; + +const blankCustomFilter: SuggestionCustomFilter = { + labeled: { + match: false, + mismatch: false, + }, + nonLabeled: { + noSuggestion: false, + noContext: false, + obsolete: false, + others: false, + }, +}; + +beforeAll(async () => { + await db.setupFixturesAndContext(stateFilterFixtures); + await Suggestions.updateStates({}); +}); + +afterAll(async () => db.disconnect()); + +describe('suggestions with CustomFilters', () => { + describe('get()', () => { + it('should return all suggestions (except processing) when no custom filter is provided', async () => { + const result = await Suggestions.get( + { + extractorId: factory.id('test_extractor').toString(), + }, + {} + ); + expect(result.suggestions).toMatchObject([ + { sharedId: 'unlabeled-obsolete', language: 'en' }, + { sharedId: 'unlabeled-obsolete', language: 'es' }, + { sharedId: 'labeled-match', language: 'en' }, + { sharedId: 'labeled-match', language: 'es' }, + { sharedId: 'labeled-mismatch', language: 'en' }, + { sharedId: 'labeled-mismatch', language: 'es' }, + { sharedId: 'unlabeled-error', language: 'en' }, + { sharedId: 'unlabeled-error', language: 'es' }, + { sharedId: 'unlabeled-no-context', language: 'en' }, + { sharedId: 'unlabeled-no-context', language: 'es' }, + { sharedId: 'unlabeled-no-suggestion', language: 'en' }, + { sharedId: 'unlabeled-no-suggestion', language: 'es' }, + ]); + }); + + it('should be able to paginate', async () => { + const result = await Suggestions.get( + { + extractorId: factory.id('test_extractor').toString(), + }, + { page: { number: 3, size: 2 } } + ); + expect(result.suggestions).toMatchObject([ + { sharedId: 'labeled-mismatch', language: 'es' }, + { sharedId: 'labeled-mismatch', language: 'en' }, + ]); + }); + + it.each([ + { + description: 'filtering for labeled - match', + customFilter: { + ...blankCustomFilter, + labeled: { + match: true, + mismatch: false, + }, + }, + expectedSuggestions: [ + { sharedId: 'labeled-match', language: 'en' }, + { sharedId: 'labeled-match', language: 'es' }, + ], + }, + { + description: 'filtering for labeled - mismatch', + customFilter: { + ...blankCustomFilter, + labeled: { + match: false, + mismatch: true, + }, + }, + expectedSuggestions: [ + { sharedId: 'labeled-mismatch', language: 'en' }, + { sharedId: 'labeled-mismatch', language: 'es' }, + ], + }, + { + description: 'filtering for nonLabeled - noSuggestion', + customFilter: { + ...blankCustomFilter, + nonLabeled: { + ...blankCustomFilter.nonLabeled, + noSuggestion: true, + }, + }, + expectedSuggestions: [ + { sharedId: 'unlabeled-no-suggestion', language: 'en' }, + { sharedId: 'unlabeled-no-suggestion', language: 'es' }, + ], + }, + { + description: 'filtering for nonLabeled - noContext', + customFilter: { + ...blankCustomFilter, + nonLabeled: { + ...blankCustomFilter.nonLabeled, + noContext: true, + }, + }, + expectedSuggestions: [ + { sharedId: 'unlabeled-no-context', language: 'en' }, + { sharedId: 'unlabeled-no-context', language: 'es' }, + { sharedId: 'unlabeled-no-suggestion', language: 'en' }, + { sharedId: 'unlabeled-no-suggestion', language: 'es' }, + ], + }, + { + description: 'filtering for nonLabeled - obsolete', + customFilter: { + ...blankCustomFilter, + nonLabeled: { + ...blankCustomFilter.nonLabeled, + obsolete: true, + }, + }, + expectedSuggestions: [ + { sharedId: 'unlabeled-obsolete', language: 'en' }, + { sharedId: 'unlabeled-obsolete', language: 'es' }, + ], + }, + { + description: 'filtering for nonLabeled - others', + customFilter: { + ...blankCustomFilter, + nonLabeled: { + ...blankCustomFilter.nonLabeled, + others: true, + }, + }, + expectedSuggestions: [ + { sharedId: 'unlabeled-error', language: 'en' }, + { sharedId: 'unlabeled-error', language: 'es' }, + ], + }, + { + description: 'filtering for labeled - match and nonLabeled - obsolete', + customFilter: { + ...blankCustomFilter, + labeled: { + match: true, + mismatch: false, + }, + nonLabeled: { + ...blankCustomFilter.nonLabeled, + obsolete: true, + }, + }, + expectedSuggestions: [ + { sharedId: 'unlabeled-obsolete', language: 'en' }, + { sharedId: 'unlabeled-obsolete', language: 'es' }, + { sharedId: 'labeled-match', language: 'en' }, + { sharedId: 'labeled-match', language: 'es' }, + ], + }, + { + description: 'filtering for nonLabeled - noSuggestion and nonLabeled - noContext', + customFilter: { + ...blankCustomFilter, + nonLabeled: { + ...blankCustomFilter.nonLabeled, + noSuggestion: true, + noContext: true, + }, + }, + expectedSuggestions: [ + { sharedId: 'unlabeled-no-context', language: 'en' }, + { sharedId: 'unlabeled-no-context', language: 'es' }, + { sharedId: 'unlabeled-no-suggestion', language: 'en' }, + { sharedId: 'unlabeled-no-suggestion', language: 'es' }, + ], + }, + ])( + 'should use the custom filter properly when $description', + async ({ customFilter, expectedSuggestions }) => { + const result = await Suggestions.get( + { + extractorId: factory.id('test_extractor').toString(), + customFilter, + }, + {} + ); + expect(result.suggestions).toMatchObject(expectedSuggestions); + } + ); + }); + + describe('aggreagate()', () => { + it('should return correct aggregation', async () => { + const result = await Suggestions.aggregate(factory.id('test_extractor').toString()); + expect(result).toMatchObject({ + total: 12, + labeled: { + _count: 4, + match: 2, + mismatch: 2, + }, + nonLabeled: { + _count: 8, + noSuggestion: 2, + noContext: 4, + obsolete: 2, + others: 2, + }, + }); + }); + }); +}); diff --git a/app/api/suggestions/specs/fixtures.ts b/app/api/suggestions/specs/fixtures.ts index d555806626..5a55c7a4a8 100644 --- a/app/api/suggestions/specs/fixtures.ts +++ b/app/api/suggestions/specs/fixtures.ts @@ -1,4 +1,6 @@ /* eslint-disable max-lines */ +import _ from 'lodash'; + import { getFixturesFactory } from 'api/utils/fixturesFactory'; import { testingDB, DBFixture } from 'api/utils/testing_db'; @@ -22,27 +24,29 @@ const shared2AgeSuggestionId = testingDB.id(); const file2Id = factory.id('F2'); const file3Id = factory.id('F3'); -const fixtures: DBFixture = { - settings: [ - { - languages: [ - { - default: true, - key: 'en', - label: 'English', - }, - { - key: 'es', - label: 'Spanish', - }, - ], - features: { - metadataExtraction: { - url: 'https://metadataextraction.com', - }, +const ixSettings = [ + { + languages: [ + { + default: true, + key: 'en' as 'en', + label: 'English', + }, + { + key: 'es' as 'es', + label: 'Spanish', + }, + ], + features: { + metadataExtraction: { + url: 'https://metadataextraction.com', }, }, - ], + }, +]; + +const fixtures: DBFixture = { + settings: _.cloneDeep(ixSettings), ixextractors: [ factory.ixExtractor('age_extractor', 'age', ['personTemplate', 'heroTemplate', 'template1']), factory.ixExtractor('title_extractor', 'title', ['heroTemplate']), @@ -122,6 +126,20 @@ const fixtures: DBFixture = { status: 'ready', error: '', }, + { + entityId: 'shared1', + fileId: factory.id('F1'), + entityTemplate: personTemplateId.toString(), + propertyName: 'age', + extractorId: factory.id('age_extractor'), + suggestedValue: '17', + segment: 'Robin is 17.', + language: 'en', + date: 5, + page: 2, + status: 'ready', + error: '', + }, { entityId: 'shared2', entityTemplate: personTemplateId.toString(), @@ -205,6 +223,20 @@ const fixtures: DBFixture = { status: 'ready', error: '', }, + { + entityId: 'shared3', + fileId: factory.id('F7'), + entityTemplate: personTemplateId.toString(), + propertyName: 'super_powers', + extractorId: factory.id('super_powers_extractor'), + suggestedValue: 'puts up with Bruce Wayne', + segment: 'he puts up with Bruce Wayne', + language: 'en', + date: 4000, + page: 3, + status: 'ready', + error: '', + }, { entityId: 'shared4', entityTemplate: personTemplateId.toString(), @@ -427,7 +459,7 @@ const fixtures: DBFixture = { sharedId: 'shared1', title: 'Robin', language: 'en', - metadata: { enemy: [{ value: 'Red Robin' }] }, + metadata: { enemy: [{ value: 'Red Robin' }], age: [{ value: 99 }] }, template: personTemplateId, }, { @@ -436,6 +468,7 @@ const fixtures: DBFixture = { title: 'Robin es', language: 'es', template: personTemplateId, + metadata: { age: [{ value: 99 }] }, }, { _id: testingDB.id(), @@ -462,11 +495,11 @@ const fixtures: DBFixture = { template: personTemplateId, }, { - _id: testingDB.id(), + _id: factory.id('Alfred-english-entity'), sharedId: 'shared3', title: 'Alfred', language: 'en', - metadata: { age: [{ value: 23 }] }, + metadata: { age: [{ value: 23 }], super_powers: [{ value: 'no super powers' }] }, template: personTemplateId, }, { @@ -513,7 +546,9 @@ const fixtures: DBFixture = { sharedId: 'shared7', title: 'The Riddler', language: 'en', - metadata: { first_encountered: [{ value: 1654732800 }] }, + metadata: { + first_encountered: [{ value: 1654732800 }], + }, template: heroTemplateId, }, { @@ -550,6 +585,15 @@ const fixtures: DBFixture = { }, ], files: [ + factory.file('F1', 'shared1', 'document', 'documentRedRobin.pdf', 'eng', '', [ + { + name: 'age', + selection: { + text: '99', + selectionRectangles: [{ top: 0, left: 0, width: 1, height: 2, page: '2' }], + }, + }, + ]), factory.file('F2', 'shared2', 'document', 'documentB.pdf', 'eng', '', [ { name: 'super_powers', @@ -611,6 +655,23 @@ const fixtures: DBFixture = { }, ]), factory.file('F6', 'shared8', 'document', 'documentRiddler.pdf', 'eng', '', []), + factory.file('F7', 'shared3', 'document', 'documentAlfred.pdf', 'eng', '', [ + { + name: 'super_powers', + selection: { + text: 'no super powers', + selectionRectangles: [ + { + top: 0, + left: 0, + width: 0, + height: 0, + page: '1', + }, + ], + }, + }, + ]), ], templates: [ { @@ -686,11 +747,392 @@ const fixtures: DBFixture = { ], }; +const stateFilterFixtures: DBFixture = { + settings: _.cloneDeep(ixSettings), + templates: [ + factory.template('template1', [ + factory.property('testprop', 'text'), + factory.property('unusedprop', 'text'), + ]), + ], + entities: [ + ...factory.entityInMultipleLanguages(['es', 'en'], 'labeled-match', 'template1', { + testprop: [{ value: 'test-labeled-match' }], + }), + ...factory.entityInMultipleLanguages(['es', 'en'], 'labeled-mismatch', 'template1', { + testprop: [{ value: 'test-labeled-mismatch' }], + }), + ...factory.entityInMultipleLanguages(['es', 'en'], 'unlabeled-no-suggestion', 'template1', { + testprop: [{ value: 'test-unlabeled-no-suggestion' }], + }), + ...factory.entityInMultipleLanguages(['es', 'en'], 'unlabeled-no-context', 'template1', { + testprop: [{ value: 'test-unlabeled-no-context' }], + }), + ...factory.entityInMultipleLanguages(['es', 'en'], 'unlabeled-obsolete', 'template1', { + testprop: [{ value: 'test-unlabeled-obsolete' }], + }), + ...factory.entityInMultipleLanguages(['es', 'en'], 'unlabeled-processing', 'template1', { + testprop: [{ value: 'test-unlabeled-processing' }], + }), + ...factory.entityInMultipleLanguages(['es', 'en'], 'unlabeled-error', 'template1', { + testprop: [{ value: 'test-unlabeled-error' }], + }), + ], + files: [ + factory.file('label-match-file-en', 'labeled-match', 'document', 'lmfen.pdf', 'en', undefined, [ + factory.fileExtractedMetadata('testprop', 'test-labeled-match'), + ]), + factory.file('label-match-file-es', 'labeled-match', 'document', 'lmfes.pdf', 'es', undefined, [ + factory.fileExtractedMetadata('testprop', 'test-labeled-match'), + ]), + factory.file( + 'label-mismatch-file-en', + 'labeled-mismatch', + 'document', + 'lmismfen.pdf', + 'en', + undefined, + [factory.fileExtractedMetadata('testprop', 'test-labeled-mismatch')] + ), + factory.file( + 'label-mismatch-file-es', + 'labeled-mismatch', + 'document', + 'lmismfes.pdf', + 'es', + undefined, + [factory.fileExtractedMetadata('testprop', 'test-labeled-mismatch')] + ), + factory.file( + 'unlabeled-no-suggestion-file-en', + 'unlabeled-no-suggestion', + 'document', + 'unslfen.pdf', + 'en', + undefined + ), + factory.file( + 'unlabeled-no-suggestion-file-es', + 'unlabeled-no-suggestion', + 'document', + 'unslfes.pdf', + 'es', + undefined + ), + factory.file( + 'unlabeled-no-context-file-en', + 'unlabeled-no-context', + 'document', + 'unlcen.pdf', + 'en', + undefined + ), + factory.file( + 'unlabeled-no-context-file-es', + 'unlabeled-no-context', + 'document', + 'unlces.pdf', + 'es', + undefined + ), + factory.file( + 'unlabeled-obsolete-file-en', + 'unlabeled-obsolete', + 'document', + 'unloen.pdf', + 'en', + undefined + ), + factory.file( + 'unlabeled-obsolete-file-es', + 'unlabeled-obsolete', + 'document', + 'unloes.pdf', + 'es', + undefined + ), + factory.file( + 'unlabeled-others-file-en', + 'unlabeled-others', + 'document', + 'unlothen.pdf', + 'en', + undefined + ), + factory.file( + 'unlabeled-others-file-es', + 'unlabeled-others', + 'document', + 'unlotes.pdf', + 'es', + undefined + ), + factory.file( + 'unlabeled-processing-file-en', + 'unlabeled-processing', + 'document', + 'unlpen.pdf', + 'en', + undefined + ), + factory.file( + 'unlabeled-processing-file-es', + 'unlabeled-processing', + 'document', + 'unlpes.pdf', + 'es', + undefined + ), + factory.file( + 'unlabeled-error-file-en', + 'unlabeled-error', + 'document', + 'unleen.pdf', + 'en', + undefined + ), + factory.file( + 'unlabeled-error-file-es', + 'unlabeled-error', + 'document', + 'unlees.pdf', + 'es', + undefined + ), + ], + ixmodels: [factory.ixModel('test_model', 'test_extractor', 1000)], + ixextractors: [ + factory.ixExtractor('test_extractor', 'testprop', ['template1']), + factory.ixExtractor('unused_extractor', 'unused_prop', ['template1']), + ], + ixsuggestions: [ + factory.ixSuggestion( + 'label-match-suggestion-en', + 'test_extractor', + 'labeled-match', + 'template1', + 'label-match-file-en', + 'testprop', + { + status: 'ready', + date: 1001, + language: 'en', + suggestedValue: 'test-labeled-match', + } + ), + factory.ixSuggestion( + 'label-match-suggestion-es', + 'test_extractor', + 'labeled-match', + 'template1', + 'label-match-file-es', + 'testprop', + { + status: 'ready', + date: 1001, + language: 'es', + suggestedValue: 'test-labeled-match', + } + ), + factory.ixSuggestion( + 'label-mismatch-suggestion-en', + 'test_extractor', + 'labeled-mismatch', + 'template1', + 'label-mismatch-file-en', + 'testprop', + { + status: 'ready', + date: 1001, + language: 'en', + suggestedValue: 'test-labeled-mismatch-mismatch', + } + ), + factory.ixSuggestion( + 'label-mismatch-suggestion-es', + 'test_extractor', + 'labeled-mismatch', + 'template1', + 'label-mismatch-file-es', + 'testprop', + { + status: 'ready', + date: 1001, + language: 'es', + suggestedValue: 'test-labeled-mismatch-mismatch', + } + ), + factory.ixSuggestion( + 'unlabeled-no-suggestion-suggestion-en', + 'test_extractor', + 'unlabeled-no-suggestion', + 'template1', + 'unlabeled-no-suggestion-file-en', + 'testprop', + { + status: 'ready', + date: 1001, + language: 'en', + suggestedValue: '', + } + ), + factory.ixSuggestion( + 'unlabeled-no-suggestion-suggestion-es', + 'test_extractor', + 'unlabeled-no-suggestion', + 'template1', + 'unlabeled-no-suggestion-file-es', + 'testprop', + { + status: 'ready', + date: 1001, + language: 'es', + suggestedValue: '', + } + ), + factory.ixSuggestion( + 'unlabeled-no-context-suggestion-en', + 'test_extractor', + 'unlabeled-no-context', + 'template1', + 'unlabeled-no-context-file-en', + 'testprop', + { + status: 'ready', + date: 1001, + language: 'en', + suggestedValue: 'test-unlabeled-no-context', + } + ), + factory.ixSuggestion( + 'unlabeled-no-context-suggestion-es', + 'test_extractor', + 'unlabeled-no-context', + 'template1', + 'unlabeled-no-context-file-es', + 'testprop', + { + status: 'ready', + date: 1001, + language: 'es', + suggestedValue: 'test-unlabeled-no-context', + } + ), + factory.ixSuggestion( + 'unlabeled-obsolete-suggestion-en', + 'test_extractor', + 'unlabeled-obsolete', + 'template1', + 'unlabeled-obsolete-file-en', + 'testprop', + { + status: 'ready', + date: 999, + language: 'en', + suggestedValue: 'test-unlabeled-obsolete', + segment: 'test-unlabeled-obsolete', + } + ), + factory.ixSuggestion( + 'unlabeled-obsolete-suggestion-es', + 'test_extractor', + 'unlabeled-obsolete', + 'template1', + 'unlabeled-obsolete-file-es', + 'testprop', + { + status: 'ready', + date: 999, + language: 'es', + suggestedValue: 'test-unlabeled-obsolete', + segment: 'test-unlabeled-obsolete', + } + ), + factory.ixSuggestion( + 'unlabeled-processing-suggestion-en', + 'test_extractor', + 'unlabeled-processing', + 'template1', + 'unlabeled-processing-file-en', + 'testprop', + { + status: 'processing', + date: 1001, + language: 'en', + suggestedValue: 'test-unlabeled-processing', + segment: 'test-unlabeled-processing', + } + ), + factory.ixSuggestion( + 'unlabeled-processing-suggestion-es', + 'test_extractor', + 'unlabeled-processing', + 'template1', + 'unlabeled-processing-file-es', + 'testprop', + { + status: 'processing', + date: 1001, + language: 'es', + suggestedValue: 'test-unlabeled-processing', + segment: 'test-unlabeled-processing', + } + ), + factory.ixSuggestion( + 'unlabeled-error-suggestion-en', + 'test_extractor', + 'unlabeled-error', + 'template1', + 'unlabeled-error-file-en', + 'testprop', + { + status: 'failed', + date: 1001, + language: 'en', + suggestedValue: 'test-unlabeled-error', + segment: 'test-unlabeled-error', + error: 'some error happened', + } + ), + factory.ixSuggestion( + 'unlabeled-error-suggestion-es', + 'test_extractor', + 'unlabeled-error', + 'template1', + 'unlabeled-error-file-es', + 'testprop', + { + status: 'failed', + date: 1001, + language: 'es', + suggestedValue: 'test-unlabeled-error', + segment: 'test-unlabeled-error', + error: 'some error happened', + } + ), + factory.ixSuggestion( + 'unusedsuggestion', + 'unused_extractor', + 'unused', + 'template1', + 'unused-file', + 'unusedprop', + { + status: 'ready', + date: 1001, + language: 'en', + suggestedValue: 'test-unused', + } + ), + ], +}; + export { factory, file2Id, file3Id, fixtures, + stateFilterFixtures, shared2esId, shared2enId, shared6enId, diff --git a/app/api/suggestions/specs/routes.spec.ts b/app/api/suggestions/specs/routes.spec.ts index ab44bed6ae..79d84052ff 100644 --- a/app/api/suggestions/specs/routes.spec.ts +++ b/app/api/suggestions/specs/routes.spec.ts @@ -3,24 +3,20 @@ import request from 'supertest'; import { Application, NextFunction, Request, Response } from 'express'; import entities from 'api/entities'; -import { WithId } from 'api/odm'; import { search } from 'api/search'; import { factory, fixtures, - heroTemplateId, - personTemplateId, shared2enId, shared2esId, shared6enId, + stateFilterFixtures, suggestionSharedId6Enemy, suggestionSharedId6Title, } from 'api/suggestions/specs/fixtures'; import { suggestionsRoutes } from 'api/suggestions/routes'; import { testingEnvironment } from 'api/utils/testingEnvironment'; import { setUpApp } from 'api/utils/testingRoutes'; -import { EntitySchema } from 'shared/types/entityType'; -import { SuggestionState } from 'shared/types/suggestionSchema'; import { Suggestions } from '../suggestions'; jest.mock( @@ -39,31 +35,31 @@ jest.mock('api/services/informationextraction/InformationExtraction', () => ({ }, })); -const sortAggregateById = (array: { _id: string; count: number }[]) => - array.sort((a, b) => a._id.localeCompare(b._id)); +let user: { username: string; role: string } | undefined; +const getUser = () => user; -describe('suggestions routes', () => { - let user: { username: string; role: string } | undefined; - const getUser = () => user; +beforeEach(async () => { + user = { username: 'user 1', role: 'admin' }; + jest.spyOn(search, 'indexEntities').mockImplementation(async () => Promise.resolve()); +}); + +const app: Application = setUpApp( + suggestionsRoutes, + (req: Request, _res: Response, next: NextFunction) => { + (req as any).user = getUser(); + next(); + } +); +afterAll(async () => { + await testingEnvironment.tearDown(); +}); + +describe('suggestions routes', () => { beforeAll(async () => { await testingEnvironment.setUp(fixtures); await Suggestions.updateStates({}); }); - beforeEach(async () => { - user = { username: 'user 1', role: 'admin' }; - jest.spyOn(search, 'indexEntities').mockImplementation(async () => Promise.resolve()); - }); - - const app: Application = setUpApp( - suggestionsRoutes, - (req: Request, _res: Response, next: NextFunction) => { - (req as any).user = getUser(); - next(); - } - ); - - afterAll(async () => testingEnvironment.tearDown()); describe('GET /api/suggestions', () => { it('should return the suggestions filtered by the request language and the property name', async () => { @@ -76,6 +72,26 @@ describe('suggestions routes', () => { }) .expect(200); expect(response.body.suggestions).toMatchObject([ + { + entityId: shared2enId.toString(), + sharedId: 'shared2', + entityTitle: 'Batman en', + propertyName: 'super_powers', + suggestedValue: 'scientific knowledge', + segment: 'he relies on his own scientific knowledge', + state: { + labeled: true, + withValue: true, + withSuggestion: true, + match: true, + hasContext: true, + obsolete: false, + processing: false, + error: false, + }, + language: 'en', + page: 5, + }, { entityId: shared2esId.toString(), sharedId: 'shared2', @@ -83,30 +99,41 @@ describe('suggestions routes', () => { propertyName: 'super_powers', suggestedValue: 'scientific knowledge es', segment: 'el confía en su propio conocimiento científico', - state: SuggestionState.labelMismatch, + state: { + labeled: true, + withValue: true, + withSuggestion: true, + match: false, + hasContext: true, + obsolete: false, + processing: false, + error: false, + }, language: 'es', page: 5, }, { - entityId: shared2enId.toString(), - sharedId: 'shared2', - entityTitle: 'Batman en', + entityId: factory.id('Alfred-english-entity').toString(), + sharedId: 'shared3', + entityTitle: 'Alfred', propertyName: 'super_powers', - suggestedValue: 'scientific knowledge', - segment: 'he relies on his own scientific knowledge', - state: SuggestionState.labelMatch, + suggestedValue: 'puts up with Bruce Wayne', + segment: 'he puts up with Bruce Wayne', + state: { + labeled: true, + withValue: true, + withSuggestion: true, + match: false, + hasContext: true, + obsolete: false, + processing: false, + error: false, + }, language: 'en', - page: 5, + page: 3, }, ]); expect(response.body.totalPages).toBe(1); - expect(response.body.aggregations).toMatchObject({ - template: [{ _id: personTemplateId.toString(), count: 2 }], - state: [ - { _id: 'Match / Label', count: 1 }, - { _id: 'Mismatch / Label', count: 1 }, - ], - }); }); it('should include failed suggestions but not processing ones', async () => { @@ -118,30 +145,23 @@ describe('suggestions routes', () => { }, }) .expect(200); - expect(response.body.suggestions).toMatchObject( - expect.arrayContaining([ - expect.objectContaining({ - entityTitle: 'Joker', - propertyName: 'age', - segment: 'Joker age is 45', - sharedId: 'shared4', - state: 'Error', - suggestedValue: null, - }), - ]) + const joker = response.body.suggestions.find( + (suggestion: any) => suggestion.entityTitle === 'Joker' ); - expect(response.body.suggestions).not.toMatchObject( - expect.arrayContaining([ - expect.objectContaining({ - entityTitle: 'Alfred', - propertyName: 'age', - segment: 'Alfred 67 years old processing', - currentValue: 23, - sharedId: 'shared3', - state: 'Mismatch / Value', - }), - ]) + expect(joker).toMatchObject({ + entityTitle: 'Joker', + propertyName: 'age', + segment: 'Joker age is 45', + sharedId: 'shared4', + state: { + error: true, + }, + suggestedValue: null, + }); + const alfred = response.body.suggestions.find( + (suggestion: any) => suggestion.segment === 'Alfred 67 years old processing' ); + expect(alfred).toBeUndefined(); }); describe('pagination', () => { @@ -155,10 +175,12 @@ describe('suggestions routes', () => { page: { number: 2, size: 2 }, }) .expect(200); + expect(response.body.suggestions).toMatchObject([ { entityTitle: 'Alfred' }, { entityTitle: 'Robin' }, ]); + expect(response.body.totalPages).toBe(3); }); @@ -186,146 +208,104 @@ describe('suggestions routes', () => { .query({ filter: { extractorId: factory.id('enemy_extractor').toString(), - states: [SuggestionState.empty], + customFilter: { + labeled: { + match: false, + mismatch: false, + }, + nonLabeled: { + noSuggestion: true, + noContext: false, + obsolete: false, + others: false, + }, + }, }, }) .expect(200); expect(response.body.suggestions).toEqual([ expect.objectContaining({ entityTitle: 'Catwoman', - state: SuggestionState.empty, + state: { + labeled: false, + withValue: false, + withSuggestion: false, + match: false, + hasContext: true, + obsolete: false, + processing: false, + error: false, + }, suggestedValue: '', currentValue: '', }), ]); }); - - it('should filter by entity template', async () => { - const response = await request(app) - .get('/api/suggestions/') - .query({ - filter: { - extractorId: factory.id('title_extractor').toString(), - entityTemplates: [personTemplateId.toString()], - }, - }) - .expect(200); - expect(response.body.suggestions).toMatchObject([ - { - propertyName: 'title', - entityTemplateId: personTemplateId.toString(), - sharedId: 'shared4', - language: 'en', - }, - { - propertyName: 'title', - entityTemplateId: personTemplateId.toString(), - sharedId: 'shared3', - language: 'en', - }, - { - propertyName: 'title', - entityTemplateId: personTemplateId.toString(), - sharedId: 'shared1', - language: 'en', - }, - { - propertyName: 'title', - entityTemplateId: personTemplateId.toString(), - sharedId: 'shared1', - language: 'es', - }, - ]); - }); }); - describe('aggregations', () => { - it('should return aggregations', async () => { + describe('sorting', () => { + it('should sort by entity title', async () => { const response = await request(app) - .get('/api/suggestions/') + .get('/api/suggestions') .query({ filter: { - extractorId: factory.id('title_extractor').toString(), + extractorId: factory.id('super_powers_extractor').toString(), }, + sort: { property: 'entityTitle', order: 'desc' }, }) .expect(200); - expect(response.body.aggregations).toMatchObject({ - template: sortAggregateById([ - { _id: heroTemplateId.toString(), count: 2 }, - { _id: personTemplateId.toString(), count: 4 }, - ]), - state: [ - { _id: SuggestionState.valueMatch, count: 2 }, - { _id: SuggestionState.valueMismatch, count: 4 }, - ], + + expect(response.body.suggestions[0]).toMatchObject({ + sharedId: 'shared2', + entityTitle: 'Batman es', + language: 'es', }); - }); - it('should return aggregations for a specific template', async () => { - const response = await request(app) - .get('/api/suggestions/') - .query({ - filter: { - extractorId: factory.id('title_extractor').toString(), - entityTemplates: [heroTemplateId.toString()], - }, - }) - .expect(200); - expect(response.body.aggregations).toMatchObject({ - template: sortAggregateById([ - { _id: heroTemplateId.toString(), count: 2 }, - { _id: personTemplateId.toString(), count: 4 }, - ]), - state: [ - { _id: SuggestionState.valueMatch, count: 1 }, - { _id: SuggestionState.valueMismatch, count: 1 }, - ], + expect(response.body.suggestions[1]).toMatchObject({ + sharedId: 'shared2', + entityTitle: 'Batman en', + language: 'en', }); - }); - it('should return aggregations for a specific state', async () => { - const response = await request(app) - .get('/api/suggestions/') - .query({ - filter: { - extractorId: factory.id('title_extractor').toString(), - states: [SuggestionState.valueMatch], - }, - }) - .expect(200); - expect(response.body.aggregations).toMatchObject({ - template: sortAggregateById([ - { _id: heroTemplateId.toString(), count: 1 }, - { _id: personTemplateId.toString(), count: 1 }, - ]), - state: [ - { _id: SuggestionState.valueMatch, count: 2 }, - { _id: SuggestionState.valueMismatch, count: 4 }, - ], + expect(response.body.suggestions[2]).toMatchObject({ + sharedId: 'shared3', + entityTitle: 'Alfred', + language: 'en', }); + + expect(response.body.totalPages).toBe(1); }); - it('should return aggregations for a specific template and state', async () => { + it('should sort by current value', async () => { const response = await request(app) - .get('/api/suggestions/') + .get('/api/suggestions') .query({ filter: { - extractorId: factory.id('title_extractor').toString(), - entityTemplates: [heroTemplateId.toString()], - states: [SuggestionState.valueMatch], + extractorId: factory.id('super_powers_extractor').toString(), }, + sort: { property: 'currentValue' }, }) .expect(200); - expect(response.body.aggregations).toMatchObject({ - template: sortAggregateById([ - { _id: heroTemplateId.toString(), count: 1 }, - { _id: personTemplateId.toString(), count: 1 }, - ]), - state: [ - { _id: SuggestionState.valueMatch, count: 1 }, - { _id: SuggestionState.valueMismatch, count: 1 }, - ], + + expect(response.body.suggestions[0]).toMatchObject({ + currentValue: 'conocimiento científico', + entityTitle: 'Batman es', + sharedId: 'shared2', + }); + + expect(response.body.suggestions[1]).toMatchObject({ + currentValue: 'no super powers', + entityTitle: 'Alfred', + sharedId: 'shared3', + }); + + expect(response.body.suggestions[2]).toMatchObject({ + currentValue: 'scientific knowledge', + entityTitle: 'Batman en', + sharedId: 'shared2', }); + + expect(response.body.totalPages).toBe(1); }); }); @@ -391,12 +371,13 @@ describe('suggestions routes', () => { await request(app) .post('/api/suggestions/accept') .send({ - suggestion: { - _id: suggestionSharedId6Title, - sharedId: 'shared6', - entityId: shared6enId, - }, - allLanguages: false, + suggestions: [ + { + _id: suggestionSharedId6Title, + sharedId: 'shared6', + entityId: shared6enId, + }, + ], }) .expect(200); @@ -417,37 +398,7 @@ describe('suggestions routes', () => { '+fullText' ); }); - it('should update the suggestion for all the languages', async () => { - await request(app) - .post('/api/suggestions/accept') - .send({ - allLanguages: true, - suggestion: { - _id: suggestionSharedId6Enemy, - sharedId: 'shared6', - entityId: shared6enId, - }, - }) - .expect(200); - const actualEntities = await entities.get({ sharedId: 'shared6' }); - expect(actualEntities).toMatchObject([ - { - metadata: { enemy: [{ value: 'Batman' }], age: [{ value: 40 }] }, - }, - { - metadata: { enemy: [{ value: 'Batman' }], age: [{ value: 40 }] }, - }, - { - metadata: { enemy: [{ value: 'Batman' }], age: [{ value: 40 }] }, - }, - ]); - const entityIds = actualEntities.map((e: WithId) => e._id); - expect(search.indexEntities).toHaveBeenCalledWith( - { _id: { $in: expect.arrayContaining(entityIds) } }, - '+fullText' - ); - }); it('should reject with unauthorized when user has not admin role', async () => { user = { username: 'user 1', role: 'editor' }; const response = await request(app) @@ -465,3 +416,59 @@ describe('suggestions routes', () => { }); }); }); + +describe('aggregation routes', () => { + describe('GET /api/suggestions/aggregation', () => { + beforeAll(async () => { + await testingEnvironment.setUp(stateFilterFixtures); + await Suggestions.updateStates({}); + }); + + describe('validation', () => { + it('should return a validation error if params are not valid', async () => { + const invalidQuery = { additionParam: true }; + const response = await request(app).get('/api/suggestions/aggregation').query(invalidQuery); + expect(response.status).toBe(400); + + const emptyQuery = {}; + const response2 = await request(app).get('/api/suggestions/aggregation').query(emptyQuery); + expect(response2.status).toBe(400); + }); + }); + + describe('authentication', () => { + it('should reject with unauthorized when the user does not have the admin role', async () => { + user = { username: 'user 1', role: 'editor' }; + const response = await request(app) + .get('/api/suggestions/aggregation') + .query({}) + .expect(401); + expect(response.unauthorized).toBe(true); + }); + }); + + it('should return the aggregation of suggestions', async () => { + const response = await request(app) + .get('/api/suggestions/aggregation') + .query({ + extractorId: factory.id('test_extractor').toString(), + }) + .expect(200); + expect(response.body).toEqual({ + total: 12, + labeled: { + _count: 4, + match: 2, + mismatch: 2, + }, + nonLabeled: { + _count: 8, + noSuggestion: 2, + noContext: 4, + obsolete: 2, + others: 2, + }, + }); + }); + }); +}); diff --git a/app/api/suggestions/specs/stats.spec.ts b/app/api/suggestions/specs/stats.spec.ts deleted file mode 100644 index 4f6b11fdcf..0000000000 --- a/app/api/suggestions/specs/stats.spec.ts +++ /dev/null @@ -1,194 +0,0 @@ -import { getFixturesFactory } from 'api/utils/fixturesFactory'; -import { testingEnvironment } from 'api/utils/testingEnvironment'; -import { DBFixture, testingDB } from 'api/utils/testing_db'; -import { SuggestionState } from 'shared/types/suggestionSchema'; -import { getStats } from '../stats'; - -const fixturesFactory = getFixturesFactory(); - -const suggestionBase = { - entityId: '', - propertyName: 'age', - entityTemplate: fixturesFactory.id('template').toString(), - extractorId: fixturesFactory.id('age_extractor'), - suggestedValue: '', - segment: '', - language: '', -}; - -const fixtures: DBFixture = { - ixsuggestions: [ - { - _id: testingDB.id(), - ...suggestionBase, - state: SuggestionState.labelEmpty, - }, - { - _id: testingDB.id(), - ...suggestionBase, - state: SuggestionState.labelMatch, - }, - { - _id: testingDB.id(), - ...suggestionBase, - state: SuggestionState.labelMismatch, - }, - { - _id: testingDB.id(), - ...suggestionBase, - state: SuggestionState.valueEmpty, - }, - { - _id: testingDB.id(), - ...suggestionBase, - state: SuggestionState.valueMatch, - }, - { - _id: testingDB.id(), - ...suggestionBase, - state: SuggestionState.valueMismatch, - }, - { - _id: testingDB.id(), - ...suggestionBase, - state: SuggestionState.obsolete, - }, - { - _id: testingDB.id(), - ...suggestionBase, - state: SuggestionState.empty, - }, - { - _id: testingDB.id(), - ...suggestionBase, - state: SuggestionState.emptyMismatch, - }, - ], -}; - -beforeAll(async () => { - await testingEnvironment.setUp(fixtures); -}); - -afterAll(async () => { - await testingEnvironment.tearDown(); -}); - -describe('when the property exists', () => { - it('should return the training counts', async () => { - expect(await getStats(fixturesFactory.id('age_extractor').toString())).toMatchObject({ - counts: { - labeled: 3, - nonLabeledMatching: 1, - nonLabeledNotMatching: 1, - emptyOrObsolete: 4, - all: fixtures.ixsuggestions!.length, - }, - }); - }); - - it.each([ - { - state: SuggestionState.labelMatch, - action: 'count as correct', - result: 1, - }, - { - state: SuggestionState.labelMismatch, - action: 'count as incorrect', - result: 0, - }, - { - state: SuggestionState.valueMatch, - action: 'count as correct', - result: 1, - }, - { - state: SuggestionState.valueMismatch, - action: 'count as incorrect', - result: 0, - }, - { - state: SuggestionState.empty, - action: 'not count', - result: 0, - }, - { - state: SuggestionState.obsolete, - action: 'not count', - result: 0, - }, - { - state: SuggestionState.labelEmpty, - action: 'count as incorrect', - result: 0, - }, - { - state: SuggestionState.valueEmpty, - action: 'count as incorrect', - result: 0, - }, - { - state: SuggestionState.error, - action: 'not count', - result: 0, - }, - { - state: SuggestionState.processing, - action: 'not count', - result: 0, - }, - { - state: SuggestionState.emptyMismatch, - action: 'not count', - result: 0, - }, - ])('$state state should $action in accuracy', async ({ state, result }) => { - const input = { - _id: testingDB.id(), - ...suggestionBase, - state, - }; - await testingEnvironment.setUp({ ixsuggestions: [input] }); - const stats = await getStats(fixturesFactory.id('age_extractor').toString()); - expect(stats.accuracy).toEqual(result); - }); - - it.each([ - { - states: [SuggestionState.labelMatch, SuggestionState.labelMismatch, SuggestionState.empty], - }, - { - states: [ - SuggestionState.valueMatch, - SuggestionState.valueMismatch, - SuggestionState.processing, - ], - }, - { - states: [ - SuggestionState.labelMatch, - SuggestionState.labelEmpty, - SuggestionState.emptyMismatch, - ], - }, - { - states: [SuggestionState.valueMatch, SuggestionState.valueEmpty, SuggestionState.error], - }, - ])('should return accuracy correctly', async ({ states }) => { - const inputs = states.map(state => ({ - _id: testingDB.id(), - ...suggestionBase, - state, - })); - await testingEnvironment.setUp({ ixsuggestions: inputs }); - const stats = await getStats(fixturesFactory.id('age_extractor').toString()); - expect(stats.accuracy).toEqual(0.5); - }); -}); - -describe('when the property does not exists', () => { - it('should not fail', async () => { - await getStats(fixturesFactory.id('non_existing_extractor').toString()); - }); -}); diff --git a/app/api/suggestions/specs/suggestions.spec.ts b/app/api/suggestions/specs/suggestions.spec.ts index 063090b6ea..d340696ee7 100644 --- a/app/api/suggestions/specs/suggestions.spec.ts +++ b/app/api/suggestions/specs/suggestions.spec.ts @@ -1,7 +1,11 @@ import db from 'api/utils/testing_db'; -import { EntitySuggestionType, IXSuggestionType } from 'shared/types/suggestionType'; -import { SuggestionState } from 'shared/types/suggestionSchema'; +import { + EntitySuggestionType, + IXSuggestionStateType, + IXSuggestionType, + IXSuggestionsFilter, +} from 'shared/types/suggestionType'; import { Suggestions } from '../suggestions'; import { factory, @@ -15,28 +19,48 @@ import { shared2AgeSuggestionId, } from './fixtures'; -const getSuggestions = async (extractorId: string, size = 5) => - Suggestions.get({ extractorId }, { page: { size, number: 1 } }); +const getSuggestions = async (filter: IXSuggestionsFilter, size = 5) => + Suggestions.get(filter, { page: { size, number: 1 } }); const findOneSuggestion = async (query: any): Promise => db.mongodb ?.collection('ixsuggestions') .findOne({ ...query }) as unknown as Promise; -const stateUpdateCases = [ +const stateUpdateCases: { + state: Partial; + reason: string; + suggestionQuery: any; +}[] = [ { - state: SuggestionState.obsolete, - reason: 'the suggestion is older than the model', + state: { obsolete: true }, + reason: 'obsolete, if the suggestion is older than the model', suggestionQuery: { entityId: 'shared5', propertyName: 'age' }, }, { - state: SuggestionState.valueEmpty, - reason: 'entity value exists, file label is empty, suggestion is empty', + state: { + withValue: true, + withSuggestion: false, + labeled: false, + match: false, + obsolete: false, + processing: false, + error: false, + }, + reason: 'if entity value exists, file label is empty, suggestion is empty', suggestionQuery: { entityId: 'shared3', propertyName: 'age' }, }, { - state: SuggestionState.labelMatch, - reason: 'file label exists, suggestion and entity value exist and match', + state: { + labeled: true, + withValue: true, + withSuggestion: true, + match: true, + obsolete: false, + processing: false, + error: false, + }, + reason: 'if file label exists, suggestion and entity value exist and match', suggestionQuery: { entityId: 'shared2', propertyName: 'super_powers', @@ -45,8 +69,17 @@ const stateUpdateCases = [ }, }, { - state: SuggestionState.labelMatch, - reason: 'property is a date, file label exists, suggestion and entity value exist and match', + state: { + labeled: true, + withValue: true, + withSuggestion: true, + match: true, + obsolete: false, + processing: false, + error: false, + }, + reason: + 'when property is a date, and if file label exists, suggestion and entity value exist and match', suggestionQuery: { entityId: 'shared7', propertyName: 'first_encountered', @@ -54,8 +87,16 @@ const stateUpdateCases = [ }, }, { - state: SuggestionState.empty, - reason: 'entity value, file label, suggestion are all empty', + state: { + labeled: false, + withValue: false, + withSuggestion: false, + match: false, + obsolete: false, + processing: false, + error: false, + }, + reason: 'if entity value, file label, suggestion are all empty', suggestionQuery: { entityId: 'shared8', propertyName: 'enemy', @@ -63,8 +104,16 @@ const stateUpdateCases = [ }, }, { - state: SuggestionState.labelEmpty, - reason: 'entity value and file label exists, suggestion is empty', + state: { + labeled: true, + withValue: true, + withSuggestion: false, + match: false, + obsolete: false, + processing: false, + error: false, + }, + reason: 'if entity value and file label exists, suggestion is empty', suggestionQuery: { entityId: 'shared6', propertyName: 'enemy', @@ -73,8 +122,17 @@ const stateUpdateCases = [ }, }, { - state: SuggestionState.labelEmpty, - reason: 'property is a date, entity value and file label exists, suggestion is empty', + state: { + labeled: true, + withValue: true, + withSuggestion: false, + match: false, + obsolete: false, + processing: false, + error: false, + }, + reason: + 'when property is a date, and if entity value and file label exists, suggestion is empty', suggestionQuery: { entityId: 'shared7', propertyName: 'first_encountered', @@ -82,17 +140,33 @@ const stateUpdateCases = [ }, }, { - state: SuggestionState.labelMismatch, - reason: 'file label exists, suggestion and entity value exist but do not match', + state: { + labeled: true, + withValue: true, + withSuggestion: true, + match: false, + obsolete: false, + processing: false, + error: false, + }, + reason: 'if file label exists, suggestion and entity value exist but do not match', suggestionQuery: { propertyName: 'super_powers', language: 'es', }, }, { - state: SuggestionState.labelMismatch, + state: { + labeled: true, + withValue: true, + withSuggestion: true, + match: false, + obsolete: false, + processing: false, + error: false, + }, reason: - 'property is a date, file label exists, suggestion and entity value exist but do not match', + 'when property is a date, if file label exists, suggestion and entity value exist but do not match', suggestionQuery: { entityId: 'shared7', propertyName: 'first_encountered', @@ -100,17 +174,33 @@ const stateUpdateCases = [ }, }, { - state: SuggestionState.valueMatch, - reason: 'file label is empty, but suggestion and entity value exist and match', + state: { + labeled: false, + withValue: true, + withSuggestion: true, + match: true, + obsolete: false, + processing: false, + error: false, + }, + reason: 'if file label is empty, but suggestion and entity value exist and match', suggestionQuery: { entityId: 'shared1', propertyName: 'enemy', }, }, { - state: SuggestionState.valueMatch, + state: { + labeled: false, + withValue: true, + withSuggestion: true, + match: true, + obsolete: false, + processing: false, + error: false, + }, reason: - 'property is a date, file label is empty, but suggestion and entity value exist and match', + 'when property is a date, and if file label is empty, but suggestion and entity value exist and match', suggestionQuery: { entityId: 'shared8', propertyName: 'first_encountered', @@ -118,8 +208,16 @@ const stateUpdateCases = [ }, }, { - state: SuggestionState.valueMismatch, - reason: 'file label is empty, suggestion and entity value exist but do not match', + state: { + labeled: false, + withValue: true, + withSuggestion: true, + match: false, + obsolete: false, + processing: false, + error: false, + }, + reason: 'if file label is empty, suggestion and entity value exist but do not match', suggestionQuery: { entityId: 'shared6', propertyName: 'enemy', @@ -201,7 +299,7 @@ describe('suggestions', () => { }, { page: { size: 50, number: 1 } } ); - expect(suggestions.length).toBe(2); + expect(suggestions.length).toBe(3); }); it('should return suggestion and extra entity information', async () => { @@ -210,6 +308,31 @@ describe('suggestions', () => { { page: { size: 50, number: 1 } } ); expect(suggestions).toMatchObject([ + { + fileId: file2Id, + propertyName: 'super_powers', + extractorId: factory.id('super_powers_extractor'), + suggestedValue: 'scientific knowledge', + segment: 'he relies on his own scientific knowledge', + language: 'en', + date: 4, + page: 5, + currentValue: 'scientific knowledge', + labeledValue: 'scientific knowledge', + state: { + labeled: true, + withValue: true, + withSuggestion: true, + match: true, + hasContext: true, + obsolete: false, + processing: false, + error: false, + }, + entityId: shared2enId, + sharedId: 'shared2', + entityTitle: 'Batman en', + }, { fileId: file3Id, propertyName: 'super_powers', @@ -221,41 +344,70 @@ describe('suggestions', () => { page: 5, currentValue: 'conocimiento científico', labeledValue: 'conocimiento científico', - state: 'Mismatch / Label', + state: { + labeled: true, + withValue: true, + withSuggestion: true, + match: false, + hasContext: true, + obsolete: false, + processing: false, + error: false, + }, entityId: shared2esId, sharedId: 'shared2', entityTitle: 'Batman es', }, { - fileId: file2Id, + fileId: factory.id('F7'), propertyName: 'super_powers', extractorId: factory.id('super_powers_extractor'), - suggestedValue: 'scientific knowledge', - segment: 'he relies on his own scientific knowledge', + segment: 'he puts up with Bruce Wayne', + currentValue: 'no super powers', + date: 4000, + page: 3, + entityId: factory.id('Alfred-english-entity'), + entityTemplateId: personTemplateId, + entityTitle: 'Alfred', + error: '', + labeledValue: 'no super powers', language: 'en', - date: 4, - page: 5, - currentValue: 'scientific knowledge', - labeledValue: 'scientific knowledge', - state: 'Match / Label', - entityId: shared2enId, - sharedId: 'shared2', - entityTitle: 'Batman en', + sharedId: 'shared3', + state: { + error: false, + hasContext: true, + labeled: true, + match: false, + obsolete: false, + processing: false, + withSuggestion: true, + withValue: true, + }, + suggestedValue: 'puts up with Bruce Wayne', }, ]); }); it('should return match status', async () => { - const { suggestions: superPowersSuggestions } = await getSuggestions( - factory.id('super_powers_extractor').toString() - ); + const { suggestions: superPowersSuggestions } = await getSuggestions({ + extractorId: factory.id('super_powers_extractor').toString(), + }); expect( superPowersSuggestions.find((s: EntitySuggestionType) => s.language === 'en').state - ).toBe(SuggestionState.labelMatch); + ).toEqual({ + labeled: true, + withValue: true, + withSuggestion: true, + match: true, + hasContext: true, + obsolete: false, + processing: false, + error: false, + }); const { suggestions: enemySuggestions } = await getSuggestions( - factory.id('enemy_extractor').toString(), + { extractorId: factory.id('enemy_extractor').toString() }, 6 ); @@ -263,61 +415,126 @@ describe('suggestions', () => { enemySuggestions.filter( (s: EntitySuggestionType) => s.sharedId === 'shared6' && s.language === 'en' )[1].state - ).toBe(SuggestionState.labelEmpty); + ).toEqual({ + labeled: false, + withValue: true, + withSuggestion: true, + match: false, + hasContext: true, + obsolete: false, + processing: false, + error: false, + }); expect( enemySuggestions.find((s: EntitySuggestionType) => s.sharedId === 'shared1').state - ).toBe(SuggestionState.valueMatch); + ).toEqual({ + labeled: false, + withValue: true, + withSuggestion: true, + match: true, + hasContext: true, + obsolete: false, + processing: false, + error: false, + }); expect( enemySuggestions.find( (s: EntitySuggestionType) => s.sharedId === 'shared8' && s.language === 'en' ).state - ).toBe(SuggestionState.empty); + ).toEqual({ + labeled: false, + withValue: false, + withSuggestion: false, + match: false, + hasContext: true, + obsolete: false, + processing: false, + error: false, + }); - const { suggestions: ageSuggestions } = await getSuggestions( - factory.id('age_extractor').toString() - ); + const { suggestions: ageSuggestions } = await getSuggestions({ + extractorId: factory.id('age_extractor').toString(), + }); - expect(ageSuggestions.length).toBe(4); - expect(ageSuggestions.find((s: EntitySuggestionType) => s.sharedId === 'shared5').state).toBe( - SuggestionState.obsolete - ); + expect(ageSuggestions.length).toBe(5); + expect( + ageSuggestions.find((s: EntitySuggestionType) => s.sharedId === 'shared5').state.obsolete + ).toEqual(true); - expect(ageSuggestions.find((s: EntitySuggestionType) => s.sharedId === 'shared3').state).toBe( - SuggestionState.valueEmpty - ); + expect( + ageSuggestions.find((s: EntitySuggestionType) => s.sharedId === 'shared3').state + ).toEqual({ + labeled: false, + withValue: true, + withSuggestion: false, + match: false, + hasContext: true, + obsolete: false, + processing: false, + error: false, + }); }); it('should return mismatch status', async () => { - const { suggestions: superPowersSuggestions } = await getSuggestions( - factory.id('super_powers_extractor').toString() - ); + const { suggestions: superPowersSuggestions } = await getSuggestions({ + extractorId: factory.id('super_powers_extractor').toString(), + }); expect( superPowersSuggestions.find((s: EntitySuggestionType) => s.language === 'es').state - ).toBe(SuggestionState.labelMismatch); + ).toEqual({ + labeled: true, + withValue: true, + withSuggestion: true, + match: false, + hasContext: true, + obsolete: false, + processing: false, + error: false, + }); - const { suggestions: enemySuggestions } = await getSuggestions( - factory.id('enemy_extractor').toString() - ); + const { suggestions: enemySuggestions } = await getSuggestions({ + extractorId: factory.id('enemy_extractor').toString(), + }); expect( enemySuggestions.find( (s: EntitySuggestionType) => s.sharedId === 'shared6' && s.language === 'en' ).state - ).toBe(SuggestionState.valueMismatch); + ).toEqual({ + labeled: true, + withValue: true, + withSuggestion: false, + match: false, + hasContext: true, + obsolete: false, + processing: false, + error: false, + }); expect( enemySuggestions.find( (s: EntitySuggestionType) => s.sharedId === 'shared9' && s.language === 'en' ).state - ).toBe(SuggestionState.emptyMismatch); + ).toEqual({ + labeled: false, + withValue: false, + withSuggestion: true, + match: false, + hasContext: true, + obsolete: false, + processing: false, + error: false, + }); }); it('should return error status', async () => { - const { suggestions } = await getSuggestions(factory.id('age_extractor').toString()); - expect(suggestions.find((s: EntitySuggestionType) => s.sharedId === 'shared4').state).toBe( - SuggestionState.error - ); + const { suggestions } = await getSuggestions({ + extractorId: factory.id('age_extractor').toString(), + }); + expect( + suggestions.find((s: EntitySuggestionType) => s.sharedId === 'shared4').state.error + ).toBe(true); }); }); @@ -326,70 +543,128 @@ describe('suggestions', () => { await Suggestions.updateStates({}); }); - it('should accept a suggestion', async () => { - const { suggestions } = await getSuggestions(factory.id('super_powers_extractor').toString()); - const labelMismatchedSuggestion = suggestions.find( - (sug: any) => sug.state === SuggestionState.labelMismatch + it('should accept suggestions', async () => { + const { suggestions } = await getSuggestions({ + extractorId: factory.id('super_powers_extractor').toString(), + }); + const labelMismatchedSuggestions = suggestions.filter( + (sug: any) => sug.state.labeled && !sug.state.match ); + const ids = new Set(labelMismatchedSuggestions.map((sug: any) => sug._id.toString())); await Suggestions.accept( + labelMismatchedSuggestions.map((sug: any) => ({ + _id: sug._id, + sharedId: sug.sharedId, + entityId: sug.entityId, + })) + ); + const { suggestions: newSuggestions } = await getSuggestions({ + extractorId: factory.id('super_powers_extractor').toString(), + }); + const changedSuggestions = newSuggestions.filter((sug: any) => ids.has(sug._id.toString())); + const matchState = { + labeled: true, + withValue: true, + withSuggestion: true, + match: true, + hasContext: true, + obsolete: false, + processing: false, + error: false, + }; + expect(changedSuggestions).toMatchObject([ { - _id: suggestionId, - sharedId: labelMismatchedSuggestion.sharedId, - entityId: labelMismatchedSuggestion.entityId, + _id: labelMismatchedSuggestions[0]._id, + state: matchState, + suggestedValue: labelMismatchedSuggestions[0].suggestedValue, + labeledValue: labelMismatchedSuggestions[0].suggestedValue, }, - false - ); - const { suggestions: newSuggestions } = await getSuggestions( - factory.id('super_powers_extractor').toString() - ); - const changedSuggestion = newSuggestions.find( - (sg: any) => sg._id.toString() === suggestionId.toString() - ); + { + _id: labelMismatchedSuggestions[1]._id, + state: matchState, + suggestedValue: labelMismatchedSuggestions[1].suggestedValue, + labeledValue: labelMismatchedSuggestions[1].suggestedValue, + }, + ]); + }); - expect(changedSuggestion.state).toBe(SuggestionState.labelMatch); - expect(changedSuggestion.suggestedValue).toEqual(changedSuggestion.labeledValue); + it('should require all suggestions to come from the same extractor', async () => { + const [ageSuggestion] = (await getSuggestions({ extractorId: factory.id('age_extractor') })) + .suggestions; + const [superPowersSuggestion] = ( + await getSuggestions({ + extractorId: factory.id('super_powers_extractor'), + }) + ).suggestions; + await expect( + Suggestions.accept([ + { + _id: ageSuggestion._id, + sharedId: ageSuggestion.sharedId, + entityId: ageSuggestion.entityId, + }, + { + _id: superPowersSuggestion._id, + sharedId: superPowersSuggestion.sharedId, + entityId: superPowersSuggestion.entityId, + }, + ]) + ).rejects.toThrow('All suggestions must come from the same extractor'); }); + it('should not accept a suggestion with an error', async () => { - const { suggestions } = await getSuggestions(factory.id('age_extractor').toString()); + const { suggestions } = await getSuggestions({ + extractorId: factory.id('age_extractor').toString(), + }); const errorSuggestion = suggestions.find( (s: EntitySuggestionType) => s.sharedId === 'shared4' ); try { - await Suggestions.accept( + await Suggestions.accept([ { _id: errorSuggestion._id, sharedId: errorSuggestion.sharedId, entityId: errorSuggestion.entityId, }, - true - ); + ]); } catch (e: any) { - expect(e?.message).toBe('Suggestion has an error'); + expect(e?.message).toBe('Some Suggestions have an error.'); } }); + it('should update entities of all languages if property name is numeric or date', async () => { - const { suggestions } = await getSuggestions(factory.id('age_extractor').toString()); - const shared2Suggestion = suggestions.find(sug => sug.sharedId === 'shared2'); - await Suggestions.accept( + const { suggestions } = await getSuggestions({ + extractorId: factory.id('age_extractor').toString(), + }); + const suggestionsToAccept = suggestions.filter( + sug => sug.sharedId === 'shared2' || sug.sharedId === 'shared1' + ); + await Suggestions.accept([ { - _id: shared2AgeSuggestionId, - sharedId: shared2Suggestion.sharedId, - entityId: shared2Suggestion.entityId, + _id: suggestionsToAccept[0]._id, + sharedId: suggestionsToAccept[0].sharedId, + entityId: suggestionsToAccept[0].entityId, }, - false - ); + { + _id: suggestionsToAccept[1]._id, + sharedId: suggestionsToAccept[1].sharedId, + entityId: suggestionsToAccept[1].entityId, + }, + ]); - const entities = await db.mongodb + const entities1 = await db.mongodb ?.collection('entities') - .find({ sharedId: shared2Suggestion.sharedId }) + .find({ sharedId: 'shared1' }) .toArray(); + const ages1 = entities1?.map(entity => entity.metadata.age[0].value); + expect(ages1).toEqual(['17', '17']); - const propertyValues = entities?.map(entity => entity.metadata.age); - expect(propertyValues).not.toBe(undefined); - const ages = propertyValues?.map(value => value[0].value) as string[]; - expect(ages[0]).toEqual('20'); - expect(ages[1]).toEqual('20'); - expect(ages[2]).toEqual('20'); + const entities2 = await db.mongodb + ?.collection('entities') + .find({ sharedId: 'shared2' }) + .toArray(); + const ages2 = entities2?.map(entity => entity.metadata.age[0].value); + expect(ages2).toEqual(['20', '20', '20']); }); }); @@ -399,14 +674,18 @@ describe('suggestions', () => { await Suggestions.save(newErroringSuggestion); expect(await findOneSuggestion({ entityId: 'new_erroring_suggestion' })).toMatchObject({ ...newErroringSuggestion, - state: SuggestionState.error, + state: { + error: true, + }, }); const original = await findOneSuggestion({}); const changed: IXSuggestionType = { ...original, status: 'failed' }; await Suggestions.save(changed); expect(await findOneSuggestion({ _id: original._id })).toMatchObject({ ...changed, - state: SuggestionState.error, + state: { + error: true, + }, }); }); }); @@ -416,33 +695,34 @@ describe('suggestions', () => { await Suggestions.save(newProcessingSuggestion); expect(await findOneSuggestion({ entityId: 'new_processing_suggestion' })).toMatchObject({ ...newProcessingSuggestion, - state: 'Processing', + state: { + processing: true, + }, }); const original = await findOneSuggestion({}); const changed: IXSuggestionType = { ...original, status: 'processing' }; await Suggestions.save(changed); expect(await findOneSuggestion({ _id: original._id })).toMatchObject({ ...changed, - state: 'Processing', + state: { + processing: true, + }, }); }); }); }); describe('updateStates()', () => { - it.each(stateUpdateCases)( - 'should mark $state in state if $reason', - async ({ state, suggestionQuery }) => { - const original = await findOneSuggestion(suggestionQuery); - const idQuery = { _id: original._id }; - await Suggestions.updateStates(idQuery); - const changed = await findOneSuggestion(idQuery); - expect(changed).toMatchObject({ - ...original, - state, - }); - } - ); + it.each(stateUpdateCases)('should mark $reason', async ({ state, suggestionQuery }) => { + const original = await findOneSuggestion(suggestionQuery); + const idQuery = { _id: original._id }; + await Suggestions.updateStates(idQuery); + const changed = await findOneSuggestion(idQuery); + expect(changed).toMatchObject({ + ...original, + state, + }); + }); }); describe('setObsolete()', () => { @@ -450,8 +730,8 @@ describe('suggestions', () => { const query = { entityId: 'shared1' }; await Suggestions.setObsolete(query); const obsoletes = await db.mongodb?.collection('ixsuggestions').find(query).toArray(); - expect(obsoletes?.every(s => s.state === SuggestionState.obsolete)).toBe(true); - expect(obsoletes?.length).toBe(3); + expect(obsoletes?.every(s => s.state.obsolete)).toBe(true); + expect(obsoletes?.length).toBe(4); }); }); @@ -460,7 +740,7 @@ describe('suggestions', () => { const query = { entityId: 'shared1' }; await Suggestions.markSuggestionsWithoutSegmentation(query); const notSegmented = await db.mongodb?.collection('ixsuggestions').find(query).toArray(); - expect(notSegmented?.every(s => s.state === SuggestionState.error)).toBe(true); + expect(notSegmented?.every(s => s.state.error)).toBe(true); }); it('should not mark suggestions when segmentations are correct', async () => { @@ -475,9 +755,9 @@ describe('suggestions', () => { .find({ _id: shared2AgeSuggestionId }) .toArray(); expect(segmented?.length).toBe(1); - expect(segmented?.every(s => s.state === SuggestionState.error)).toBe(false); + expect(segmented?.every(s => s.state?.error)).toBe(false); expect(notSegmented?.length).toBe(1); - expect(notSegmented?.every(s => s.state === SuggestionState.error)).toBe(true); + expect(notSegmented?.every(s => s.state.error)).toBe(true); }); }); @@ -507,12 +787,16 @@ describe('suggestions', () => { expect(await findOneSuggestion({ entityId: newErroringSuggestion.entityId })).toMatchObject({ ...newErroringSuggestion, - state: SuggestionState.error, + state: { + error: true, + }, }); expect(await findOneSuggestion({ entityId: newProcessingSuggestion.entityId })).toMatchObject( { ...newProcessingSuggestion, - state: SuggestionState.processing, + state: { + processing: true, + }, } ); }); diff --git a/app/api/suggestions/stats.ts b/app/api/suggestions/stats.ts deleted file mode 100644 index 1ff5cca661..0000000000 --- a/app/api/suggestions/stats.ts +++ /dev/null @@ -1,95 +0,0 @@ -import { ObjectId } from 'mongodb'; -import { ObjectIdSchema } from 'shared/types/commonTypes'; -import { SuggestionState } from 'shared/types/suggestionSchema'; -import { SuggestionsStats } from 'shared/types/suggestionStats'; -import { IXSuggestionsModel } from './IXSuggestionsModel'; - -interface StateGroup { - _id: T; - count: number; -} - -interface Groups { - buckets: StateGroup[]; - all: StateGroup<'all'>[]; -} - -const addCount = (sum: number, group: StateGroup) => sum + group.count; - -const addCountsOf = (groups: Groups, _states: SuggestionState[]) => { - const states = new Set(_states); - return groups.buckets.filter(g => states.has(g._id)).reduce(addCount, 0); -}; - -const getGroups = async (extractorId: ObjectIdSchema): Promise => - IXSuggestionsModel.db - .aggregate([ - { $match: { extractorId } }, - { - $facet: { - buckets: [ - { - $group: { - _id: '$state', - count: { - $sum: 1, - }, - }, - }, - ], - all: [ - { - $count: 'count', - }, - ], - }, - }, - ]) - .then(([result]) => result); - -const calcAccuracy = (groups: Groups) => { - const correct = addCountsOf(groups, [SuggestionState.labelMatch, SuggestionState.valueMatch]); - const incorect = addCountsOf(groups, [ - SuggestionState.labelMismatch, - SuggestionState.valueMismatch, - SuggestionState.labelEmpty, - SuggestionState.valueEmpty, - ]); - const total = correct + incorect; - return total ? correct / total : 0; -}; - -const getStats = async (_extractorId: string): Promise => { - const extractorId = new ObjectId(_extractorId); - const groups = await getGroups(extractorId); - - const labeled = addCountsOf(groups, [ - SuggestionState.labelMatch, - SuggestionState.labelMismatch, - SuggestionState.labelEmpty, - ]); - const nonLabeledMatching = addCountsOf(groups, [SuggestionState.valueMatch]); - const nonLabeledNotMatching = addCountsOf(groups, [SuggestionState.valueMismatch]); - const emptyOrObsolete = addCountsOf(groups, [ - SuggestionState.empty, - SuggestionState.obsolete, - SuggestionState.valueEmpty, - SuggestionState.emptyMismatch, - ]); - const all = groups.all[0]?.count || 0; - - const accuracy = calcAccuracy(groups); - - return { - counts: { - labeled, - nonLabeledMatching, - nonLabeledNotMatching, - emptyOrObsolete, - all, - }, - accuracy, - }; -}; - -export { getStats }; diff --git a/app/api/suggestions/suggestions.ts b/app/api/suggestions/suggestions.ts index 05dd28870f..85474c0fff 100644 --- a/app/api/suggestions/suggestions.ts +++ b/app/api/suggestions/suggestions.ts @@ -1,30 +1,38 @@ -import { FilterQuery } from 'mongoose'; - +import { ObjectId } from 'mongodb'; import entities from 'api/entities/entities'; import { files } from 'api/files/files'; +import { EnforcedWithId } from 'api/odm'; import settings from 'api/settings/settings'; import { IXSuggestionsModel } from 'api/suggestions/IXSuggestionsModel'; import templates from 'api/templates'; +import { syncedPromiseLoop } from 'shared/data_utils/promiseUtils'; import { ExtractedMetadataSchema, LanguagesListSchema, ObjectIdSchema, } from 'shared/types/commonTypes'; import { EntitySchema } from 'shared/types/entityType'; -import { SuggestionState } from 'shared/types/suggestionSchema'; -import { IXSuggestionsFilter, IXSuggestionType } from 'shared/types/suggestionType'; -import { ObjectId } from 'mongodb'; +import { FileType } from 'shared/types/fileType'; +import { + IXSuggestionAggregation, + IXSuggestionsFilter, + IXSuggestionsQuery, + IXSuggestionType, + SuggestionCustomFilter, +} from 'shared/types/suggestionType'; +import { objectIndex } from 'shared/data_utils/objectIndex'; import { getSegmentedFilesIds } from 'api/services/informationextraction/getFiles'; import { registerEventListeners } from './eventListeners'; import { + baseQueryFragment, + filterFragments, getCurrentValueStage, getEntityStage, getFileStage, getLabeledValueStage, getMatchStage, - groupByAndSort, + groupByAndCount, } from './pipelineStages'; -import { getStats } from './stats'; import { updateStates } from './updateState'; interface AcceptedSuggestion { @@ -35,74 +43,114 @@ interface AcceptedSuggestion { const updateEntitiesWithSuggestion = async ( allLanguages: boolean, - acceptedSuggestion: AcceptedSuggestion, - suggestion: IXSuggestionType + acceptedSuggestions: AcceptedSuggestion[], + suggestions: IXSuggestionType[] ) => { + const sharedIds = acceptedSuggestions.map(s => s.sharedId); + const entityIds = acceptedSuggestions.map(s => s.entityId); + const { propertyName } = suggestions[0]; const query = allLanguages - ? { sharedId: acceptedSuggestion.sharedId } - : { sharedId: acceptedSuggestion.sharedId, _id: acceptedSuggestion.entityId }; + ? { sharedId: { $in: sharedIds } } + : { sharedId: { $in: sharedIds }, _id: { $in: entityIds } }; const storedEntities = await entities.get(query, '+permissions'); + + const acceptedSuggestionsBySharedId = objectIndex( + acceptedSuggestions, + as => as.sharedId, + as => as + ); + const suggestionsById = objectIndex( + suggestions, + s => s._id?.toString() || '', + s => s + ); + + const getValue = (entity: EntitySchema) => + suggestionsById[acceptedSuggestionsBySharedId[entity.sharedId?.toString() || '']._id.toString()] + .suggestedValue; + const entitiesToUpdate = - suggestion.propertyName !== 'title' + propertyName !== 'title' ? storedEntities.map((entity: EntitySchema) => ({ ...entity, metadata: { ...entity.metadata, - [suggestion.propertyName]: [{ value: suggestion.suggestedValue }], + [propertyName]: [ + { + value: getValue(entity), + }, + ], }, permissions: entity.permissions || [], })) : storedEntities.map((entity: EntitySchema) => ({ ...entity, - title: suggestion.suggestedValue, + title: getValue(entity), })); await entities.saveMultiple(entitiesToUpdate); }; -const updateExtractedMetadata = async (suggestion: IXSuggestionType) => { - const fetchedFiles = await files.get({ _id: suggestion.fileId }); +const updateExtractedMetadata = async (suggestions: IXSuggestionType[]) => { + const fetchedFiles = await files.get({ _id: { $in: suggestions.map(s => s.fileId) } }); + const suggestionsByFileId = objectIndex( + suggestions, + s => s.fileId?.toString() || '', + s => s + ); - if (!fetchedFiles?.length) return Promise.resolve(); - const file = fetchedFiles[0]; + await syncedPromiseLoop(fetchedFiles, async (file: EnforcedWithId) => { + const suggestion = suggestionsByFileId[file._id.toString()]; + file.extractedMetadata = file.extractedMetadata ? file.extractedMetadata : []; - file.extractedMetadata = file.extractedMetadata ? file.extractedMetadata : []; - const extractedMetadata = file.extractedMetadata.find( - (em: any) => em.name === suggestion.propertyName - ) as ExtractedMetadataSchema; + const extractedMetadata = file.extractedMetadata.find( + (em: any) => em.name === suggestion.propertyName + ) as ExtractedMetadataSchema; - if (!extractedMetadata) { - file.extractedMetadata.push({ - name: suggestion.propertyName, - timestamp: Date(), - selection: { + if (!extractedMetadata) { + file.extractedMetadata.push({ + name: suggestion.propertyName, + timestamp: Date(), + selection: { + text: suggestion.suggestedText || suggestion.suggestedValue?.toString(), + selectionRectangles: suggestion.selectionRectangles, + }, + }); + } else { + extractedMetadata.timestamp = Date(); + extractedMetadata.selection = { text: suggestion.suggestedText || suggestion.suggestedValue?.toString(), selectionRectangles: suggestion.selectionRectangles, - }, - }); - } else { - extractedMetadata.timestamp = Date(); - extractedMetadata.selection = { - text: suggestion.suggestedText || suggestion.suggestedValue?.toString(), - selectionRectangles: suggestion.selectionRectangles, - }; - } - return files.save(file); + }; + } + + return files.save(file); + }); }; const buildListQuery = ( - filters: FilterQuery, + extractorId: ObjectId, + customFilter: SuggestionCustomFilter | undefined, setLanguages: LanguagesListSchema | undefined, offset: number, - limit: number + limit: number, + sort?: IXSuggestionsQuery['sort'] ) => { + const sortOrder = sort?.order === 'desc' ? -1 : 1; + const sorting = sort?.property ? { [sort.property]: sortOrder } : { date: 1, state: -1 }; + const pipeline = [ - ...getMatchStage(filters), - { $sort: { date: 1, state: -1 } }, - { $skip: offset }, - { $limit: limit }, + ...getMatchStage(extractorId, customFilter), ...getEntityStage(setLanguages!), ...getCurrentValueStage(), + { + $addFields: { + entityTitle: '$entity.title', + }, + }, + { $sort: sorting }, + { $skip: offset }, + { $limit: limit }, ...getFileStage(), ...getLabeledValueStage(), { @@ -110,7 +158,7 @@ const buildListQuery = ( entityId: '$entity._id', entityTemplateId: '$entity.template', sharedId: '$entity.sharedId', - entityTitle: '$entity.title', + entityTitle: 1, fileId: 1, language: 1, propertyName: 1, @@ -130,59 +178,56 @@ const buildListQuery = ( return pipeline; }; -const buildTemplateAggregationsQuery = (_filters: FilterQuery) => { - const { entityTemplate, ...filters } = _filters; - const pipeline = [...getMatchStage(filters), ...groupByAndSort('$entityTemplate')]; - return pipeline; -}; +async function getLabeledCounts(extractorId: ObjectId) { + const labeledAggregationQuery = [ + { + $match: { + ...baseQueryFragment(extractorId), + ...filterFragments.labeled._fragment, + }, + }, + ...groupByAndCount('$state.match'), + ]; + const labeledAggregation: { _id: boolean; count: number }[] = + await IXSuggestionsModel.db.aggregate(labeledAggregationQuery); + const matchCount = + labeledAggregation.find((aggregation: any) => aggregation._id === true)?.count || 0; + const mismatchCount = + labeledAggregation.find((aggregation: any) => aggregation._id === false)?.count || 0; + const labeledCount = matchCount + mismatchCount; + return { labeledCount, matchCount, mismatchCount }; +} -const buildStateAggregationsQuery = (_filters: FilterQuery) => { - const { state, ...filters } = _filters; - const pipeline = [...getMatchStage(filters), ...groupByAndSort('$state')]; - return pipeline; +const getNonLabeledCounts = async (_extractorId: ObjectId) => { + const extractorId = new ObjectId(_extractorId); + const unlabeledMatch = { + ...baseQueryFragment(extractorId), + ...filterFragments.nonLabeled._fragment, + }; + const nonLabeledCount = await IXSuggestionsModel.count(unlabeledMatch); + const noContextCount = await IXSuggestionsModel.count({ + ...unlabeledMatch, + ...filterFragments.nonLabeled.noContext, + }); + const noSuggestionCount = await IXSuggestionsModel.count({ + ...unlabeledMatch, + ...filterFragments.nonLabeled.noSuggestion, + }); + const obsoleteCount = await IXSuggestionsModel.count({ + ...unlabeledMatch, + ...filterFragments.nonLabeled.obsolete, + }); + const othersCount = await IXSuggestionsModel.count({ + ...unlabeledMatch, + ...filterFragments.nonLabeled.others, + }); + return { nonLabeledCount, noContextCount, noSuggestionCount, obsoleteCount, othersCount }; }; -const fetchAndAggregateSuggestions = async ( - _filters: Omit, - setLanguages: LanguagesListSchema | undefined, - offset: number, - limit: number -) => { - const { - states, - entityTemplates, - ...filters - }: { - states?: string[]; - entityTemplates?: string[]; - extractorId?: ObjectIdSchema; - state?: { $in: string[] }; - entityTemplate?: { $in: string[] }; - } = _filters; - if (states) filters.state = { $in: _filters.states || [] }; - if (entityTemplates) filters.entityTemplate = { $in: _filters.entityTemplates || [] }; - - const count = await IXSuggestionsModel.db - .aggregate([{ $match: { ...filters, status: { $ne: 'processing' } } }, { $count: 'count' }]) - .then(result => (result?.length ? result[0].count : 0)); - - const suggestions = await IXSuggestionsModel.db.aggregate( - buildListQuery(filters, setLanguages, offset, limit) - ); - - const templateAggregations = await IXSuggestionsModel.db.aggregate( - buildTemplateAggregationsQuery(filters) - ); - - const stateAggregations = await IXSuggestionsModel.db.aggregate( - buildStateAggregationsQuery(filters) - ); - - return { - suggestions, - aggregations: { template: templateAggregations, state: stateAggregations }, - totalPages: Math.ceil(count / limit), - }; +const readFilter = (filter: IXSuggestionsFilter) => { + const { customFilter, extractorId: _extractorId } = filter; + const extractorId = new ObjectId(_extractorId); + return { customFilter, extractorId }; }; const Suggestions = { @@ -190,23 +235,60 @@ const Suggestions = { getByEntityId: async (sharedId: string) => IXSuggestionsModel.get({ entityId: sharedId }), getByExtractor: async (extractorId: ObjectIdSchema) => IXSuggestionsModel.get({ extractorId }), - get: async (filter: IXSuggestionsFilter, options: { page: { size: number; number: number } }) => { + get: async ( + filter: IXSuggestionsFilter, + options: { + page?: IXSuggestionsQuery['page']; + sort?: IXSuggestionsQuery['sort']; + } + ) => { const offset = options && options.page ? options.page.size * (options.page.number - 1) : 0; const DEFAULT_LIMIT = 30; const limit = options.page?.size || DEFAULT_LIMIT; const { languages: setLanguages } = await settings.get(); - const { language, ...filters } = filter; - filters.extractorId = new ObjectId(filter.extractorId); + const { customFilter, extractorId } = readFilter(filter); + + const count = await IXSuggestionsModel.db + .aggregate(getMatchStage(extractorId, customFilter, true)) + .then(result => (result?.length ? result[0].count : 0)); + + const suggestions = await IXSuggestionsModel.db.aggregate( + buildListQuery(extractorId, customFilter, setLanguages, offset, limit, options.sort) + ); - return fetchAndAggregateSuggestions(filters, setLanguages, offset, limit); + return { + suggestions, + totalPages: Math.ceil(count / limit), + }; }, - getStats, + aggregate: async (_extractorId: ObjectIdSchema): Promise => { + const extractorId = new ObjectId(_extractorId); + const { labeledCount, matchCount, mismatchCount } = await getLabeledCounts(extractorId); + const { nonLabeledCount, noContextCount, noSuggestionCount, obsoleteCount, othersCount } = + await getNonLabeledCounts(extractorId); + const totalCount = labeledCount + nonLabeledCount; + return { + total: totalCount, + labeled: { + _count: labeledCount, + match: matchCount, + mismatch: mismatchCount, + }, + nonLabeled: { + _count: nonLabeledCount, + noContext: noContextCount, + noSuggestion: noSuggestionCount, + obsolete: obsoleteCount, + others: othersCount, + }, + }; + }, updateStates, setObsolete: async (query: any) => - IXSuggestionsModel.updateMany(query, { $set: { state: SuggestionState.obsolete } }), + IXSuggestionsModel.updateMany(query, { $set: { 'state.obsolete': true } }), markSuggestionsWithoutSegmentation: async (query: any) => { const segmentedFilesIds = await getSegmentedFilesIds(); @@ -215,45 +297,39 @@ const Suggestions = { ...query, fileId: { $nin: segmentedFilesIds }, }, - { $set: { state: SuggestionState.error } } + { $set: { 'state.error': true } } ); }, save: async (suggestion: IXSuggestionType) => Suggestions.saveMultiple([suggestion]), saveMultiple: async (_suggestions: IXSuggestionType[]) => { - const toSave: IXSuggestionType[] = []; - const toSaveAndUpdate: IXSuggestionType[] = []; - _suggestions.forEach(s => { - if (s.status === 'failed') { - toSave.push({ ...s, state: SuggestionState.error }); - } else if (s.status === 'processing') { - toSave.push({ ...s, state: SuggestionState.processing }); - } else { - toSaveAndUpdate.push(s); - } - }); - await IXSuggestionsModel.saveMultiple(toSave); - const toUpdate = await IXSuggestionsModel.saveMultiple(toSaveAndUpdate); - if (toUpdate.length) await updateStates({ _id: { $in: toUpdate.map(s => s._id) } }); + const toUpdate = await IXSuggestionsModel.saveMultiple(_suggestions); + if (toUpdate.length > 0) await updateStates({ _id: { $in: toUpdate.map(s => s._id) } }); }, - accept: async (acceptedSuggestion: AcceptedSuggestion, allLanguages: boolean) => { - const suggestion = await IXSuggestionsModel.getById(acceptedSuggestion._id); - if (!suggestion) { - throw new Error('Suggestion not found'); + accept: async (acceptedSuggestions: AcceptedSuggestion[]) => { + const acceptedIds = Array.from(new Set(acceptedSuggestions.map(s => s._id.toString()))); + const suggestions = await IXSuggestionsModel.get({ _id: { $in: acceptedIds } }); + const extractors = new Set(suggestions.map(s => s.extractorId.toString())); + if (extractors.size > 1) { + throw new Error('All suggestions must come from the same extractor'); } - if (suggestion.error !== '') { - throw new Error('Suggestion has an error'); + const foundIds = new Set(suggestions.map(s => s._id.toString())); + if (!acceptedIds.every(id => foundIds.has(id))) { + throw new Error('Suggestion(s) not found.'); } - let shouldUpdateAllLanguages = allLanguages; - const property = await templates.getPropertyByName(suggestion.propertyName); - if (property && ['numeric', 'date'].includes(property.type)) { - shouldUpdateAllLanguages = true; + if (suggestions.some(s => s.error !== '')) { + throw new Error('Some Suggestions have an error.'); } - await updateEntitiesWithSuggestion(shouldUpdateAllLanguages, acceptedSuggestion, suggestion); - await updateExtractedMetadata(suggestion); - await Suggestions.updateStates({ _id: acceptedSuggestion._id }); + + const { propertyName } = suggestions[0]; + const property = await templates.getPropertyByName(propertyName); + const allLanguage = property.type === 'numeric' || property.type === 'date'; + + await updateEntitiesWithSuggestion(allLanguage, acceptedSuggestions, suggestions); + await updateExtractedMetadata(suggestions); + await Suggestions.updateStates({ _id: { $in: acceptedIds.map(id => new ObjectId(id)) } }); }, deleteByEntityId: async (sharedId: string) => { diff --git a/app/api/suggestions/updateState.ts b/app/api/suggestions/updateState.ts index d7fd6baf30..1c1e4edc32 100644 --- a/app/api/suggestions/updateState.ts +++ b/app/api/suggestions/updateState.ts @@ -48,7 +48,7 @@ const getModelCreationDateStage = () => [ const findSuggestions = (query: any, languages: LanguagesListSchema) => IXSuggestionsModel.db .aggregateCursor([ - { $match: { ...query, status: { $ne: 'processing' } } }, + { $match: { ...query } }, ...getEntityStage(languages), ...getCurrentValueStage(), { @@ -72,6 +72,9 @@ const findSuggestions = (query: any, languages: LanguagesListSchema) => date: 1, propertyName: 1, extractorId: 1, + status: 1, + state: 1, + segment: 1, }, }, ]) diff --git a/app/api/sync/routes.ts b/app/api/sync/routes.ts index b30de220d7..afaa054695 100644 --- a/app/api/sync/routes.ts +++ b/app/api/sync/routes.ts @@ -9,6 +9,7 @@ import { Application, Request } from 'express'; import { TranslationType } from 'shared/translationType'; import { FileType } from 'shared/types/fileType'; +import { TemplateSchema } from 'shared/types/templateType'; import { needsAuthorization } from '../auth'; const diskStorage = multer.diskStorage({ @@ -79,6 +80,29 @@ const preserveTranslations = async (syncData: TranslationType): Promise => { + const syncDataArray = Array.isArray(syncData) ? syncData : [syncData]; + const syncedDefault = syncDataArray.find(template => template.default); + if (syncedDefault) { + const [otherDefault] = (await models + .templates() + .get({ _id: { $ne: syncedDefault._id }, default: true })) as TemplateSchema[]; + if (otherDefault) { + return [ + { + ...otherDefault, + default: false, + }, + ...syncDataArray, + ]; + } + } + + return syncData; +}; + export default (app: Application) => { app.post('/api/sync', needsAuthorization(['admin']), async (req, res, next) => { try { @@ -91,6 +115,10 @@ export default (app: Application) => { req.body.data = await preserveTranslations(req.body.data); } + if (req.body.namespace === 'templates') { + req.body.data = await keepOnlyOneDefaultTemplate(req.body.data); + } + await (Array.isArray(req.body.data) ? models[req.body.namespace]().saveMultiple(req.body.data) : models[req.body.namespace]().save(req.body.data)); diff --git a/app/api/sync/specs/routes.spec.js b/app/api/sync/specs/routes.spec.js index 7d5022248f..ae555c7399 100644 --- a/app/api/sync/specs/routes.spec.js +++ b/app/api/sync/specs/routes.spec.js @@ -99,6 +99,37 @@ describe('sync', () => { expect(templates.saveMultiple).toHaveBeenCalledWith([{ _id: 'id1' }, { _id: 'id2' }]); expect(index.updateMapping).toHaveBeenCalledWith(req.body.data); }); + + it('should set the rest of the templates as non-default if the provided is default', async () => { + jest.spyOn(index, 'updateMapping').mockImplementation(() => {}); + const templates = { + saveMultiple: jest.fn(), + get: jest.fn().mockResolvedValue([ + { + _id: 'prevDefault', + name: 'Previous default', + default: true, + }, + ]), + }; + models.templates = () => templates; + + req.body = { + namespace: 'templates', + data: { _id: 'id', default: true }, + }; + + await routes.post('/api/sync', req); + expect(templates.saveMultiple).toHaveBeenCalledWith([ + { + _id: 'prevDefault', + name: 'Previous default', + default: false, + }, + { _id: 'id', default: true }, + ]); + expect(index.updateMapping).toHaveBeenCalledWith(req.body.data); + }); }); describe('when namespace is entities', () => { diff --git a/app/api/templates/specs/templates.spec.js b/app/api/templates/specs/templates.spec.js index 79d55b8efe..bce97c9c96 100644 --- a/app/api/templates/specs/templates.spec.js +++ b/app/api/templates/specs/templates.spec.js @@ -1,6 +1,3 @@ -/* eslint-disable max-lines */ -/* eslint-disable max-statements */ - import Ajv from 'ajv'; import db from 'api/utils/testing_db'; import documents from 'api/documents/documents.js'; @@ -587,7 +584,7 @@ describe('templates', () => { describe('getPropertyByName()', () => { it('should get properties with the name provided', async () => { const newTemplate = { - name: 'created template 2', + name: 'created template 2', commonProperties: [{ name: 'title', label: 'Title', type: 'text' }], properties: [ { label: 'label', type: 'text' }, @@ -600,11 +597,46 @@ describe('templates', () => { expect(property.type).toEqual('date'); }); - it('should throw an error when no template is found', async () => { + it('should throw an error when the property is not found', async () => { try { await templates.getPropertyByName('nonexistent property name'); } catch (e) { - expect(e.message).toEqual('No template with the given property name'); + expect(e.message).toEqual('Properties not found: nonexistent property name'); + } + }); + }); + + describe('getPropertiesByName()', () => { + it('should get properties with the name provided', async () => { + const newTemplate = { + name: 'created template 3', + commonProperties: [{ name: 'title', label: 'Title', type: 'text' }], + properties: [ + { label: 'label', type: 'text' }, + { label: 'Date', type: 'date' }, + ], + }; + const newTemplate2 = { + name: 'created template 4', + commonProperties: [{ name: 'title', label: 'Title', type: 'text' }], + properties: [{ label: 'number', type: 'numeric' }], + }; + await templates.save(newTemplate); + await templates.save(newTemplate2); + const properties = await templates.getPropertiesByName(['date', 'label', 'number', 'title']); + expect(properties).toMatchObject([ + { name: 'title', type: 'text' }, + { name: 'label', type: 'text' }, + { name: 'date', type: 'date' }, + { name: 'number', type: 'numeric' }, + ]); + }); + + it('should throw an error when a property is not found', async () => { + try { + await templates.getPropertiesByName(['nonexistent property name']); + } catch (e) { + expect(e.message).toEqual('Properties not found: nonexistent property name'); } }); }); diff --git a/app/api/templates/templates.ts b/app/api/templates/templates.ts index 7fbb1bd805..c28acf0805 100644 --- a/app/api/templates/templates.ts +++ b/app/api/templates/templates.ts @@ -1,3 +1,5 @@ +import { ObjectId } from 'mongodb'; + import entities from 'api/entities'; import { populateGeneratedIdByTemplate } from 'api/entities/generatedIdPropertyAutoFiller'; import { applicationEventsBus } from 'api/eventsbus'; @@ -7,7 +9,7 @@ import { updateMapping } from 'api/search/entitiesIndex'; import settings from 'api/settings/settings'; import dictionariesModel from 'api/thesauri/dictionariesModel'; import createError from 'api/utils/Error'; -import { ObjectId } from 'mongodb'; +import { objectIndex } from 'shared/data_utils/objectIndex'; import { propertyTypes } from 'shared/propertyTypes'; import { ContextType } from 'shared/translationSchema'; import { ensure } from 'shared/tsUtils'; @@ -246,14 +248,34 @@ export default { return model.get(query); }, - async getPropertyByName(propertyName: string): Promise { + async getPropertyByName(propertyName: string): Promise { + const [property] = await this.getPropertiesByName([propertyName]); + return property; + }, + + async getPropertiesByName(propertyNames: string[]): Promise { + const nameSet = new Set(propertyNames); const templates = await this.get({ - $or: [{ 'properties.name': propertyName }, { 'commonProperties.name': propertyName }], + $or: [ + { 'properties.name': { $in: propertyNames } }, + { 'commonProperties.name': { $in: propertyNames } }, + ], }); - if (!templates.length) { - throw createError('No template with the given property name'); + const allProperties = templates + .map(template => [template.properties || [], template.commonProperties || []]) + .flat() + .flat() + .filter(t => nameSet.has(t.name)); + const propertiesByName = objectIndex( + allProperties, + p => p.name, + p => p + ); + const missingProperties = propertyNames.filter(name => !propertiesByName[name]); + if (missingProperties.length > 0) { + throw createError(`Properties not found: ${missingProperties.join(', ')}`); } - return templates[0].properties?.find(property => property.name === propertyName); + return Array.from(Object.values(propertiesByName)); }, async setAsDefault(_id: string) { diff --git a/app/api/utils/fixturesFactory.ts b/app/api/utils/fixturesFactory.ts index 1c478a162d..6d08a15a5d 100644 --- a/app/api/utils/fixturesFactory.ts +++ b/app/api/utils/fixturesFactory.ts @@ -17,10 +17,10 @@ import { import { UpdateLog } from 'api/updatelogs'; import { IXExtractorType } from 'shared/types/extractorType'; import { IXSuggestionType } from 'shared/types/suggestionType'; -import { SuggestionState } from 'shared/types/suggestionSchema'; import { WithId } from 'api/odm/model'; import { TemplateSchema } from 'shared/types/templateType'; import { getV2FixturesFactoryElements } from 'api/common.v2/testing/fixturesFactory'; +import { IXModelType } from 'shared/types/IXModelType'; import { PermissionSchema } from 'shared/types/permissionType'; function getIdMapper() { @@ -141,6 +141,18 @@ function getFixturesFactory() { }); }, + fileExtractedMetadata: ( + propertyName: string, + text: string, + rectangles = [{ top: 0, left: 0, width: 0, height: 0, page: '1' }] + ): ExtractedMetadataSchema => ({ + name: propertyName, + selection: { + text, + selectionRectangles: rectangles, + }, + }), + file: ( id: string, entity: string | undefined, @@ -248,6 +260,18 @@ function getFixturesFactory() { templates: templates.map(idMapper), }), + ixModel: ( + name: string, + extractor: string, + creationDate = 1, + status: IXModelType['status'] = 'ready' + ): IXModelType => ({ + _id: idMapper(name), + status, + creationDate, + extractorId: idMapper(extractor), + }), + ixSuggestion: ( suggestionId: string, extractor: string, @@ -269,7 +293,16 @@ function getFixturesFactory() { segment: '', suggestedValue: '', date: 1, - state: SuggestionState.valueEmpty, + state: { + labeled: false, + withValue: true, + withSuggestion: false, + match: false, + hasContext: false, + obsolete: false, + processing: false, + error: false, + }, ...otherProps, }), diff --git a/app/api/utils/testing_db.ts b/app/api/utils/testing_db.ts index 19c64eee08..2e9e5c105a 100644 --- a/app/api/utils/testing_db.ts +++ b/app/api/utils/testing_db.ts @@ -15,6 +15,7 @@ import { IXSuggestionType } from 'shared/types/suggestionType'; import { ThesaurusSchema } from 'shared/types/thesaurusType'; import { UserGroupSchema } from 'shared/types/userGroupType'; import uniqueID from 'shared/uniqueID'; +import { config } from 'api/config'; import { UserSchema } from '../../shared/types/userType'; import { elasticTesting } from './elastic_testing'; import { testingTenants } from './testingTenants'; @@ -71,7 +72,7 @@ const fixturer = { let mongooseConnection: Connection; const initMongoServer = async (dbName: string) => { - const uri = 'mongodb://localhost/'; + const uri = config.DBHOST; mongooseConnection = await DB.connect(`${uri}${dbName}`); connected = true; }; diff --git a/app/react/App/App.js b/app/react/App/App.js index e40a7b469c..bdc795f74c 100644 --- a/app/react/App/App.js +++ b/app/react/App/App.js @@ -67,7 +67,7 @@ const App = ({ customParams }) => {
-