From 9dfaf5669e8ed80819414a163aa1518fb027c173 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 12 Jun 2024 17:48:12 -0500 Subject: [PATCH 1/3] Bump react-datepicker from 6.6.0 to 6.9.0 (#6882) Bumps [react-datepicker](https://github.com/Hacker0x01/react-datepicker) from 6.6.0 to 6.9.0. - [Release notes](https://github.com/Hacker0x01/react-datepicker/releases) - [Commits](https://github.com/Hacker0x01/react-datepicker/compare/v6.6.0...v6.9.0) --- updated-dependencies: - dependency-name: react-datepicker dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 3a7a189934..f6a5db458a 100644 --- a/package.json +++ b/package.json @@ -192,7 +192,7 @@ "qs": "^6.12.1", "react": "^18.3.1", "react-color": "^2.19.3", - "react-datepicker": "6.6.0", + "react-datepicker": "6.9.0", "react-device-detect": "^2.2.3", "react-dnd": "^16.0.1", "react-dnd-html5-backend": "^16.0.1", diff --git a/yarn.lock b/yarn.lock index 30188fd77b..1f8b75b95b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -15058,10 +15058,10 @@ react-confetti@^6.1.0: dependencies: tween-functions "^1.2.0" -react-datepicker@6.6.0: - version "6.6.0" - resolved "https://registry.yarnpkg.com/react-datepicker/-/react-datepicker-6.6.0.tgz#0128547211c8fece08fef0b5406efffff2d36f1f" - integrity sha512-ERC0/Q4pPC9bNIcGUpdCbHc+oCxhkU3WI3UOGHkyJ3A9fqALCYpEmLc5S5xvAd7DuCDdbsyW97oRPM6pWWwjww== +react-datepicker@6.9.0: + version "6.9.0" + resolved "https://registry.yarnpkg.com/react-datepicker/-/react-datepicker-6.9.0.tgz#0ad234dad81d567ae64cad79697bbad69c95490b" + integrity sha512-QTxuzeem7BUfVFWv+g5WuvzT0c5BPo+XTCNbMTZKSZQLU+cMMwSUHwspaxuIcDlwNcOH0tiJ+bh1fJ2yxOGYWA== dependencies: "@floating-ui/react" "^0.26.2" clsx "^2.1.0" From 43440984338225e442f504b66dc862538bdf4a55 Mon Sep 17 00:00:00 2001 From: LaszloKecskes Date: Thu, 13 Jun 2024 10:34:40 +0200 Subject: [PATCH 2/3] 6798 full validation ix service results (#6875) * error handling test outline * remove loop test * testing for the valid cases * first test, refactor type definitions * first validation * finished validations * refactor formatting * update failing tests * update failing test * allow extra types * handle title properly * fix number material sending * emit types * extra check for typescript * remove eslint ignores --- .../InformationExtraction.ts | 5 +- .../informationextraction/getFiles.ts | 2 + .../informationextraction/ixextractors.ts | 24 +- .../specs/InformationExtraction.spec.ts | 32 +- .../informationextraction/specs/fixtures.ts | 2 - .../specs/suggestionFormatting.spec.ts | 503 ++++++++++++++++++ .../suggestionFormatting.ts | 195 +++++-- .../specs/stringToTypeOfProperty.spec.ts | 17 - app/shared/stringToTypeOfProperty.ts | 22 - app/shared/tsUtils.ts | 20 + app/shared/types/suggestionSchema.ts | 63 +++ app/shared/types/suggestionType.d.ts | 35 ++ 12 files changed, 802 insertions(+), 118 deletions(-) create mode 100644 app/api/services/informationextraction/specs/suggestionFormatting.spec.ts delete mode 100644 app/shared/specs/stringToTypeOfProperty.spec.ts delete mode 100644 app/shared/stringToTypeOfProperty.ts diff --git a/app/api/services/informationextraction/InformationExtraction.ts b/app/api/services/informationextraction/InformationExtraction.ts index b658c38830..3a3e58a3ca 100644 --- a/app/api/services/informationextraction/InformationExtraction.ts +++ b/app/api/services/informationextraction/InformationExtraction.ts @@ -281,7 +281,10 @@ class InformationExtraction { templates, template => template.properties || [] ); - const property = allProps.find(p => p.name === extractor.property); + const property = + extractor.property === 'title' + ? { name: 'title' as 'title', type: 'title' as 'title' } + : allProps.find(p => p.name === extractor.property); const suggestion = await formatSuggestion( property, diff --git a/app/api/services/informationextraction/getFiles.ts b/app/api/services/informationextraction/getFiles.ts index 8ddac8b566..9c66ab0270 100644 --- a/app/api/services/informationextraction/getFiles.ts +++ b/app/api/services/informationextraction/getFiles.ts @@ -174,6 +174,8 @@ async function getFilesForTraining(templates: ObjectIdSchema[], property: string let stringValue: string; if (propertyType === propertyTypes.date) { stringValue = moment(value * 1000).format('YYYY-MM-DD'); + } else if (propertyType === propertyTypes.numeric) { + stringValue = value?.toString() || ''; } else { stringValue = value; } diff --git a/app/api/services/informationextraction/ixextractors.ts b/app/api/services/informationextraction/ixextractors.ts index 4817d6ef5d..e420b6d2b9 100644 --- a/app/api/services/informationextraction/ixextractors.ts +++ b/app/api/services/informationextraction/ixextractors.ts @@ -8,10 +8,12 @@ import { createBlankSuggestionsForExtractor, createBlankSuggestionsForPartialExtractor, } from 'api/suggestions/blankSuggestions'; -import { propertyTypes } from 'shared/propertyTypes'; import { IXExtractorModel as model } from './IXExtractorModel'; -const ALLOWED_PROPERTY_TYPES: (typeof propertyTypes)[keyof typeof propertyTypes][] = [ +type AllowedPropertyTypes = 'title' | 'text' | 'numeric' | 'date' | 'select' | 'multiselect'; + +const ALLOWED_PROPERTY_TYPES: AllowedPropertyTypes[] = [ + 'title', 'text', 'numeric', 'date', @@ -19,6 +21,17 @@ const ALLOWED_PROPERTY_TYPES: (typeof propertyTypes)[keyof typeof propertyTypes] 'multiselect', ]; +const allowedTypeSet = new Set(ALLOWED_PROPERTY_TYPES); + +const typeIsAllowed = (type: string): type is AllowedPropertyTypes => allowedTypeSet.has(type); + +const checkTypeIsAllowed = (type: string) => { + if (!typeIsAllowed(type)) { + throw new Error('Property type not allowed.'); + } + return type; +}; + const templatePropertyExistenceCheck = async (propertyName: string, templateIds: string[]) => { const tArray = await templates.get({ _id: { $in: templateIds } }); const usedTemplates = objectIndex( @@ -43,9 +56,7 @@ const templatePropertyExistenceCheck = async (propertyName: string, templateIds: throw new Error('Missing property.'); } - if (!ALLOWED_PROPERTY_TYPES.includes(property.type)) { - throw new Error('Property type not allowed.'); - } + checkTypeIsAllowed(property.type); }); }; @@ -134,3 +145,6 @@ export const Extractors = { await Suggestions.delete({ entityTemplate: templateId, extractorId: { $in: extractorIds } }); }, }; + +export type { AllowedPropertyTypes }; +export { ALLOWED_PROPERTY_TYPES, typeIsAllowed, checkTypeIsAllowed }; diff --git a/app/api/services/informationextraction/specs/InformationExtraction.spec.ts b/app/api/services/informationextraction/specs/InformationExtraction.spec.ts index fd8de75bfe..6dcb28a90e 100644 --- a/app/api/services/informationextraction/specs/InformationExtraction.spec.ts +++ b/app/api/services/informationextraction/specs/InformationExtraction.spec.ts @@ -610,7 +610,7 @@ describe('InformationExtraction', () => { IXExternalService.setResults([ { tenant: 'tenant1', - property_name: 'property1', + id: factory.id('prop1extractor').toString(), xml_file_name: 'documentA.xml', text: 'text_in_other_language', segment_text: 'segmented_text_in_other_language', @@ -626,7 +626,7 @@ describe('InformationExtraction', () => { }, { tenant: 'tenant1', - property_name: 'property1', + id: factory.id('prop1extractor').toString(), xml_file_name: 'documentD.xml', text: 'text_in_eng_language', segment_text: 'segmented_text_in_eng_language', @@ -813,7 +813,7 @@ describe('InformationExtraction', () => { }, ]); - await saveSuggestionProcess('F1', 'A1', 'eng', 'prop4extractor'); + await saveSuggestionProcess('F1', 'A1', 'eng', 'prop1extractor'); await informationExtraction.processResults({ params: { id: factory.id('prop1extractor').toString() }, @@ -823,25 +823,11 @@ describe('InformationExtraction', () => { data_url: 'http://localhost:1234/suggestions_results', }); - await informationExtraction.processResults({ - params: { id: factory.id('prop4extractor').toString() }, - tenant: 'tenant1', - task: 'suggestions', - success: true, - data_url: 'http://localhost:1234/suggestions_results', - }); - const suggestionsText = await IXSuggestionsModel.get({ status: 'ready', extractorId: factory.id('prop1extractor'), }); expect(suggestionsText.length).toBe(1); - - const suggestionsMarkdown = await IXSuggestionsModel.get({ - status: 'ready', - propertyName: 'property4', - }); - expect(suggestionsMarkdown.length).toBe(1); }); }); @@ -885,16 +871,19 @@ describe('InformationExtraction', () => { id: factory.id('extractorWithSelect').toString(), xml_file_name: 'documentG.xml', values: [{ id: 'A', label: 'A' }], + segment_text: 'it is A', }, { id: factory.id('extractorWithSelect').toString(), xml_file_name: 'documentH.xml', values: [{ id: 'B', label: 'B' }], + segment_text: 'it is B', }, { id: factory.id('extractorWithSelect').toString(), xml_file_name: 'documentI.xml', values: [{ id: 'A', label: 'A' }], + segment_text: 'it is A', }, ], 'value' @@ -947,18 +936,21 @@ describe('InformationExtraction', () => { fileId: factory.id('F17'), entityId: 'A17', suggestedValue: 'A', + segment: 'it is A', }, { ...expectedBase, fileId: factory.id('F18'), entityId: 'A18', suggestedValue: 'B', + segment: 'it is B', }, { ...expectedBase, fileId: factory.id('F19'), entityId: 'A19', suggestedValue: 'A', + segment: 'it is A', state: { ...expectedBase.state, withValue: false, @@ -976,6 +968,7 @@ describe('InformationExtraction', () => { id: factory.id('extractorWithMultiselect').toString(), xml_file_name: 'documentG.xml', values: [{ id: 'A', label: 'A' }], + segment_text: 'it is A', }, { id: factory.id('extractorWithMultiselect').toString(), @@ -984,6 +977,7 @@ describe('InformationExtraction', () => { { id: 'B', label: 'B' }, { id: 'C', label: 'C' }, ], + segment_text: 'it is B or C', }, { id: factory.id('extractorWithMultiselect').toString(), @@ -992,6 +986,7 @@ describe('InformationExtraction', () => { { id: 'A', label: 'A' }, { id: 'C', label: 'C' }, ], + segment_text: 'it is A or C', }, ], 'value' @@ -1044,18 +1039,21 @@ describe('InformationExtraction', () => { fileId: factory.id('F17'), entityId: 'A17', suggestedValue: ['A'], + segment: 'it is A', }, { ...expectedBase, fileId: factory.id('F18'), entityId: 'A18', suggestedValue: ['B', 'C'], + segment: 'it is B or C', }, { ...expectedBase, fileId: factory.id('F19'), entityId: 'A19', suggestedValue: ['A', 'C'], + segment: 'it is A or C', state: { ...expectedBase.state, withValue: false, diff --git a/app/api/services/informationextraction/specs/fixtures.ts b/app/api/services/informationextraction/specs/fixtures.ts index 9713847afc..dc7dc2909d 100644 --- a/app/api/services/informationextraction/specs/fixtures.ts +++ b/app/api/services/informationextraction/specs/fixtures.ts @@ -36,7 +36,6 @@ const fixtures: DBFixture = { ]), factory.ixExtractor('prop2extractor', 'property2', ['templateToSegmentA']), factory.ixExtractor('prop3extractor', 'property3', ['templateToSegmentA']), - factory.ixExtractor('prop4extractor', 'property4', ['templateToSegmentA']), factory.ixExtractor('extractorWithOneFailedSegmentation', 'property15', ['templateToSegmentC']), factory.ixExtractor('extractorWithSelect', 'property_select', ['templateToSegmentD']), factory.ixExtractor('extractorWithMultiselect', 'property_multiselect', ['templateToSegmentD']), @@ -562,7 +561,6 @@ const fixtures: DBFixture = { factory.property('property1', 'text'), factory.property('property2', 'date'), factory.property('property3', 'numeric'), - factory.property('property4', 'markdown'), ]), factory.template('templateToSegmentB', [factory.property('property1', 'text')]), factory.template('templateToSegmentC', [factory.property('property15', 'text')]), diff --git a/app/api/services/informationextraction/specs/suggestionFormatting.spec.ts b/app/api/services/informationextraction/specs/suggestionFormatting.spec.ts new file mode 100644 index 0000000000..c9c828c4f5 --- /dev/null +++ b/app/api/services/informationextraction/specs/suggestionFormatting.spec.ts @@ -0,0 +1,503 @@ +import { getFixturesFactory } from 'api/utils/fixturesFactory'; +import { PropertySchema } from 'shared/types/commonTypes'; +import { EntitySchema } from 'shared/types/entityType'; +import { IXSuggestionType } from 'shared/types/suggestionType'; +import { formatSuggestion } from '../suggestionFormatting'; +import { InternalIXResultsMessage } from '../InformationExtraction'; + +const fixtureFactory = getFixturesFactory(); + +const successMessage: InternalIXResultsMessage = { + tenant: 'tenant', + task: 'suggestions', + params: { + id: fixtureFactory.id('extractor_id'), + }, + data_url: 'data_url', + file_url: 'file_url', + success: true, +}; + +const properties: Record = { + text: fixtureFactory.property('text_property', 'text'), + numeric: fixtureFactory.property('numeric_property', 'numeric'), + date: fixtureFactory.property('date_property', 'date'), + select: fixtureFactory.property('select_property', 'select'), + multiselect: fixtureFactory.property('multiselect_property', 'multiselect'), +}; + +const entities: Record = { + title: fixtureFactory.entity('entity_id', 'entity_template', {}), + text: fixtureFactory.entity('entity_id', 'entity_template', { + text_property: [{ value: 'previous_value' }], + }), + numeric: fixtureFactory.entity('entity_id', 'entity_template', { + numeric_property: [{ value: 0 }], + }), + date: fixtureFactory.entity('entity_id', 'entity_template', { + date_property: [{ value: 0 }], + }), + select: fixtureFactory.entity('entity_id', 'entity_template', { + select_property: [{ value: 'A_id', label: 'A' }], + }), + multiselect: fixtureFactory.entity('entity_id', 'entity_template', { + multiselect_property: [ + { value: 'A_id', label: 'A' }, + { value: 'B_id', label: 'B' }, + ], + }), +}; + +const currentSuggestionBase = { + _id: fixtureFactory.id('suggestion_id'), + entityId: 'entity_id', + extractorId: fixtureFactory.id('extractor_id'), + entityTemplate: 'entity_template', + fileId: fixtureFactory.id('file_id'), + segment: 'previous_context', + language: 'en', + page: 100, + date: 1, + status: 'ready' as 'ready', + error: '', + selectionRectangles: [ + { + top: 13, + left: 13, + width: 13, + height: 13, + page: '100', + }, + ], +}; + +const currentSuggestions: Record = { + title: { + ...currentSuggestionBase, + propertyName: 'title', + suggestedValue: 'previous_value', + }, + text: { + ...currentSuggestionBase, + propertyName: 'text_property', + suggestedValue: 'previous_value', + }, + numeric: { + ...currentSuggestionBase, + propertyName: 'numeric_property', + suggestedValue: 0, + }, + date: { + ...currentSuggestionBase, + propertyName: 'date_property', + suggestedValue: 0, + }, + select: { + ...currentSuggestionBase, + propertyName: 'select_property', + suggestedValue: 'A_id', + }, + multiselect: { + ...currentSuggestionBase, + propertyName: 'multiselect_property', + suggestedValue: ['A_id', 'B_id'], + }, +}; + +const rawSuggestionBase = { + tenant: 'tenant', + id: 'extractor_id', + xml_file_name: 'file.xml', + segment_text: 'new context', +}; + +const suggestedDateTimeStamp = 1717743209000; +const suggestedDateText = new Date(suggestedDateTimeStamp).toISOString(); + +const validRawSuggestions = { + title: { + ...rawSuggestionBase, + text: 'recommended_value', + segments_boxes: [ + { + top: 0, + left: 0, + width: 0, + height: 0, + page_number: 1, + }, + ], + }, + text: { + ...rawSuggestionBase, + text: 'recommended_value', + segments_boxes: [ + { + top: 0, + left: 0, + width: 0, + height: 0, + page_number: 1, + }, + ], + }, + numeric: { + ...rawSuggestionBase, + text: '42', + segments_boxes: [ + { + top: 0, + left: 0, + width: 0, + height: 0, + page_number: 1, + }, + ], + }, + date: { + ...rawSuggestionBase, + text: suggestedDateText, + segments_boxes: [ + { + top: 0, + left: 0, + width: 0, + height: 0, + page_number: 1, + }, + ], + }, + select: { + ...rawSuggestionBase, + values: [{ id: 'B_id', label: 'B' }], + }, + multiselect: { + ...rawSuggestionBase, + values: [ + { id: 'C_id', label: 'C' }, + { id: 'D_id', label: 'D' }, + ], + }, +}; + +describe('formatSuggestion', () => { + it.each([ + { + case: 'missing properties', + property: properties.text, + rawSuggestion: { + ...validRawSuggestions.text, + tenant: undefined, + }, + currentSuggestion: currentSuggestions.text, + entity: entities.text, + expectedErrorMessage: ": must have required property 'tenant'", + }, + { + case: 'invalid tenant type', + property: properties.text, + rawSuggestion: { + ...validRawSuggestions.text, + tenant: 1, + }, + currentSuggestion: currentSuggestions.text, + entity: entities.text, + expectedErrorMessage: '/tenant: must be string', + }, + { + case: 'invalid id type', + property: properties.text, + rawSuggestion: { + ...validRawSuggestions.text, + id: 1, + }, + currentSuggestion: currentSuggestions.text, + entity: entities.text, + expectedErrorMessage: '/id: must be string', + }, + { + case: 'invalid xml_file_name type', + property: properties.text, + rawSuggestion: { + ...validRawSuggestions.text, + xml_file_name: 1, + }, + currentSuggestion: currentSuggestions.text, + entity: entities.text, + expectedErrorMessage: '/xml_file_name: must be string', + }, + { + case: 'invalid text type', + property: properties.text, + rawSuggestion: { + ...validRawSuggestions.text, + text: 1, + }, + currentSuggestion: currentSuggestions.text, + entity: entities.text, + expectedErrorMessage: '/text: must be string', + }, + { + case: 'invalid segment_text type', + property: properties.text, + rawSuggestion: { + ...validRawSuggestions.text, + segment_text: 1, + }, + currentSuggestion: currentSuggestions.text, + entity: entities.text, + expectedErrorMessage: '/segment_text: must be string', + }, + { + case: 'invalid segments_boxes subtype', + property: properties.text, + rawSuggestion: { + ...validRawSuggestions.text, + segments_boxes: [ + { + top: 0, + left: 0, + width: 0, + height: 0, + page_number: '1', + }, + ], + }, + currentSuggestion: currentSuggestions.text, + entity: entities.text, + expectedErrorMessage: '/segments_boxes/0/page_number: must be number', + }, + { + case: 'invalid select values type', + property: properties.select, + rawSuggestion: { + ...validRawSuggestions.select, + values: 1, + }, + currentSuggestion: currentSuggestions.select, + entity: entities.select, + expectedErrorMessage: '/values: must be array', + }, + { + case: 'invalid select values subtype', + property: properties.select, + rawSuggestion: { + ...validRawSuggestions.select, + values: [{ id: 1, label: 'value_label' }], + }, + currentSuggestion: currentSuggestions.select, + entity: entities.select, + expectedErrorMessage: '/values/0/id: must be string', + }, + { + case: 'invalid select values length', + property: properties.select, + rawSuggestion: { + ...validRawSuggestions.select, + values: [ + { id: 'B_id', label: 'B' }, + { id: 'C_id', label: 'C' }, + ], + }, + currentSuggestion: currentSuggestions.select, + entity: entities.select, + expectedErrorMessage: 'Select suggestions must have one or zero values.', + }, + { + case: 'invalid multiselect values type', + property: properties.multiselect, + rawSuggestion: { + ...validRawSuggestions.multiselect, + values: 1, + }, + currentSuggestion: currentSuggestions.multiselect, + entity: entities.multiselect, + expectedErrorMessage: '/values: must be array', + }, + { + case: 'invalid multiselect values subtype', + property: properties.multiselect, + rawSuggestion: { + ...validRawSuggestions.multiselect, + values: [{ id: 1, label: 'value_label' }], + }, + currentSuggestion: currentSuggestions.multiselect, + entity: entities.multiselect, + expectedErrorMessage: '/values/0/id: must be string', + }, + ])( + 'should throw error if $case', + async ({ property, rawSuggestion, currentSuggestion, entity, expectedErrorMessage }) => { + const cb = async () => + formatSuggestion( + property, + // @ts-expect-error + rawSuggestion, + currentSuggestion, + entity, + successMessage + ); + await expect(cb).rejects.toThrow(expectedErrorMessage); + } + ); + + it('should allow extra properties', async () => { + const property = properties.text; + const rawSuggestion = { + ...validRawSuggestions.text, + extra: 'extra', + }; + const result = await formatSuggestion( + property, + rawSuggestion, + currentSuggestions.text, + entities.text, + successMessage + ); + expect(result).toEqual({ + ...currentSuggestions.text, + date: expect.any(Number), + suggestedValue: 'recommended_value', + segment: 'new context', + selectionRectangles: [ + { + top: 0, + left: 0, + width: 0, + height: 0, + page: '1', + }, + ], + }); + }); + + it.each([ + { + case: 'valid title suggestions', + property: { name: 'title' as 'title', type: 'title' as 'title' }, + rawSuggestion: validRawSuggestions.title, + currentSuggestion: currentSuggestions.title, + entity: entities.title, + expectedResult: { + ...currentSuggestions.title, + date: expect.any(Number), + suggestedValue: 'recommended_value', + segment: 'new context', + selectionRectangles: [ + { + top: 0, + left: 0, + width: 0, + height: 0, + page: '1', + }, + ], + }, + }, + { + case: 'valid text suggestions', + property: properties.text, + rawSuggestion: validRawSuggestions.text, + currentSuggestion: currentSuggestions.text, + entity: entities.text, + expectedResult: { + ...currentSuggestions.text, + date: expect.any(Number), + suggestedValue: 'recommended_value', + segment: 'new context', + selectionRectangles: [ + { + top: 0, + left: 0, + width: 0, + height: 0, + page: '1', + }, + ], + }, + }, + { + case: 'valid numeric suggestions', + property: properties.numeric, + rawSuggestion: validRawSuggestions.numeric, + currentSuggestion: currentSuggestions.numeric, + entity: entities.numeric, + expectedResult: { + ...currentSuggestions.numeric, + date: expect.any(Number), + suggestedValue: 42, + segment: 'new context', + selectionRectangles: [ + { + top: 0, + left: 0, + width: 0, + height: 0, + page: '1', + }, + ], + }, + }, + { + case: 'valid date suggestions', + property: properties.date, + rawSuggestion: validRawSuggestions.date, + currentSuggestion: currentSuggestions.date, + entity: entities.date, + expectedResult: { + ...currentSuggestions.date, + date: expect.any(Number), + suggestedValue: suggestedDateTimeStamp / 1000, + suggestedText: suggestedDateText, + segment: 'new context', + selectionRectangles: [ + { + top: 0, + left: 0, + width: 0, + height: 0, + page: '1', + }, + ], + }, + }, + { + case: 'valid select suggestions', + property: properties.select, + rawSuggestion: validRawSuggestions.select, + currentSuggestion: currentSuggestions.select, + entity: entities.select, + expectedResult: { + ...currentSuggestions.select, + date: expect.any(Number), + suggestedValue: 'B_id', + segment: 'new context', + }, + }, + { + case: 'valid multiselect suggestions', + property: properties.multiselect, + rawSuggestion: validRawSuggestions.multiselect, + currentSuggestion: currentSuggestions.multiselect, + entity: entities.multiselect, + expectedResult: { + ...currentSuggestions.multiselect, + date: expect.any(Number), + suggestedValue: ['C_id', 'D_id'], + segment: 'new context', + }, + }, + ])( + 'should return formatted suggestion for $case', + async ({ property, rawSuggestion, currentSuggestion, entity, expectedResult }) => { + const result = await formatSuggestion( + property, + rawSuggestion, + currentSuggestion, + entity, + successMessage + ); + expect(result).toEqual(expectedResult); + } + ); +}); diff --git a/app/api/services/informationextraction/suggestionFormatting.ts b/app/api/services/informationextraction/suggestionFormatting.ts index 919604c768..d69118d449 100644 --- a/app/api/services/informationextraction/suggestionFormatting.ts +++ b/app/api/services/informationextraction/suggestionFormatting.ts @@ -1,73 +1,157 @@ -import { stringToTypeOfProperty } from 'shared/stringToTypeOfProperty'; +import Ajv from 'ajv'; + +import date from 'api/utils/date'; import { PropertySchema } from 'shared/types/commonTypes'; import { EntitySchema } from 'shared/types/entityType'; -import { IXSuggestionType } from 'shared/types/suggestionType'; +import { + CommonSuggestion, + IXSuggestionType, + TextSelectionSuggestion, + ValuesSelectionSuggestion, +} from 'shared/types/suggestionType'; +import { + TextSelectionSuggestionSchema, + ValuesSelectionSuggestionSchema, +} from 'shared/types/suggestionSchema'; +import { syncWrapValidator } from 'shared/tsUtils'; import { InternalIXResultsMessage } from './InformationExtraction'; +import { AllowedPropertyTypes, checkTypeIsAllowed } from './ixextractors'; -interface CommonSuggestion { - tenant: string; - id: string; - xml_file_name: string; -} +type RawSuggestion = TextSelectionSuggestion | ValuesSelectionSuggestion; -interface TextSelectionSuggestion extends CommonSuggestion { - text: string; - segment_text: string; - segments_boxes: { - top: number; - left: number; - width: number; - height: number; - page_number: number; - }[]; +class RawSuggestionValidationError extends Error { + constructor(message: string) { + super(message); + this.name = 'RawSuggestionValidationError'; + } } -interface ValuesSelectionSuggestion extends CommonSuggestion { - values: { id: string; label: string }[]; - segment_text: string; -} +type TitleAsProperty = { + name: 'title'; + type: 'title'; +}; -type RawSuggestion = TextSelectionSuggestion | ValuesSelectionSuggestion; +const createAjvValidator = (schema: any) => { + const ajv = new Ajv({ allErrors: true }); + ajv.addVocabulary(['tsType']); + return syncWrapValidator(ajv.compile(schema)); +}; + +const textSelectionAjv = createAjvValidator(TextSelectionSuggestionSchema); +const valuesSelectionAjv = createAjvValidator(ValuesSelectionSuggestionSchema); + +const textSelectionValidator = ( + suggestion: RawSuggestion +): suggestion is TextSelectionSuggestion => { + textSelectionAjv(suggestion); + return true; +}; + +const valuesSelectionValidator = ( + suggestion: RawSuggestion +): suggestion is ValuesSelectionSuggestion => { + valuesSelectionAjv(suggestion); + return true; +}; const VALIDATORS = { - text: (suggestion: RawSuggestion): suggestion is TextSelectionSuggestion => 'text' in suggestion, - select: (suggestion: RawSuggestion): suggestion is ValuesSelectionSuggestion => - 'values' in suggestion && (suggestion.values.length === 1 || suggestion.values.length === 0), - multiselect: (suggestion: RawSuggestion): suggestion is ValuesSelectionSuggestion => - 'values' in suggestion, + title: textSelectionValidator, + text: textSelectionValidator, + numeric: textSelectionValidator, + date: textSelectionValidator, + select: (suggestion: RawSuggestion): suggestion is ValuesSelectionSuggestion => { + if (!valuesSelectionValidator(suggestion)) { + throw new RawSuggestionValidationError('Select suggestion is not valid.'); + } + + if (!('values' in suggestion) || suggestion.values.length > 1) { + throw new RawSuggestionValidationError('Select suggestions must have one or zero values.'); + } + + return true; + }, + multiselect: valuesSelectionValidator, +}; + +const simpleSuggestion = ( + suggestedValue: string | number | null, + rawSuggestion: TextSelectionSuggestion +) => ({ + suggestedValue, + segment: rawSuggestion.segment_text, + selectionRectangles: rawSuggestion.segments_boxes.map((box: any) => { + const rect = { ...box, page: box.page_number.toString() }; + delete rect.page_number; + return rect; + }), +}); + +const textFormatter = ( + rawSuggestion: RawSuggestion, + _currentSuggestion: IXSuggestionType, + _entity: EntitySchema +) => { + if (!VALIDATORS.text(rawSuggestion)) { + throw new Error('Text suggestion is not valid.'); + } + + const rawText = rawSuggestion.text; + const suggestedValue = rawText.trim(); + + const suggestion: Partial = simpleSuggestion(suggestedValue, rawSuggestion); + + return suggestion; }; -const FORMATTERS = { - text: ( +const FORMATTERS: Record< + AllowedPropertyTypes, + ( rawSuggestion: RawSuggestion, - property: PropertySchema | undefined, currentSuggestion: IXSuggestionType, entity: EntitySchema + ) => Partial +> = { + title: textFormatter, + text: textFormatter, + numeric: ( + rawSuggestion: RawSuggestion, + _currentSuggestion: IXSuggestionType, + _entity: EntitySchema ) => { - if (!VALIDATORS.text(rawSuggestion)) { - throw new Error('Text suggestion is not valid.'); + if (!VALIDATORS.numeric(rawSuggestion)) { + throw new Error('Numeric suggestion is not valid.'); } - const suggestedValue = stringToTypeOfProperty( - rawSuggestion.text, - property?.type, + const suggestedValue = parseFloat(rawSuggestion.text.trim()) || null; + const suggestion: Partial = simpleSuggestion(suggestedValue, rawSuggestion); + + return suggestion; + }, + date: ( + rawSuggestion: RawSuggestion, + currentSuggestion: IXSuggestionType, + entity: EntitySchema + ) => { + if (!VALIDATORS.date(rawSuggestion)) { + throw new Error('Date suggestion is not valid.'); + } + + const suggestedValue = date.dateToSeconds( + rawSuggestion.text.trim(), currentSuggestion?.language || entity.language ); - const suggestion: Partial = { - suggestedValue, - ...(property?.type === 'date' ? { suggestedText: rawSuggestion.text } : {}), - segment: rawSuggestion.segment_text, - selectionRectangles: rawSuggestion.segments_boxes.map((box: any) => { - const rect = { ...box, page: box.page_number.toString() }; - delete rect.page_number; - return rect; - }), + ...simpleSuggestion(suggestedValue, rawSuggestion), + suggestedText: rawSuggestion.text, }; return suggestion; }, - select: (rawSuggestion: RawSuggestion) => { + select: ( + rawSuggestion: RawSuggestion, + _currentSuggestion: IXSuggestionType, + _entity: EntitySchema + ) => { if (!VALIDATORS.select(rawSuggestion)) { throw new Error('Select suggestion is not valid.'); } @@ -81,7 +165,11 @@ const FORMATTERS = { return suggestion; }, - multiselect: (rawSuggestion: RawSuggestion) => { + multiselect: ( + rawSuggestion: RawSuggestion, + _currentSuggestion: IXSuggestionType, + _entity: EntitySchema + ) => { if (!VALIDATORS.multiselect(rawSuggestion)) { throw new Error('Multiselect suggestion is not valid.'); } @@ -97,30 +185,29 @@ const FORMATTERS = { }, }; -const DEFAULTFORMATTER = FORMATTERS.text; +type PropertyOrTitle = PropertySchema | TitleAsProperty | undefined; const formatRawSuggestion = ( rawSuggestion: RawSuggestion, - property: PropertySchema | undefined, + property: PropertyOrTitle, currentSuggestion: IXSuggestionType, entity: EntitySchema ) => { - const formatter = - // @ts-ignore - (property?.type || '') in FORMATTERS ? FORMATTERS[property.type] : DEFAULTFORMATTER; - return formatter(rawSuggestion, property, currentSuggestion, entity); + const type = checkTypeIsAllowed(property?.type || ''); + const formatter = FORMATTERS[type]; + return formatter(rawSuggestion, currentSuggestion, entity); }; const readMessageSuccess = (message: InternalIXResultsMessage) => message.success ? {} : { - status: 'failed', + status: 'failed' as 'failed', error: message.error_message ? message.error_message : 'Unknown error', }; const formatSuggestion = async ( - property: PropertySchema | undefined, + property: PropertyOrTitle, rawSuggestion: RawSuggestion, currentSuggestion: IXSuggestionType, entity: EntitySchema, diff --git a/app/shared/specs/stringToTypeOfProperty.spec.ts b/app/shared/specs/stringToTypeOfProperty.spec.ts deleted file mode 100644 index 0bb0a68978..0000000000 --- a/app/shared/specs/stringToTypeOfProperty.spec.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { stringToTypeOfProperty } from 'shared/stringToTypeOfProperty'; - -it('should convert a string to a date timestamp', () => { - expect(stringToTypeOfProperty('January 1, 1999', 'date')).toBe(915148800); -}); - -it('should convert a string to a number timestamp', () => { - expect(stringToTypeOfProperty('1234', 'numeric')).toBe(1234); -}); - -it('should keep the string if not date or numeric', () => { - expect(stringToTypeOfProperty('some string', 'select')).toBe('some string'); -}); - -it('should keep null if the input is null', () => { - expect(stringToTypeOfProperty(null, 'date')).toBe(null); -}); diff --git a/app/shared/stringToTypeOfProperty.ts b/app/shared/stringToTypeOfProperty.ts deleted file mode 100644 index a9c956b45f..0000000000 --- a/app/shared/stringToTypeOfProperty.ts +++ /dev/null @@ -1,22 +0,0 @@ -import date from 'api/utils/date'; -import { PropertySchema } from './types/commonTypes'; - -const stringToTypeOfProperty = ( - text: string | null, - propertyType: PropertySchema['type'] | undefined, - language?: string -) => { - if (!text) return text; - - const trimmedText = text.trim(); - switch (propertyType) { - case 'numeric': - return parseFloat(trimmedText) || null; - case 'date': - return date.dateToSeconds(trimmedText, language); - default: - return trimmedText; - } -}; - -export { stringToTypeOfProperty }; diff --git a/app/shared/tsUtils.ts b/app/shared/tsUtils.ts index e4ef219a7a..0b4417980e 100644 --- a/app/shared/tsUtils.ts +++ b/app/shared/tsUtils.ts @@ -31,6 +31,26 @@ export function wrapValidator(validator: any) { }; } +export function syncWrapValidator(validator: any) { + return (value: any) => { + const valid = validator(value); + + if (!valid) { + const { errors } = validator; + const e = new ValidationError(errors); + e.message = errors + .map( + ({ instancePath, message }: { instancePath: string; message: string }) => + `${instancePath}: ${message}` + ) + .join('/n'); + throw e; + } + + return valid; + }; +} + export async function sleep(ms: number) { return new Promise(resolve => { setTimeout(resolve, ms); diff --git a/app/shared/types/suggestionSchema.ts b/app/shared/types/suggestionSchema.ts index 182ac0358a..f98d29ace1 100644 --- a/app/shared/types/suggestionSchema.ts +++ b/app/shared/types/suggestionSchema.ts @@ -7,6 +7,69 @@ import { propertyTypes } from 'shared/propertyTypes'; export const emitSchemaTypes = true; +const commonSuggestionMessageProperties = { + tenant: { type: 'string', minLength: 1 }, + id: { type: 'string', minLength: 1 }, + xml_file_name: { type: 'string', minLength: 1 }, +}; + +export const CommonSuggestionSchema = { + type: 'object', + title: 'CommonSuggestion', + properties: { + ...commonSuggestionMessageProperties, + }, + required: ['tenant', 'id', 'xml_file_name', 'segment_text'], +}; + +export const TextSelectionSuggestionSchema = { + type: 'object', + title: 'TextSelectionSuggestion', + properties: { + ...commonSuggestionMessageProperties, + text: { type: 'string' }, + segment_text: { type: 'string' }, + segments_boxes: { + type: 'array', + items: { + type: 'object', + additionalProperties: false, + properties: { + top: { type: 'number' }, + left: { type: 'number' }, + width: { type: 'number' }, + height: { type: 'number' }, + page_number: { type: 'number' }, + }, + required: ['top', 'left', 'width', 'height', 'page_number'], + }, + }, + }, + required: ['tenant', 'id', 'xml_file_name', 'text', 'segment_text', 'segments_boxes'], +}; + +export const ValuesSelectionSuggestionSchema = { + type: 'object', + title: 'ValuesSelectionSuggestion', + properties: { + ...commonSuggestionMessageProperties, + values: { + type: 'array', + items: { + type: 'object', + additionalProperties: false, + properties: { + id: { type: 'string', minLength: 1 }, + label: { type: 'string', minLength: 1 }, + }, + required: ['id', 'label'], + }, + }, + segment_text: { type: 'string' }, + }, + required: ['tenant', 'id', 'xml_file_name', 'values', 'segment_text'], +}; + export const IXSuggestionStateSchema = { type: 'object', additionalProperties: false, diff --git a/app/shared/types/suggestionType.d.ts b/app/shared/types/suggestionType.d.ts index 32cf41df20..a21cdf9ee9 100644 --- a/app/shared/types/suggestionType.d.ts +++ b/app/shared/types/suggestionType.d.ts @@ -7,6 +7,13 @@ import { SelectionRectanglesSchema, } from 'shared/types/commonTypes'; +export interface CommonSuggestion { + tenant: string; + id: string; + xml_file_name: string; + [k: string]: unknown | undefined; +} + export interface EntitySuggestionType { _id?: ObjectIdSchema; entityId: string; @@ -121,3 +128,31 @@ export interface IXSuggestionsFilter { extractorId: ObjectIdSchema; customFilter?: SuggestionCustomFilter; } + +export interface TextSelectionSuggestion { + tenant: string; + id: string; + xml_file_name: string; + text: string; + segment_text: string; + segments_boxes: { + top: number; + left: number; + width: number; + height: number; + page_number: number; + }[]; + [k: string]: unknown | undefined; +} + +export interface ValuesSelectionSuggestion { + tenant: string; + id: string; + xml_file_name: string; + values: { + id: string; + label: string; + }[]; + segment_text: string; + [k: string]: unknown | undefined; +} From e9ccc7b4e0f955feda09dfe6b745de75e4c748b1 Mon Sep 17 00:00:00 2001 From: Mercy Date: Thu, 13 Jun 2024 10:55:37 -0500 Subject: [PATCH 3/3] [Stabilization fix] 6713 - Activity Log fixes (#6881) * [Stabilization-fix] 6713-activity-log-filter-fixes (#6866) * supports method as filter * extends query builder * Moves search to button * limit url * updates tests * adds additional activity log entry * updates types * move filters to side panel * supports method as filter * uses real dates on tests * restore explicit class names * filter by method * fixes withOverlay effect * label as translation * fixes from review * method as multiselect * cleaning dates from filters * update tests * refactor of component * updates tests * minor refactor * adds translation entry * limit results by search matches * listen to multiselect value changes * refactor to load form values in filters * minor refactor * updates snapshots * review fixes * using existent property daterange translation key * fixes to review & increase coverage * test to cover prop update * pagination fixes * fix for update of entity with multipart body * test for escapeEspecialChars * increases version * ensures side panel opening * temporal control to update multiselect * update story * removes updatable property * updates of multiselect values on filters * fixes date filter building * filters by date fixes * date range cleaning fixes * unit tests fixes * fix method param resolution * adds test cases * adds e2e coverage * increase version * waits for confirmation message * fixes expected title * updates expected entries --- app/api/activitylog/activityLogBuilder.ts | 16 +- app/api/activitylog/activityLogFilter.ts | 228 +++++++++++ app/api/activitylog/activitylog.js | 70 +--- app/api/activitylog/activitylogParser.ts | 6 +- .../specs/activityLogFilter.spec.ts | 340 +++++++++++++++++ app/api/activitylog/specs/activitylog.spec.js | 218 ++++++++--- .../specs/activitylogParser.spec.js | 18 +- app/api/activitylog/specs/fixtures.js | 25 +- app/react/App/styles/globals.css | 60 ++- app/react/Routes.tsx | 4 +- .../Forms/DatePicker/DatePickerComponent.tsx | 20 +- .../DatePicker/DateRangePickerComponent.tsx | 64 +++- app/react/V2/Components/Forms/MultiSelect.tsx | 13 +- .../Components/Forms/specs/MultiSelect.cy.tsx | 7 + app/react/V2/Components/UI/Pill.tsx | 10 +- .../UI/specs/DateRangePicker.cy.tsx | 10 +- .../Settings/ActivityLog/ActivityLog.tsx | 359 ++++-------------- .../Settings/ActivityLog/ActivityLogLoader.ts | 186 +++++++++ .../components/ActivityLogSidePanel.tsx | 100 ++--- .../components/FiltersSidePanel.tsx | 183 +++++++++ .../ActivityLog/components/TableElements.tsx | 2 +- .../V2/Routes/Settings/ActivityLog/index.ts | 2 + .../Filters/components/FiltersSidepanel.tsx | 67 ++-- app/react/utils/routeHelpers.ts | 10 +- .../data_utils/specs/stringUtils.spec.ts | 12 +- app/shared/data_utils/stringUtils.ts | 4 +- app/shared/types/activityLogApiSchemas.ts | 4 + app/shared/types/activityLogApiTypes.d.ts | 4 + ... should open the detail of an entry #0.png | Bin 71677 -> 0 bytes .../__snapshots__/activitylog.cy.ts.snap | 148 +++++++- cypress/e2e/settings/activitylog.cy.ts | 135 ++++++- cypress/e2e/settings/filters.cy.ts | 1 + package.json | 2 +- .../dump/uwazi_development/activitylogs.bson | Bin 633671 -> 1988 bytes .../activitylogs.metadata.json | 2 +- 35 files changed, 1730 insertions(+), 600 deletions(-) create mode 100644 app/api/activitylog/activityLogFilter.ts create mode 100644 app/api/activitylog/specs/activityLogFilter.spec.ts create mode 100644 app/react/V2/Routes/Settings/ActivityLog/ActivityLogLoader.ts create mode 100644 app/react/V2/Routes/Settings/ActivityLog/components/FiltersSidePanel.tsx create mode 100644 app/react/V2/Routes/Settings/ActivityLog/index.ts delete mode 100644 cypress/e2e/settings/__image_snapshots__/Activity log should open the detail of an entry #0.png diff --git a/app/api/activitylog/activityLogBuilder.ts b/app/api/activitylog/activityLogBuilder.ts index e23f71794e..39bb217847 100644 --- a/app/api/activitylog/activityLogBuilder.ts +++ b/app/api/activitylog/activityLogBuilder.ts @@ -1,4 +1,4 @@ -export enum Methods { +enum Methods { Create = 'CREATE', Update = 'UPDATE', Delete = 'DELETE', @@ -12,7 +12,7 @@ const buildActivityLogEntry = (builder: ActivityLogBuilder) => ({ ...(builder.extra && { extra: builder.extra }), }); -export interface EntryValue { +interface EntryValue { idField?: string; nameField?: string; id?: any; @@ -23,12 +23,12 @@ export interface EntryValue { desc: string; } -export interface LogActivity { +interface LogActivity { name?: string; [k: string]: any | undefined; } -export class ActivityLogBuilder { +class ActivityLogBuilder { description: string; action: Methods; @@ -89,7 +89,8 @@ const changeToUpdate = (entryValue: EntryValue): EntryValue => { function checkForUpdate(body: any, entryValue: EntryValue) { const content = JSON.parse(body); - const id = entryValue.idField ? content[entryValue.idField] : null; + const json = content.entity ? JSON.parse(content.entity) : content; + const id = entryValue.idField ? json[entryValue.idField] : null; let activityInput = { ...entryValue }; if (id && entryValue.method !== Methods.Delete) { activityInput = changeToUpdate(entryValue); @@ -103,7 +104,7 @@ const getActivityInput = (entryValue: EntryValue, body: any) => { return idPost ? checkForUpdate(body, entryValue) : entryValue; }; -export const buildActivityEntry = async (entryValue: EntryValue, data: any) => { +const buildActivityEntry = async (entryValue: EntryValue, data: any) => { const body = data.body && data.body !== '{}' ? data.body : data.query || '{}'; const activityInput = getActivityInput(entryValue, body); const activityEntryBuilder = new ActivityLogBuilder(JSON.parse(body), activityInput); @@ -112,3 +113,6 @@ export const buildActivityEntry = async (entryValue: EntryValue, data: any) => { activityEntryBuilder.makeExtra(); return activityEntryBuilder.build(); }; + +export type { ActivityLogBuilder, EntryValue, LogActivity }; +export { Methods, buildActivityEntry }; diff --git a/app/api/activitylog/activityLogFilter.ts b/app/api/activitylog/activityLogFilter.ts new file mode 100644 index 0000000000..217a479c93 --- /dev/null +++ b/app/api/activitylog/activityLogFilter.ts @@ -0,0 +1,228 @@ +import { isEmpty } from 'lodash'; +import { FilterQuery } from 'mongoose'; +import moment from 'moment'; +import { ActivityLogGetRequest } from 'shared/types/activityLogApiTypes'; +import { ActivityLogEntryType } from 'shared/types/activityLogEntryType'; +import { escapeEspecialChars } from 'shared/data_utils/stringUtils'; +import { ParsedActions } from './activitylogParser'; +import { EntryValue } from './activityLogBuilder'; + +type ActivityLogQuery = Required['query']; +type ActivityLogQueryTime = Required['time']; +const prepareToFromRanges = (sanitizedTime: ActivityLogQueryTime) => { + const fromDate = sanitizedTime.from && new Date(sanitizedTime.from); + const toDate = sanitizedTime.to && moment(new Date(sanitizedTime.to)).add(1, 'day').toDate(); + return { + ...(fromDate && { $gte: fromDate.getTime() }), + ...(toDate && { $lt: toDate.getTime() }), + }; +}; +const parsedActionsEntries = Object.entries(ParsedActions); +const queryURL = (matchedEntries: [string, EntryValue][]) => + matchedEntries.map(([key]) => { + const entries = key.split(/\/(.*)/s); + const condition = { + $and: [ + { url: { $regex: `^\\/${escapeEspecialChars(entries[1])}$` } }, + { method: entries[0] }, + ], + }; + return condition; + }); +const andCondition = (method: string, regex: string, property: string = 'body') => ({ + $and: [{ method }, { [property]: { $regex: regex } }], +}); +const bodyCondition = (methods: string[]) => { + const orContent: FilterQuery[] = []; + methods.forEach(method => { + switch (method) { + case 'CREATE': + orContent.push({ + $and: [ + andCondition('POST', '^(?!{"_id").*'), + andCondition('POST', '^(?!{"entity":"{\\\\"_id).*'), + ], + }); + break; + case 'UPDATE': + orContent.push(andCondition('POST', '^({"_id").*')); + orContent.push(andCondition('POST', '^({"entity":"{\\\\"_id).*')); + orContent.push(andCondition('PUT', '^({"_id").*')); + orContent.push(andCondition('PUT', '^({"entity":"{\\\\"_id).*')); + break; + case 'DELETE': + orContent.push(andCondition('DELETE', '^({"_id").*')); + orContent.push(andCondition('DELETE', '^({"_id").*', 'query')); + orContent.push(andCondition('DELETE', '^({"sharedId").*', 'query')); + break; + default: + orContent.push({ method }); + break; + } + }); + return orContent; +}; +const sanitizeTime = (time: ActivityLogQueryTime) => (memo: {}, k: string) => + time[k] !== null ? Object.assign(memo, { [k]: time[k] }) : memo; +const equivalentHttpMethod = (method: string): string => + ['CREATE', 'UPDATE'].includes(method.toUpperCase()) ? 'POST' : method.toUpperCase(); +const matchWithParsedEntry = ( + key: string, + queryMethods: string[], + value: EntryValue, + methods: string[] +) => + key.toUpperCase().match(`(${queryMethods.join('|')}).*`) && + ((value.method || '').toUpperCase().match(`(${methods.join('|')}).*`) || + value.desc.toUpperCase().match(`(${methods.join('|')}).*`)); +const reduceUniqueCondition: ( + condition: FilterQuery +) => FilterQuery = (condition: FilterQuery) => { + const keys = Object.keys(condition); + return keys.reduce((memo, key) => { + if (['$and', '$or'].includes(key) && condition[key]?.length === 1) { + const reducedCondition = reduceUniqueCondition(condition[key][0]); + return { ...memo, ...reducedCondition }; + } + return { ...memo, [key]: condition[key] }; + }, {}); +}; +class ActivityLogFilter { + andQuery: FilterQuery[] = []; + + searchQuery: FilterQuery[] = []; + + query: ActivityLogQuery; + + constructor(requestQuery: ActivityLogQuery) { + const methodFilter = ((requestQuery || {}).method || []).map(method => method.toUpperCase()); + this.query = { ...requestQuery, method: methodFilter }; + } + + timeQuery() { + const { time = {}, before = -1 } = this.query || {}; + const sanitizedTime: ActivityLogQueryTime = Object.keys(time).reduce(sanitizeTime(time), {}); + if (before === -1 && isEmpty(sanitizedTime)) { + return; + } + const timeFilter = { + ...(!isEmpty(sanitizedTime) ? { time: prepareToFromRanges(sanitizedTime) } : {}), + ...(before !== -1 ? { time: { $lt: before } } : {}), + }; + this.andQuery.push(timeFilter); + } + + setRequestFilter(property: 'url' | 'query' | 'body' | 'params', exact = false) { + const filterValue = (this.query || {})[property]; + if (filterValue !== undefined) { + const exp = escapeEspecialChars(filterValue); + this.andQuery.push({ [property]: { $regex: exact ? `^${exp}$` : exp } }); + } + } + + prepareRegexpQueries = () => { + this.setRequestFilter('url', true); + this.setRequestFilter('query'); + this.setRequestFilter('body'); + this.setRequestFilter('params'); + }; + + searchFilter() { + const { search } = this.query || {}; + if (search !== undefined) { + const regex = { $regex: `.*${escapeEspecialChars(search)}.*`, $options: 'si' }; + this.searchQuery.push({ + $or: [ + { + method: { + $regex: `${escapeEspecialChars(search.toUpperCase().replace('CREATE', 'POST'))}`, + }, + }, + { url: { $regex: `^${escapeEspecialChars(search)}$` } }, + { query: regex }, + { body: regex }, + { params: regex }, + ], + }); + } + } + + methodFilter() { + const { method: methods = [] } = this.query || {}; + if (methods.length > 0) { + const queryMethods = methods.map(equivalentHttpMethod); + const matchedEntries = parsedActionsEntries.filter(([key, value]) => + matchWithParsedEntry(key, queryMethods, value, methods) + ); + const bodyTerm = bodyCondition(methods); + if (bodyTerm.length > 0) { + this.andQuery.push({ $or: bodyTerm }); + } + if (matchedEntries.length > 0) { + const orUrlItems = queryURL(matchedEntries); + this.andQuery.push({ $or: orUrlItems }); + } + } + } + + openSearchFilter() { + const { search } = this.query || {}; + if (search === undefined) { + return; + } + const matchedURLs = parsedActionsEntries.filter(([_key, value]) => + value.desc.toLowerCase().includes(search.toLocaleLowerCase() || '') + ); + const orUrlItems = queryURL(matchedURLs); + if (matchedURLs.length > 0) { + this.searchQuery.push({ $or: orUrlItems }); + } else { + this.searchFilter(); + } + if (this.searchQuery.length > 0) { + this.andQuery.push({ $or: this.searchQuery }); + } + } + + findFilter() { + const { find } = this.query || {}; + if (find) { + const regex = { $regex: `.*${escapeEspecialChars(find)}.*`, $options: 'si' }; + this.andQuery.push({ + $or: [ + { method: regex }, + { url: regex }, + { query: regex }, + { body: regex }, + { params: regex }, + ], + }); + } + } + + userFilter() { + const { username } = this.query || {}; + if (username) { + const orUser: FilterQuery[] = [ + { username: { $regex: `.*${escapeEspecialChars(username)}.*`, $options: 'si' } }, + ]; + if (username === 'anonymous') { + orUser.push({ username: null }); + } + this.andQuery.push({ $or: orUser }); + } + } + + prepareQuery() { + this.prepareRegexpQueries(); + this.methodFilter(); + this.openSearchFilter(); + this.findFilter(); + this.userFilter(); + this.timeQuery(); + const rootQuery = !isEmpty(this.andQuery) ? reduceUniqueCondition({ $and: this.andQuery }) : {}; + return rootQuery; + } +} +export type { ActivityLogQueryTime }; +export { ActivityLogFilter, prepareToFromRanges, bodyCondition }; diff --git a/app/api/activitylog/activitylog.js b/app/api/activitylog/activitylog.js index c93408b862..fc1e1ef373 100644 --- a/app/api/activitylog/activitylog.js +++ b/app/api/activitylog/activitylog.js @@ -1,6 +1,7 @@ import { sortingParams } from 'shared/types/activityLogApiSchemas'; import model from './activitylogModel'; import { getSemanticData } from './activitylogParser'; +import { ActivityLogFilter } from './activityLogFilter'; const sortingParamsAsSet = new Set(sortingParams); @@ -12,68 +13,6 @@ const validateSortingParam = param => { } }; -const prepareRegexpQueries = query => { - const result = {}; - - if (query.url) { - result.url = new RegExp(query.url); - } - if (query.query) { - result.query = new RegExp(query.query); - } - if (query.body) { - result.body = new RegExp(query.body); - } - if (query.params) { - result.params = new RegExp(query.params); - } - - return result; -}; - -const prepareQuery = query => { - if (!query.find) { - return prepareRegexpQueries(query); - } - const term = new RegExp(query.find); - return { - $or: [{ method: term }, { url: term }, { query: term }, { body: term }, { params: term }], - }; -}; - -const prepareToFromRanges = sanitizedTime => { - const time = {}; - - if (sanitizedTime.from) { - time.$gte = parseInt(sanitizedTime.from, 10) * 1000; - } - - if (sanitizedTime.to) { - time.$lte = parseInt(sanitizedTime.to, 10) * 1000; - } - - return time; -}; - -const timeQuery = ({ time = {}, before = null }) => { - const sanitizedTime = Object.keys(time).reduce( - (memo, k) => (time[k] !== null ? Object.assign(memo, { [k]: time[k] }) : memo), - {} - ); - - if (before === null && !Object.keys(sanitizedTime).length) { - return {}; - } - - const result = { time: prepareToFromRanges(sanitizedTime) }; - - if (before !== null) { - result.time.$lt = parseInt(before, 10); - } - - return result; -}; - const getPagination = query => { const { page } = query; const limit = parseInt(query.limit || 15, 10); @@ -112,12 +51,7 @@ export default { isValidSortingParam, async get(query = {}) { - const mongoQuery = Object.assign(prepareQuery(query), timeQuery(query)); - - if (query.method && query.method.length) { - mongoQuery.method = { $in: query.method }; - } - + const mongoQuery = new ActivityLogFilter(query).prepareQuery(); if (query.username) { mongoQuery.username = query.username !== 'anonymous' ? query.username : { $in: [null, query.username] }; diff --git a/app/api/activitylog/activitylogParser.ts b/app/api/activitylog/activitylogParser.ts index f78b494982..7f35cb39c6 100644 --- a/app/api/activitylog/activitylogParser.ts +++ b/app/api/activitylog/activitylogParser.ts @@ -2,7 +2,7 @@ import * as helpers from 'api/activitylog/helpers'; import { nameFunc } from 'api/activitylog/helpers'; import { buildActivityEntry, Methods, EntryValue } from 'api/activitylog/activityLogBuilder'; -const entryValues: { [key: string]: EntryValue } = { +const ParsedActions: { [key: string]: EntryValue } = { 'POST/api/users': { desc: 'Updated user', method: Methods.Update, @@ -226,7 +226,7 @@ const getSemanticData = async (data: any) => { if (action === 'MIGRATE') { return helpers.migrationLog(data); } - const entryValue = entryValues[action] || { + const entryValue = ParsedActions[action] || { desc: '', extra: () => `${data.method}: ${data.url}`, method: 'RAW', @@ -245,4 +245,4 @@ const getSemanticData = async (data: any) => { return { ...activityEntry }; }; -export { getSemanticData }; +export { getSemanticData, ParsedActions }; diff --git a/app/api/activitylog/specs/activityLogFilter.spec.ts b/app/api/activitylog/specs/activityLogFilter.spec.ts new file mode 100644 index 0000000000..a4e0ec2eca --- /dev/null +++ b/app/api/activitylog/specs/activityLogFilter.spec.ts @@ -0,0 +1,340 @@ +import { + ActivityLogFilter, + ActivityLogQueryTime, + bodyCondition, + prepareToFromRanges, +} from '../activityLogFilter'; + +describe('activityLogFilter', () => { + describe('prepareToFromRanges', () => { + it.each` + from | to | expected + ${undefined} | ${1000011600000} | ${{ $lt: 1000098000000 }} + ${1000011600000} | ${1000011600000} | ${{ $gte: 1000011600000, $lt: 1000098000000 }} + ${1000011600000} | ${undefined} | ${{ $gte: 1000011600000 }} + ${1000011600000} | ${1500008400000} | ${{ $gte: 1000011600000, $lt: 1500094800000 }} + `("should create a date condition from: '$from' to: '$to' ", ({ from, to, expected }) => { + const timeCondition: ActivityLogQueryTime = { from, to }; + const result = prepareToFromRanges(timeCondition); + expect(result).toEqual(expected); + }); + }); + + describe('bodyCondition', () => { + it('should check for no _id when searching by CREATE', () => { + const result = bodyCondition(['CREATE']); + expect(result).toEqual([ + { + $and: [ + { + $and: [ + { + method: 'POST', + }, + { + body: { + $regex: '^(?!{"_id").*', + }, + }, + ], + }, + { + $and: [ + { + method: 'POST', + }, + { + body: { + $regex: '^(?!{"entity":"{\\\\"_id).*', + }, + }, + ], + }, + ], + }, + ]); + }); + it('should check for _id when searching by UPDATE', () => { + const result = bodyCondition(['UPDATE']); + expect(result).toEqual([ + { + $and: [ + { + method: 'POST', + }, + { + body: { + $regex: '^({"_id").*', + }, + }, + ], + }, + { + $and: [ + { + method: 'POST', + }, + { + body: { + $regex: '^({"entity":"{\\\\"_id).*', + }, + }, + ], + }, + { + $and: [ + { + method: 'PUT', + }, + { + body: { + $regex: '^({"_id").*', + }, + }, + ], + }, + { + $and: [ + { + method: 'PUT', + }, + { + body: { + $regex: '^({"entity":"{\\\\"_id).*', + }, + }, + ], + }, + ]); + }); + it('should check for _id in query when searching by DELETE', () => { + const result = bodyCondition(['DELETE']); + expect(result).toEqual([ + { + $and: [ + { + method: 'DELETE', + }, + { + body: { + $regex: '^({"_id").*', + }, + }, + ], + }, + { + $and: [ + { + method: 'DELETE', + }, + { + query: { + $regex: '^({"_id").*', + }, + }, + ], + }, + { + $and: [ + { + method: 'DELETE', + }, + { + query: { + $regex: '^({"sharedId").*', + }, + }, + ], + }, + ]); + }); + it('should only check for method MIGRATE', () => { + const result = bodyCondition(['MIGRATE']); + expect(result).toEqual([ + { + method: 'MIGRATE', + }, + ]); + }); + it('should combine conditions for several methods', () => { + const result = bodyCondition(['MIGRATE', 'DELETE', 'POST']); + expect(result).toEqual([ + { + method: 'MIGRATE', + }, + { + $and: [ + { + method: 'DELETE', + }, + { + body: { + $regex: '^({"_id").*', + }, + }, + ], + }, + { + $and: [ + { + method: 'DELETE', + }, + { + query: { + $regex: '^({"_id").*', + }, + }, + ], + }, + { + $and: [ + { + method: 'DELETE', + }, + { + query: { + $regex: '^({"sharedId").*', + }, + }, + ], + }, + { + method: 'POST', + }, + ]); + }); + }); + + describe('ActivityLogFilter', () => { + describe('searchFilter', () => { + it('should build a query for semantic search', () => { + const filters = new ActivityLogFilter({ search: 'new user' }); + const query = filters.prepareQuery(); + expect(query).toEqual({ + $and: [ + { + url: { + $regex: '^\\/api\\/users\\/new$', + }, + }, + { + method: 'POST', + }, + ], + }); + }); + it('should build a query for semantic search', () => { + const filters = new ActivityLogFilter({ search: 'new user' }); + const query = filters.prepareQuery(); + expect(query).toEqual({ + $and: [ + { + url: { + $regex: '^\\/api\\/users\\/new$', + }, + }, + { + method: 'POST', + }, + ], + }); + }); + it('should build an open query when there is not semantic matches', () => { + const filters = new ActivityLogFilter({ search: 'other content' }); + const query = filters.prepareQuery(); + expect(query).toEqual({ + $or: [ + { + method: { + $regex: 'OTHER CONTENT', + }, + }, + { + url: { + $regex: '^other content$', + }, + }, + { + query: { + $regex: '.*other content.*', + $options: 'si', + }, + }, + { + body: { + $regex: '.*other content.*', + $options: 'si', + }, + }, + { + params: { + $regex: '.*other content.*', + $options: 'si', + }, + }, + ], + }); + }); + }); + + it('should build complex queries', () => { + const filters = new ActivityLogFilter({ + username: 'admin', + search: 'Imported', + time: { from: 1000011600 }, + }); + const query = filters.prepareQuery(); + expect(query).toEqual({ + $and: [ + { + $or: [ + { + $or: [ + { + $and: [ + { + url: { + $regex: '^\\/api\\/import$', + }, + }, + { + method: 'POST', + }, + ], + }, + { + $and: [ + { + url: { + $regex: '^\\/api\\/translations\\/import$', + }, + }, + { + method: 'POST', + }, + ], + }, + ], + }, + ], + }, + { + $or: [ + { + username: { + $regex: '.*admin.*', + $options: 'si', + }, + }, + ], + }, + { + time: { + $gte: 1000011600, + }, + }, + ], + }); + }); + }); +}); diff --git a/app/api/activitylog/specs/activitylog.spec.js b/app/api/activitylog/specs/activitylog.spec.js index 69570962e1..12875ca4bc 100644 --- a/app/api/activitylog/specs/activitylog.spec.js +++ b/app/api/activitylog/specs/activitylog.spec.js @@ -38,12 +38,12 @@ describe('activitylog', () => { describe('get()', () => { it('should return all entries', async () => { const { rows: entries } = await activitylog.get(); - expect(entries.length).toBe(5); + expect(entries.length).toBe(6); }); it('should include semantic info for each result', async () => { const { rows: entries } = await activitylog.get(); - expect(activityLogParser.getSemanticData.mock.calls.length).toBe(5); + expect(activityLogParser.getSemanticData.mock.calls.length).toBe(6); expect(activityLogParser.getSemanticData).toHaveBeenCalledWith( expect.objectContaining({ method: 'DELETE', @@ -57,6 +57,7 @@ describe('activitylog', () => { }); }); + // eslint-disable-next-line max-statements describe('when filtering', () => { it('should filter by method', async () => { const { rows } = await activitylog.get({ method: ['POST'] }); @@ -66,33 +67,35 @@ describe('activitylog', () => { }); it('should filter by time', async () => { - let { rows: entries } = await activitylog.get({ time: { from: 5, to: 6 } }); + let { rows: entries } = await activitylog.get({ + time: { from: 1200002400000, to: 1300003200000 }, + }); expect(entries.length).toBe(2); - expect(entries[0].time).toBe(6000); - expect(entries[1].time).toBe(5000); + expect(entries[0].time).toBe(1300003200000); + expect(entries[1].time).toBe(1200002400000); ({ rows: entries } = await activitylog.get({ time: { from: null } })); - expect(entries.length).toBe(5); + expect(entries.length).toBe(6); ({ rows: entries } = await activitylog.get({ time: { to: null } })); - expect(entries.length).toBe(5); + expect(entries.length).toBe(6); }); it('should filter by url', async () => { - const { rows: entries } = await activitylog.get({ url: 'entities' }); + const { rows: entries } = await activitylog.get({ url: '/api/entities' }); expect(entries.length).toBe(4); }); it('should filter by query', async () => { const { rows: entries } = await activitylog.get({ query: '123' }); - expect(entries.length).toBe(1); + expect(entries.length).toBe(2); expect(entries[0].query).toBe('{"sharedId":"123"}'); }); it('should filter by body', async () => { const { rows: entries } = await activitylog.get({ body: 'Hello' }); expect(entries.length).toBe(1); - expect(entries[0].body).toBe('{"_id":"123","title":"Hello"}'); + expect(entries[0].body).toBe('{"title":"Hello"}'); }); it('should filter by username', async () => { @@ -102,26 +105,93 @@ describe('activitylog', () => { it('should filter by anonymous user', async () => { const { rows: entries } = await activitylog.get({ username: 'anonymous' }); - expect(entries.length).toBe(2); + expect(entries.length).toBe(3); expect(entries[0].username).toBeUndefined(); }); it('should allow a general find query of terms', async () => { const { rows: entries } = await activitylog.get({ find: 'Hello' }); expect(entries.length).toBe(3); - expect(entries[0].time).toBe(8000); - expect(entries[1].time).toBe(6000); - expect(entries[2].time).toBe(5000); + expect(entries[0].time).toBe(1400000400000); + expect(entries[1].time).toBe(1300003200000); + expect(entries[2].time).toBe(1200002400000); + }); + + it('should filter by semantic text', async () => { + const { rows: entries } = await activitylog.get({ search: 'Deleted entity' }); + expect(entries).toEqual([ + expect.objectContaining({ + method: 'DELETE', + url: '/api/entities', + query: '{"sharedId":"123"}', + time: 1100001600000, + username: 'admin', + }), + ]); + }); + + it('should filter by semantic method ', async () => { + const { rows: entries } = await activitylog.get({ search: 'create' }); + expect(entries).toEqual([ + expect.objectContaining({ + method: 'POST', + url: '/api/entities', + query: '{"sharedId":"123"}', + body: '{"_id":"456","title":"Entity 1"}', + time: 1500008400000, + }), + expect.objectContaining({ + method: 'POST', + url: '/api/entities', + body: '{"title":"Hello"}', + query: '{}', + time: 1200002400000, + username: 'admin', + }), + ]); + }); + + it('should filter by semantic method and a search term', async () => { + const { rows: entries } = await activitylog.get({ method: ['create'], search: 'Hello' }); + expect(entries).toEqual([ + expect.objectContaining({ + method: 'POST', + url: '/api/entities', + body: '{"title":"Hello"}', + query: '{}', + time: 1200002400000, + username: 'admin', + }), + ]); + }); + + it('should filter by a composed query ', async () => { + const { rows: entries } = await activitylog.get({ + url: '/api/entities', + username: 'admin', + method: ['POST'], + }); + expect(entries).toEqual([expect.objectContaining({ time: 1200002400000 })]); }); }); describe('Load More functionality', () => { it('should allow to load more via "before" param', async () => { const initialResults = await activitylog.get({ limit: 2 }); - assessResults(initialResults, { size: 2, remainingRows: 3, limit: 2, times: [8000, 6000] }); + assessResults(initialResults, { + size: 2, + remainingRows: 4, + limit: 2, + times: [1500008400000, 1400000400000], + }); - const nextResults = await activitylog.get({ before: 6000, limit: 2 }); - assessResults(nextResults, { size: 2, remainingRows: 1, limit: 2, times: [5000, 2000] }); + const nextResults = await activitylog.get({ before: 1300003200000, limit: 2 }); + assessResults(nextResults, { + size: 2, + remainingRows: 1, + limit: 2, + times: [1200002400000, 1100001600000], + }); }); }); @@ -130,32 +200,38 @@ describe('activitylog', () => { let results = await activitylog.get({ limit: 2 }); assessResults(results, { size: 2, - remainingRows: 3, + remainingRows: 4, limit: 2, page: undefined, - times: [8000, 6000], + times: [1500008400000, 1400000400000], }); results = await activitylog.get({ page: 1, limit: 2 }); assessResults(results, { size: 2, - remainingRows: 3, + remainingRows: 4, limit: 2, page: 1, - times: [8000, 6000], + times: [1500008400000, 1400000400000], }); results = await activitylog.get({ page: 2, limit: 2 }); assessResults(results, { size: 2, - remainingRows: 1, + remainingRows: 2, limit: 2, page: 2, - times: [5000, 2000], + times: [1300003200000, 1200002400000], }); results = await activitylog.get({ page: 3, limit: 2 }); - assessResults(results, { size: 1, remainingRows: 0, limit: 2, page: 3, times: [1000] }); + assessResults(results, { + size: 2, + remainingRows: 0, + limit: 2, + page: 3, + times: [1100001600000, 1000011600000], + }); }); it('should return an empty array if the page is out of range', async () => { @@ -170,50 +246,78 @@ describe('activitylog', () => { remainingRows: 1, limit: 2, page: 1, - times: [8000, 6000], + times: [1400000400000, 1300003200000], }); results = await activitylog.get({ page: 2, limit: 2, method: ['PUT'] }); - assessResults(results, { size: 1, remainingRows: 0, limit: 2, page: 2, times: [1000] }); + assessResults(results, { + size: 1, + remainingRows: 0, + limit: 2, + page: 2, + times: [1000011600000], + }); }); }); describe('sorting through the "sort" keyword', () => { + const checkSortResults = async (params, expectedTimeOrder) => { + const results = await activitylog.get(params); + expect(results.rows.map(r => r.time)).toEqual(expectedTimeOrder); + }; + it('without the "sort" keyword, it should sort by time, descending as default', async () => { - const results = await activitylog.get(); - expect(results.rows.map(r => r.time)).toEqual([8000, 6000, 5000, 2000, 1000]); + await checkSortResults( + {}, + [1500008400000, 1400000400000, 1300003200000, 1200002400000, 1100001600000, 1000011600000] + ); }); it('should sort by method', async () => { - let results = await activitylog.get({ sort: { prop: 'method', asc: 1 } }); - expect(results.rows.map(r => r.time)).toEqual([2000, 5000, 8000, 6000, 1000]); - - results = await activitylog.get({ sort: { prop: 'method', asc: 0 } }); - expect(results.rows.map(r => r.time)).toEqual([8000, 6000, 1000, 5000, 2000]); + await checkSortResults( + { sort: { prop: 'method', asc: 1 } }, + [1100001600000, 1500008400000, 1200002400000, 1400000400000, 1300003200000, 1000011600000] + ); + + await checkSortResults( + { sort: { prop: 'method', asc: 0 } }, + [1400000400000, 1300003200000, 1000011600000, 1500008400000, 1200002400000, 1100001600000] + ); }); it('should sort by user', async () => { - let results = await activitylog.get({ sort: { prop: 'username', asc: 1 } }); - expect(results.rows.map(r => r.time)).toEqual([8000, 6000, 5000, 2000, 1000]); - - results = await activitylog.get({ sort: { prop: 'username', asc: 0 } }); - expect(results.rows.map(r => r.time)).toEqual([1000, 5000, 2000, 8000, 6000]); + await checkSortResults( + { sort: { prop: 'username', asc: 1 } }, + [1500008400000, 1400000400000, 1300003200000, 1200002400000, 1100001600000, 1000011600000] + ); + + await checkSortResults( + { sort: { prop: 'username', asc: 0 } }, + [1000011600000, 1200002400000, 1100001600000, 1500008400000, 1400000400000, 1300003200000] + ); }); it('should sort by url', async () => { - let results = await activitylog.get({ sort: { prop: 'url', asc: 1 } }); - expect(results.rows.map(r => r.time)).toEqual([8000, 5000, 2000, 6000, 1000]); - - results = await activitylog.get({ sort: { prop: 'url', asc: 0 } }); - expect(results.rows.map(r => r.time)).toEqual([1000, 6000, 8000, 5000, 2000]); + await checkSortResults( + { sort: { prop: 'url', asc: 1 } }, + [1500008400000, 1400000400000, 1200002400000, 1100001600000, 1300003200000, 1000011600000] + ); + await checkSortResults( + { sort: { prop: 'url', asc: 0 } }, + [1000011600000, 1300003200000, 1500008400000, 1400000400000, 1200002400000, 1100001600000] + ); }); it('should sort by time', async () => { - let results = await activitylog.get({ sort: { prop: 'time', asc: 1 } }); - expect(results.rows.map(r => r.time)).toEqual([1000, 2000, 5000, 6000, 8000]); - - results = await activitylog.get({ sort: { prop: 'time', asc: 0 } }); - expect(results.rows.map(r => r.time)).toEqual([8000, 6000, 5000, 2000, 1000]); + await checkSortResults( + { sort: { prop: 'time', asc: 1 } }, + [1000011600000, 1100001600000, 1200002400000, 1300003200000, 1400000400000, 1500008400000] + ); + + await checkSortResults( + { sort: { prop: 'time', asc: 0 } }, + [1500008400000, 1400000400000, 1300003200000, 1200002400000, 1100001600000, 1000011600000] + ); }); it('should respect filters', async () => { @@ -221,13 +325,17 @@ describe('activitylog', () => { sort: { prop: 'username', asc: 1 }, method: ['PUT'], }); - expect(results.rows.map(r => r.time)).toEqual([8000, 6000, 1000]); + expect(results.rows.map(r => r.time)).toEqual([ + 1400000400000, 1300003200000, 1000011600000, + ]); results = await activitylog.get({ sort: { prop: 'username', asc: 0 }, method: ['PUT'], }); - expect(results.rows.map(r => r.time)).toEqual([1000, 8000, 6000]); + expect(results.rows.map(r => r.time)).toEqual([ + 1000011600000, 1400000400000, 1300003200000, + ]); }); it('should respect pagination', async () => { @@ -242,7 +350,7 @@ describe('activitylog', () => { remainingRows: 1, limit: 2, page: 1, - times: [8000, 6000], + times: [1400000400000, 1300003200000], }); results = await activitylog.get({ @@ -251,7 +359,13 @@ describe('activitylog', () => { limit: 2, method: ['PUT'], }); - assessResults(results, { size: 1, remainingRows: 0, limit: 2, page: 2, times: [1000] }); + assessResults(results, { + size: 1, + remainingRows: 0, + limit: 2, + page: 2, + times: [1000011600000], + }); results = await activitylog.get({ sort: { prop: 'username', asc: 1 }, diff --git a/app/api/activitylog/specs/activitylogParser.spec.js b/app/api/activitylog/specs/activitylogParser.spec.js index ab19858e7d..98245cf3a8 100644 --- a/app/api/activitylog/specs/activitylogParser.spec.js +++ b/app/api/activitylog/specs/activitylogParser.spec.js @@ -51,7 +51,7 @@ describe('Activitylog Parser', () => { describe('routes: /api/entities and /api/documents', () => { describe('method: POST', () => { - it('should beautify with body when it is a multipart request', async () => { + it('should beautify CREATE with body when it is a multipart request', async () => { await testBeautified( { method: 'POST', @@ -67,6 +67,22 @@ describe('Activitylog Parser', () => { ); }); + it('should beautify UPDATE with body when it is a multipart request', async () => { + await testBeautified( + { + method: 'POST', + url: '/api/entities', + body: `{"entity":${JSON.stringify(`{"sharedId": "m0asd", "title":"Existing Entity","template":"${firstTemplate.toString()}"}`)}}`, + }, + { + action: 'UPDATE', + description: 'Updated entity', + name: 'Existing Entity (m0asd)', + extra: 'of type Existing Template', + } + ); + }); + it('should beautify as CREATE when no ID found', async () => { await testBeautified( { diff --git a/app/api/activitylog/specs/fixtures.js b/app/api/activitylog/specs/fixtures.js index 246e3a8f04..9c526c8814 100644 --- a/app/api/activitylog/specs/fixtures.js +++ b/app/api/activitylog/specs/fixtures.js @@ -4,18 +4,24 @@ export default { method: 'POST', url: '/api/entities', query: '{}', - body: '{"_id":"123","title":"Hello"}', - time: 5000, + body: '{"title":"Hello"}', + time: 1200002400000, username: 'admin', }, - { method: 'PUT', url: '/api/entities', query: '{"name": "Hello"}', body: '{}', time: 8000 }, - { method: 'PUT', url: '/api/entities/Hello', query: '{}', body: '{}', time: 6000 }, + { + method: 'PUT', + url: '/api/entities', + query: '{"name": "Hello"}', + body: '{}', + time: 1400000400000, + }, + { method: 'PUT', url: '/api/entities/Hello', query: '{}', body: '{}', time: 1300003200000 }, { method: 'PUT', url: '/api/templates', query: '{}', body: '{}', - time: 1000, + time: 1000011600000, username: 'Hello', }, { @@ -23,8 +29,15 @@ export default { url: '/api/entities', query: '{"sharedId":"123"}', body: '{}', - time: 2000, + time: 1100001600000, username: 'admin', }, + { + method: 'POST', + url: '/api/entities', + query: '{"sharedId":"123"}', + body: '{"_id":"456","title":"Entity 1"}', + time: 1500008400000, + }, ], }; diff --git a/app/react/App/styles/globals.css b/app/react/App/styles/globals.css index ad928b9dd6..440b13a002 100644 --- a/app/react/App/styles/globals.css +++ b/app/react/App/styles/globals.css @@ -1640,6 +1640,10 @@ input[type="range"]::-ms-fill-lower { top: -2.75rem; } +.-top-4 { + top: -1rem; +} + .bottom-0 { bottom: 0px; } @@ -1801,8 +1805,8 @@ input[type="range"]::-ms-fill-lower { margin-left: auto; } -.mr-10 { - margin-right: 2.5rem; +.mr-0 { + margin-right: 0px; } .mr-2 { @@ -1983,18 +1987,6 @@ input[type="range"]::-ms-fill-lower { min-height: fit-content; } -.min-h-96 { - min-height: 24rem; -} - -.min-h-\[80\%\] { - min-height: 80%; -} - -.min-h-\[70\%\] { - min-height: 70%; -} - .\!w-full { width: 100% !important; } @@ -2099,10 +2091,6 @@ input[type="range"]::-ms-fill-lower { width: 66.666667%; } -.w-40 { - width: 10rem; -} - .w-5 { width: 1.25rem; } @@ -2281,18 +2269,6 @@ input[type="range"]::-ms-fill-lower { flex-grow: 1; } -.basis-1\/3 { - flex-basis: 33.333333%; -} - -.basis-1\/5 { - flex-basis: 20%; -} - -.basis-2\/5 { - flex-basis: 40%; -} - .border-collapse { border-collapse: collapse; } @@ -2425,10 +2401,6 @@ input[type="range"]::-ms-fill-lower { justify-content: space-between; } -.justify-items-stretch { - justify-items: stretch; -} - .gap-1 { gap: 0.25rem; } @@ -2952,6 +2924,11 @@ input[type="range"]::-ms-fill-lower { background-color: rgb(67 56 202 / var(--tw-bg-opacity)); } +.bg-red-100 { + --tw-bg-opacity: 1; + background-color: rgb(253 232 232 / var(--tw-bg-opacity)); +} + .bg-success-100 { --tw-bg-opacity: 1; background-color: rgb(220 252 231 / var(--tw-bg-opacity)); @@ -3372,6 +3349,11 @@ input[type="range"]::-ms-fill-lower { color: rgb(79 70 229 / var(--tw-text-opacity)); } +.text-blue-800 { + --tw-text-opacity: 1; + color: rgb(55 48 163 / var(--tw-text-opacity)); +} + .text-error-600 { --tw-text-opacity: 1; color: rgb(219 39 119 / var(--tw-text-opacity)); @@ -3447,6 +3429,11 @@ input[type="range"]::-ms-fill-lower { color: rgb(5 122 85 / var(--tw-text-opacity)); } +.text-green-800 { + --tw-text-opacity: 1; + color: rgb(3 84 63 / var(--tw-text-opacity)); +} + .text-indigo-700 { --tw-text-opacity: 1; color: rgb(81 69 205 / var(--tw-text-opacity)); @@ -3497,6 +3484,11 @@ input[type="range"]::-ms-fill-lower { color: rgb(55 48 163 / var(--tw-text-opacity)); } +.text-red-800 { + --tw-text-opacity: 1; + color: rgb(155 28 28 / var(--tw-text-opacity)); +} + .text-success-600 { --tw-text-opacity: 1; color: rgb(22 163 74 / var(--tw-text-opacity)); diff --git a/app/react/Routes.tsx b/app/react/Routes.tsx index a23430a29b..2b175ba633 100644 --- a/app/react/Routes.tsx +++ b/app/react/Routes.tsx @@ -43,7 +43,7 @@ import { IXdashboardLoader, IXDashboard } from 'V2/Routes/Settings/IX/IXDashboar import { IXSuggestions, IXSuggestionsLoader } from 'V2/Routes/Settings/IX/IXSuggestions'; import { PageEditor, pageEditorLoader, PagesList, pagesListLoader } from 'V2/Routes/Settings/Pages'; import { customisationLoader, Customisation } from 'V2/Routes/Settings/Customization/Customization'; -import { activityLogLoader, ActivityLog } from 'V2/Routes/Settings/ActivityLog/ActivityLog'; +import { ActivityLog, activityLogLoader } from 'V2/Routes/Settings/ActivityLog'; import { CustomUploads, customUploadsLoader } from 'V2/Routes/Settings/CustomUploads/CustomUploads'; import { FiltersTable, filtersLoader } from 'V2/Routes/Settings/Filters'; import { RouteErrorBoundary, GeneralError } from 'V2/Components/ErrorHandling'; @@ -175,7 +175,7 @@ const getRoutesLayout = ( )} - loader={activityLogLoader(headers)} + loader={activityLogLoader(headers, { settings })} /> any; onChange?: ChangeEventHandler; onBlur?: ChangeEventHandler; + className?: string; } const titleFormat = (locale: string) => { @@ -87,8 +89,9 @@ const DatePickerComponent = React.forwardRef( autoComplete, id = uniqueID(), language = 'en', - dateFormat = 'yyyy-mm-dd', + dateFormat = 'YYYY-MM-DD', hideLabel = true, + inputClassName = '', className = '', name = '', onChange = () => {}, @@ -103,8 +106,8 @@ const DatePickerComponent = React.forwardRef( const datePickerFormat = dateFormat.toLocaleLowerCase(); const fieldStyles = !(hasErrors || errorMessage) ? // eslint-disable-next-line max-len - `${className || ''} bg-gray-50 border border-gray-300 text-gray-900 sm:text-sm rounded-lg focus:ring-blue-500 focus:border-blue-500 block w-full p-2.5` - : `${className || ''} border-error-300 focus:border-error-500 focus:ring-error-500 border-2 text-error-900 bg-error-50 placeholder-error-700`; + `${inputClassName || ''} bg-gray-50 border border-gray-300 text-gray-900 sm:text-sm rounded-lg focus:ring-blue-500 focus:border-blue-500 block w-full p-2.5` + : `${inputClassName || ''} border-error-300 focus:border-error-500 focus:ring-error-500 border-2 text-error-900 bg-error-50 placeholder-error-700`; const instance = useRef(null); const locale = validateLocale(language); @@ -132,9 +135,15 @@ const DatePickerComponent = React.forwardRef( return () => (instance?.current?.hide instanceof Function ? instance?.current?.hide() : {}); }, [id, locale, labelToday, labelClear, datePickerFormat, clearFieldAction]); + useEffect(() => { + if (instance?.current && ref?.current) { + ref.current.value = isNumber(value) ? value.toString() : value || ''; + } + }, [instance, value]); + return (
-
+