From b751e717b24f7dac7645bb6ad6caf0d2cf5da17e Mon Sep 17 00:00:00 2001 From: Prabhjot Tugger <107434397+ptugger@users.noreply.github.com> Date: Mon, 10 Jul 2023 17:48:01 -0600 Subject: [PATCH] Test:Enhance export API endpoint (#840) * Ticket 14703 - Enhance Export API in CHEFS to support filtering based on status and updates * Modifications to Export API endpoint to limit user to use allowed columns only * Ticket 14703 - Enhance Export API in CHEFS to support filtering based on status and updates * Modifications to Export API endpoint to limit user to use allowed columns only * Fix issue when single column supplied and fix migration * Added Unit Tests * Buggy Test _getSubmissions included * unit test add for submissionData filterUpdatedAt filter * Work in progress for unit tests-pending * Added more unit tests for _getSubmissions function that covers with and without preferences * Removed unwanted console statements * Removed package file from root * Test:Enhance Export API for ETL (#790) * Ticket 14703 - Enhance Export API in CHEFS to support filtering based on status and updates * Modifications to Export API endpoint to limit user to use allowed columns only * Fix issue when single column supplied and fix migration * Added Unit Tests * Buggy Test _getSubmissions included * unit test add for submissionData filterUpdatedAt filter * Work in progress for unit tests-pending * Added more unit tests for _getSubmissions function that covers with and without preferences * Removed unwanted console statements * Removed package file from root --------- Co-authored-by: bigvaimarcella <111837621+bigvaimarcella@users.noreply.github.com> * Added status as param into Export api to filter submissions by Status * Updated Unit tests * Added documentation param * Removed commented statement (Reviewed) --------- Co-authored-by: bigvaimarcella <111837621+bigvaimarcella@users.noreply.github.com> --- ..._32-add-update-col-alter-submissions-vw.js | 92 ++++++ app/src/docs/v1.api-spec.yaml | 38 ++- .../common/models/views/submissionData.js | 14 + app/src/forms/form/exportService.js | 28 +- app/tests/common/dbHelper.js | 2 + ...k_submission_extract_field_csv_export.json | 275 ++++++++++++++++++ .../unit/forms/form/exportService.spec.js | 191 ++++++++++++ 7 files changed, 625 insertions(+), 15 deletions(-) create mode 100644 app/src/db/migrations/20230517012755_32-add-update-col-alter-submissions-vw.js diff --git a/app/src/db/migrations/20230517012755_32-add-update-col-alter-submissions-vw.js b/app/src/db/migrations/20230517012755_32-add-update-col-alter-submissions-vw.js new file mode 100644 index 000000000..92fd539cd --- /dev/null +++ b/app/src/db/migrations/20230517012755_32-add-update-col-alter-submissions-vw.js @@ -0,0 +1,92 @@ +exports.up = function(knex) { + return Promise.resolve() + .then(() => knex.schema.dropViewIfExists('submissions_data_vw')) + .then(() => knex.schema.raw(`create or replace view submissions_data_vw as + select + s."confirmationId", + s."formName", + s.version, + s."createdAt", + case + when u.id is null then 'public'::varchar(255) + else u."fullName" + end as "fullName", + case + when u.id is null then 'public'::varchar(255) + else u.username + end as "username", + u.email, + fs.submission -> 'data' as "submission", + fs."updatedAt", + s.deleted, + s.draft, + s."submissionId", + s."formId", + s."formVersionId", + u.id as "userId", + u."idpUserId", + u."firstName", + u."lastName", + s."formSubmissionStatusCode" as "status", + s."formSubmissionAssignedToFullName" as "assignee", + s."formSubmissionAssignedToEmail" as "assigneeEmail" + from + submissions_vw s + inner join form_submission fs on + s."submissionId" = fs.id + left outer join form_submission_user fsu on + s."submissionId" = fsu."formSubmissionId" + and fsu.permission = 'submission_create' + left outer join "user" u on + fsu."userId" = u.id + order by + s."createdAt", + s."formName", + s.version`)); +}; + +exports.down = function(knex) { + return Promise.resolve() + .then(() => knex.schema.dropViewIfExists('submissions_data_vw')) + .then(() => knex.schema.raw(`create or replace view submissions_data_vw as + select + s."confirmationId", + s."formName", + s.version, + s."createdAt", + case + when u.id is null then 'public'::varchar(255) + else u."fullName" + end as "fullName", + case + when u.id is null then 'public'::varchar(255) + else u.username + end as "username", + u.email, + fs.submission -> 'data' as "submission", + s.deleted, + s.draft, + s."submissionId", + s."formId", + s."formVersionId", + u.id as "userId", + u."idpUserId", + u."firstName", + u."lastName", + s."formSubmissionStatusCode" as "status", + s."formSubmissionAssignedToFullName" as "assignee", + s."formSubmissionAssignedToEmail" as "assigneeEmail" + from + submissions_vw s + inner join form_submission fs on + s."submissionId" = fs.id + left outer join form_submission_user fsu on + s."submissionId" = fsu."formSubmissionId" + and fsu.permission = 'submission_create' + left outer join "user" u on + fsu."userId" = u.id + order by + s."createdAt", + s."formName", + s.version`)); +}; diff --git a/app/src/docs/v1.api-spec.yaml b/app/src/docs/v1.api-spec.yaml index 29fc37174..6267c2433 100755 --- a/app/src/docs/v1.api-spec.yaml +++ b/app/src/docs/v1.api-spec.yaml @@ -337,17 +337,39 @@ paths: example: 1 required: true - in: query - name: minDate + name: preference schema: - type: string - example: '2020-12-17T08:00:00Z' - description: Start date of period included in the export + type: object + example: '{ minDate=2020-12-17T08:00:00Z, maxDate=2020-12-17T08:00:00Z, updatedMinDate=2020-12-17T08:00:00Z, updatedMaxDate=2020-12-17T08:00:00Z }' + description: form submissions export preferences + - in: query + name: deleted + schema: + type: Boolean + description: (optional) This optional parameter should be set to true if deleted records (submissions) need to be fetched + example: false - in: query - name: maxDate + name: drafts + schema: + type: Boolean + description: (optional) This optional parameter should be set to true if draft records (submissions) need to be fetched + example: false + - in: query + name: columns + schema: + type: array + description: (optional) List of form level columns (Only Allowed draft, deleted, updatedAt columns) to be include. Other then allowed columns will be ignored + example: + - draft, + - deleted, + - updatedAt + - in: query + name: status schema: type: string - example: '2020-12-17T08:00:00Z' - description: End date of period included in the export + description: Submission status to be filtered based on + example: COMPLETED + required: false responses: '200': description: Export file created for download @@ -3172,7 +3194,7 @@ components: type: String description: default value is submissions and should be changed example: submissions - preferences: + preference: type: object description: form submissions export preferences example: { minDate, maxDate } diff --git a/app/src/forms/common/models/views/submissionData.js b/app/src/forms/common/models/views/submissionData.js index ce65ce038..d3728e545 100644 --- a/app/src/forms/common/models/views/submissionData.js +++ b/app/src/forms/common/models/views/submissionData.js @@ -21,6 +21,20 @@ class SubmissionData extends Model { query.where('createdAt', '<=', maxDate); } }, + filterUpdatedAt(query, minDate, maxDate) { + if (minDate && maxDate) { + query.whereBetween('updatedAt', [minDate, maxDate]); + } else if (minDate) { + query.where('updatedAt', '>=', minDate); + } else if (maxDate) { + query.where('updatedAt', '<=', maxDate); + } + }, + filterStatus(query, value) { + if (value) { + query.where('status', value); + } + }, filterDeleted(query, value) { if (!value) { query.where('deleted', false); diff --git a/app/src/forms/form/exportService.js b/app/src/forms/form/exportService.js index 61174439b..deca97179 100644 --- a/app/src/forms/form/exportService.js +++ b/app/src/forms/form/exportService.js @@ -138,13 +138,18 @@ const service = { return `${form.snake()}_${type}.${format}`.toLowerCase(); }, - _submissionsColumns: (form) => { + _submissionsColumns: (form, params) => { // Custom columns not defined - return default column selection behavior let columns = ['confirmationId', 'formName', 'version', 'createdAt', 'fullName', 'username', 'email']; // if form has 'status updates' enabled in the form settings include these in export if (form.enableStatusUpdates) { columns = columns.concat(['status', 'assignee', 'assigneeEmail']); } + // Let's add form level columns like deleted or draft + if (params?.columns?.length) { + let optionalAcceptedColumns = ['draft', 'deleted', 'updatedAt']; //'draft', 'deleted', 'updatedAt' columns needed for ETL process at this moment + columns = columns.concat((Array.isArray(params.columns) ? [...params.columns] : [params.columns]).filter((column) => optionalAcceptedColumns.includes(column))); + } // and join the submission data return columns.concat(['submission']); }, @@ -155,7 +160,8 @@ const service = { _getData: async (exportType, formVersion, form, params = {}) => { if (EXPORT_TYPES.submissions === exportType) { - return service._getSubmissions(form, params, formVersion); + let subs = await service._getSubmissions(form, params, formVersion); + return subs; } return {}; }, @@ -188,16 +194,25 @@ const service = { } else { preference = params.preference; } - + // let submissionData; // params for this export include minDate and maxDate (full timestamp dates). - let submissionData = await SubmissionData.query() - .column(service._submissionsColumns(form, params)) + return SubmissionData.query() + .select(service._submissionsColumns(form, params)) .where('formId', form.id) .modify('filterVersion', version) .modify('filterCreatedAt', preference && preference.minDate, preference && preference.maxDate) + .modify('filterUpdatedAt', preference && preference.updatedMinDate, preference && preference.updatedMaxDate) + .modify('filterStatus', params.status) .modify('filterDeleted', params.deleted) .modify('filterDrafts', params.drafts) - .modify('orderDefault'); + .modify('orderDefault') + .then((submissionData) => { + if (submissionData == undefined || submissionData == null || submissionData.length == 0) return []; + return service._submissionFilterByUnsubmit(submissionData); + }); + }, + + _submissionFilterByUnsubmit: (submissionData) => { for (let index in submissionData) { let keys = Object.keys(submissionData[index].submission); for (let key of keys) { @@ -208,7 +223,6 @@ const service = { } return submissionData; }, - _formatSubmissionsJson: (form, data) => { return { data: data, diff --git a/app/tests/common/dbHelper.js b/app/tests/common/dbHelper.js index d485f7e30..03eca86a3 100644 --- a/app/tests/common/dbHelper.js +++ b/app/tests/common/dbHelper.js @@ -48,6 +48,8 @@ MockModel.orderBy = jest.fn().mockReturnThis(); MockModel.patch = jest.fn().mockReturnThis(); MockModel.patchAndFetchById = jest.fn().mockReturnThis(); MockModel.query = jest.fn().mockReturnThis(); +MockModel.query.select = jest.fn().mockReturnThis(); +MockModel.query.column = jest.fn().mockReturnThis(); MockModel.resolve = jest.fn().mockResolvedValue(returnValue); MockModel.returning = jest.fn().mockReturnThis(); (MockModel.skipUndefined = jest.fn(() => { diff --git a/app/tests/fixtures/submission/kitchen_sink_submission_extract_field_csv_export.json b/app/tests/fixtures/submission/kitchen_sink_submission_extract_field_csv_export.json index cb6524e3e..53280abc4 100644 --- a/app/tests/fixtures/submission/kitchen_sink_submission_extract_field_csv_export.json +++ b/app/tests/fixtures/submission/kitchen_sink_submission_extract_field_csv_export.json @@ -258,5 +258,280 @@ "didYouFishAnyBcLakesThisYear": "yes", "forWhichBcLakeRegionAreYouCompletingTheseQuestions": 1 } + }, + { + "confirmationId": "40EDE76F", + "formName": "Fisheries", + "version": 1, + "createdAt": "2023-06-02T18:09:26.559Z", + "updatedAt": "2023-06-02T18:10:26.559Z", + "draft": true, + "deleted": false, + "fullName": "Prabhjot Tugger", + "username": "ptugger", + "email": "prabhjot.tugger@gov.bc.ca", + "submission": { + "email": "prabhjot.tugger@gov.bc.ca", + "submit": true, + "oneRowPerLake": [ + { + "dataGrid": [ + { + "fishType": "rainbow", + "numberKept": 1, + "numberCaught": 1 + }, + { + "fishType": "cutthroat", + "numberKept": 2, + "numberCaught": 2 + } + ], + "lakeName": "Ran", + "closestTown": "Ranian", + "numberOfDays": 1 + }, + { + "dataGrid": [ + { + "fishType": "rainbow", + "numberKept": 1, + "numberCaught": 1 + }, + { + "fishType": "smallmouthBass", + "numberKept": 1, + "numberCaught": 1 + } + ], + "lakeName": "Cat", + "closestTown": "Catian", + "numberOfDays": 1 + } + ], + "fishermansName": "Geroge Aurth", + "didYouFishAnyBcLakesThisYear": "yes", + "forWhichBcLakeRegionAreYouCompletingTheseQuestions": 1 + } + }, + { + "confirmationId": "3E105CA1", + "formName": "Fisheries", + "version": 1, + "createdAt": "2023-06-02T18:06:26.559Z", + "updatedAt": "2023-06-02T18:06:26.559Z", + "draft": false, + "deleted": false, + "fullName": "Prabhjot Tugger", + "username": "AIDOWU", + "email": "prabhjot.tugger@gov.bc.ca", + "submission": { + "email": "prabhjot.tugger@gov.bc.ca", + "submit": true, + "oneRowPerLake": [ + { + "dataGrid": [ + { + "fishType": "rainbow", + "numberKept": 1, + "numberCaught": 1 + }, + { + "fishType": "cutthroat", + "numberKept": 2, + "numberCaught": 2 + } + ], + "lakeName": "Ran", + "closestTown": "Ranian", + "numberOfDays": 1 + }, + { + "dataGrid": [ + { + "fishType": "rainbow", + "numberKept": 1, + "numberCaught": 1 + }, + { + "fishType": "smallmouthBass", + "numberKept": 1, + "numberCaught": 1 + } + ], + "lakeName": "Cat", + "closestTown": "Catian", + "numberOfDays": 1 + } + ], + "fishermansName": "Geroge Aurth", + "didYouFishAnyBcLakesThisYear": "yes", + "forWhichBcLakeRegionAreYouCompletingTheseQuestions": 1 + } + }, + { + "confirmationId": "CD097322", + "formName": "Fisheries", + "version": 1, + "createdAt": "2023-06-03T18:06:26.559Z", + "updatedAt": "2023-06-04T18:06:26.559Z", + "draft": true, + "deleted": true, + "fullName": "Prabhjot Tugger", + "username": "AIDOWU", + "email": "prabhjot.tugger@gov.bc.ca", + "submission": { + "email": "prabhjot.tugger@gov.bc.ca", + "submit": true, + "oneRowPerLake": [ + { + "dataGrid": [ + { + "fishType": "rainbow", + "numberKept": 1, + "numberCaught": 1 + }, + { + "fishType": "cutthroat", + "numberKept": 2, + "numberCaught": 2 + } + ], + "lakeName": "Ran", + "closestTown": "Ranian", + "numberOfDays": 1 + }, + { + "dataGrid": [ + { + "fishType": "rainbow", + "numberKept": 1, + "numberCaught": 1 + }, + { + "fishType": "smallmouthBass", + "numberKept": 1, + "numberCaught": 1 + } + ], + "lakeName": "Cat", + "closestTown": "Catian", + "numberOfDays": 1 + } + ], + "fishermansName": "Geroge Aurth", + "didYouFishAnyBcLakesThisYear": "yes", + "forWhichBcLakeRegionAreYouCompletingTheseQuestions": 1 + } + }, + { + "confirmationId": "CD935788", + "formName": "Fisheries", + "version": 1, + "createdAt": "2023-06-02T18:06:26.559Z", + "updatedAt": "2023-06-02T18:06:26.559Z", + "draft": false, + "deleted": true, + "fullName": "Prabhjot Tugger", + "username": "AIDOWU", + "email": "prabhjot.tugger@gov.bc.ca", + "submission": { + "email": "prabhjot.tugger@gov.bc.ca", + "submit": true, + "oneRowPerLake": [ + { + "dataGrid": [ + { + "fishType": "rainbow", + "numberKept": 1, + "numberCaught": 1 + }, + { + "fishType": "cutthroat", + "numberKept": 2, + "numberCaught": 2 + } + ], + "lakeName": "Ran", + "closestTown": "Ranian", + "numberOfDays": 1 + }, + { + "dataGrid": [ + { + "fishType": "rainbow", + "numberKept": 1, + "numberCaught": 1 + }, + { + "fishType": "smallmouthBass", + "numberKept": 1, + "numberCaught": 1 + } + ], + "lakeName": "Cat", + "closestTown": "Catian", + "numberOfDays": 1 + } + ], + "fishermansName": "Geroge Aurth", + "didYouFishAnyBcLakesThisYear": "yes", + "forWhichBcLakeRegionAreYouCompletingTheseQuestions": 1 + } + }, + { + "confirmationId": "CD085969", + "formName": "Fisheries", + "version": 1, + "createdAt": "2023-06-02T18:06:26.559Z", + "updatedAt": "2023-06-02T18:06:26.559Z", + "draft": false, + "deleted": false, + "fullName": "Prabhjot Tugger", + "username": "AIDOWU", + "email": "prabhjot.tugger@gov.bc.ca", + "submission": { + "email": "prabhjot.tugger@gov.bc.ca", + "submit": true, + "oneRowPerLake": [ + { + "dataGrid": [ + { + "fishType": "rainbow", + "numberKept": 1, + "numberCaught": 1 + }, + { + "fishType": "cutthroat", + "numberKept": 2, + "numberCaught": 2 + } + ], + "lakeName": "Ran", + "closestTown": "Ranian", + "numberOfDays": 1 + }, + { + "dataGrid": [ + { + "fishType": "rainbow", + "numberKept": 1, + "numberCaught": 1 + }, + { + "fishType": "smallmouthBass", + "numberKept": 1, + "numberCaught": 1 + } + ], + "lakeName": "Cat", + "closestTown": "Catian", + "numberOfDays": 1 + } + ], + "fishermansName": "Geroge Aurth", + "didYouFishAnyBcLakesThisYear": "yes", + "forWhichBcLakeRegionAreYouCompletingTheseQuestions": 1 + } } ] \ No newline at end of file diff --git a/app/tests/unit/forms/form/exportService.spec.js b/app/tests/unit/forms/form/exportService.spec.js index ca7aa21d9..686d9ffcd 100644 --- a/app/tests/unit/forms/form/exportService.spec.js +++ b/app/tests/unit/forms/form/exportService.spec.js @@ -1,4 +1,14 @@ const exportService = require('../../../../src/forms/form/exportService'); +const MockModel = require('../../../../src/forms/common/models/views/submissionData'); +const _ = require('lodash'); +jest.mock('../../../../src/forms/common/models/views/submissionData', () => ({ + query: jest.fn().mockReturnThis(), + select: jest.fn().mockReturnThis(), + column: jest.fn().mockReturnThis(), + where: jest.fn().mockReturnThis(), + modify: jest.fn().mockReturnThis(), + then: jest.fn().mockReturnThis(), +})); describe('_readSchemaFields', () => { it('should get form fields in the order they appear in the kitchen sink form', async () => { @@ -339,3 +349,184 @@ describe('', () => { expect(fields.length).toEqual(19); }); }); + +describe('_submissionsColumns', () => { + const form = { + id: 'bd4dcf26-65bd-429b-967f-125500bfd8a4', + name: 'Fisheries', + description: '', + active: true, + labels: [], + createdBy: 'AIDOWU@idir', + createdAt: '2023-03-29T14:09:28.457Z', + updatedBy: 'AIDOWU@idir', + updatedAt: '2023-04-10T16:19:43.491Z', + showSubmissionConfirmation: true, + submissionReceivedEmails: [], + enableStatusUpdates: false, + enableSubmitterDraft: true, + schedule: {}, + reminder_enabled: false, + enableCopyExistingSubmission: false, + }; + + it('should return right number of columns, when no prefered columns passed as params.', async () => { + const params = { + type: 'submissions', + format: 'json', + drafts: true, + deleted: false, + version: 1, + }; + + const submissions = exportService._submissionsColumns(form, params); + expect(submissions.length).toEqual(8); + }); + + it('should return right number of columns, when 1 prefered column (deleted) passed as params.', async () => { + const params = { + type: 'submissions', + format: 'json', + drafts: true, + deleted: false, + version: 1, + columns: ['deleted'], + }; + + const submissions = exportService._submissionsColumns(form, params); + expect(submissions.length).toEqual(9); + }); + + it('should return right number of columns, when 1 prefered column (draft) passed as params.', async () => { + const params = { + type: 'submissions', + format: 'json', + drafts: true, + deleted: false, + version: 1, + columns: ['draft'], + }; + + const submissions = exportService._submissionsColumns(form, params); + expect(submissions.length).toEqual(9); + }); + + it('should return right number of columns, when 2 prefered column (draft & deleted) passed as params.', async () => { + const params = { + type: 'submissions', + format: 'json', + drafts: true, + deleted: false, + version: 1, + columns: ['draft', 'deleted'], + }; + + const submissions = exportService._submissionsColumns(form, params); + expect(submissions.length).toEqual(10); + }); + + it('should return right number of columns, when a garbage or NON-allowed column (testCol1 & testCol2) passed as params.', async () => { + const params = { + type: 'submissions', + format: 'json', + drafts: true, + deleted: false, + version: 1, + columns: ['testCol1', 'testCol2'], + }; + + const submissions = exportService._submissionsColumns(form, params); + expect(submissions.length).toEqual(8); + }); +}); + +describe('_getSubmissions', () => { + // form schema from db + const form = { + id: 'bd4dcf26-65bd-429b-967f-125500bfd8a4', + name: 'Fisheries', + description: '', + active: true, + labels: [], + createdBy: 'AIDOWU@idir', + createdAt: '2023-03-29T14:09:28.457Z', + updatedBy: 'AIDOWU@idir', + updatedAt: '2023-04-10T16:19:43.491Z', + showSubmissionConfirmation: true, + submissionReceivedEmails: [], + enableStatusUpdates: false, + enableSubmitterDraft: true, + schedule: {}, + reminder_enabled: false, + enableCopyExistingSubmission: false, + }; + + it('Should pass this test with empty preference passed to _getSubmissions', async () => { + const params = { + type: 'submissions', + draft: false, + deleted: false, + version: 1, + preference: { + updatedMinDate: '', + updatedMaxDate: '', + }, + }; + + MockModel.query.mockImplementation(() => MockModel); + exportService._submissionsColumns = jest.fn().mockReturnThis(); + + let preference; + if (params.preference && _.isString(params.preference)) { + preference = JSON.parse(params.preference); + } else { + preference = params.preference; + } + exportService._getSubmissions(form, params, params.version); + expect(MockModel.query).toHaveBeenCalledTimes(1); + expect(MockModel.modify).toHaveBeenCalledTimes(7); + expect(MockModel.modify).toHaveBeenCalledWith('filterUpdatedAt', preference && preference.updatedMinDate, preference && preference.updatedMaxDate); + }); + + it('Should pass this test without preference passed to _getSubmissions and without calling updatedAt modifier', async () => { + const params = { + type: 'submissions', + draft: false, + deleted: false, + version: 1, + }; + + MockModel.query.mockImplementation(() => MockModel); + exportService._submissionsColumns = jest.fn().mockReturnThis(); + exportService._getSubmissions(form, params, params.version); + expect(MockModel.query).toHaveBeenCalledTimes(1); + expect(MockModel.modify).toHaveBeenCalledTimes(7); + }); + + it('Should pass this test with preference passed to _getSubmissions', async () => { + const params = { + type: 'submissions', + draft: false, + deleted: false, + version: 1, + preference: { + updatedMinDate: '2020-12-10T08:00:00Z', + updatedMaxDate: '2020-12-17T08:00:00Z', + }, + }; + + MockModel.query.mockImplementation(() => MockModel); + exportService._submissionsColumns = jest.fn().mockReturnThis(); + + let preference; + if (params.preference && _.isString(params.preference)) { + preference = JSON.parse(params.preference); + } else { + preference = params.preference; + } + exportService._getSubmissions(form, params, params.version); + expect(MockModel.query).toHaveBeenCalledTimes(1); + expect(MockModel.modify).toHaveBeenCalledTimes(7); + expect(MockModel.modify).toHaveBeenCalledWith('filterUpdatedAt', preference && preference.updatedMinDate, preference && preference.updatedMaxDate); + }); +});