From d66ba14dc63a0fd9cc5872a8e1d59c69a4744a89 Mon Sep 17 00:00:00 2001 From: Vasyl Yurkovych <59879559+yurkovychv@users.noreply.github.com> Date: Thu, 3 Aug 2023 15:40:31 +0300 Subject: [PATCH] PMM-12352 - restore to different mongo rs (#661) * PMM-12352 - restore to different mongo rs * PMM-12352 - fix more tests * PMM-12352 - remove retries * PMM-12352 - tune retries * PMM-12352 - tune retries * PMM-12352 - tune retries * PMM-12352 - tune retries * PMM-12352 - trigger * PMM-12352 - trigger * PMM-12352 - fix retry test * PMM-12352 - fix pgsm test * PMM-12352 - retry for pgsm * PMM-12352 - pgsm labels * PMM-12352 - trigger --- tests/backup/inventory_test.js | 83 +++++++++++++++++-- tests/backup/pages/api/locationsAPI.js | 8 +- tests/backup/pages/inventoryPage.js | 13 +++ tests/ia/pages/ruleTemplatesPage.js | 9 +- tests/ia/ruleTemplates_test.js | 4 +- tests/ia/templates/template.yaml | 1 - .../pmm_pgsm_integration_test.js | 24 ++++-- 7 files changed, 120 insertions(+), 22 deletions(-) diff --git a/tests/backup/inventory_test.js b/tests/backup/inventory_test.js index 4eac94c91..89b1ce79f 100644 --- a/tests/backup/inventory_test.js +++ b/tests/backup/inventory_test.js @@ -18,12 +18,24 @@ const mongoServiceName = 'mongo-backup-inventory-1'; const mongoServiceName2 = 'mongo-backup-inventory-2'; const mongoServiceName3 = 'mongo-backup-inventory-3'; +const mongoExtraServiceName = 'mongo-extra-1'; +const mongoExtraServiceName2 = 'mongo-extra-2'; +const mongoExtraServiceName3 = 'mongo-extra-3'; + +let mongoClient; + const mongoConnection = { username: 'pmm', password: 'pmmpass', port: 27027, }; +const mongoConnectionReplica2 = { + username: 'pmm', + password: 'pmmpass', + port: 27037, +}; + Feature('BM: Backup Inventory'); BeforeSuite(async ({ @@ -42,11 +54,17 @@ BeforeSuite(async ({ locationsAPI.storageLocationConnection, location.description, ); + await I.mongoConnect(mongoConnection); I.say(await I.verifyCommand(`docker exec rs101 pmm-admin add mongodb --username=pmm --password=pmmpass --port=27017 --service-name=${mongoServiceName} --replication-set=rs --cluster=rs`)); I.say(await I.verifyCommand(`docker exec rs102 pmm-admin add mongodb --username=pmm --password=pmmpass --port=27017 --service-name=${mongoServiceName2} --replication-set=rs --cluster=rs`)); I.say(await I.verifyCommand(`docker exec rs103 pmm-admin add mongodb --username=pmm --password=pmmpass --port=27017 --service-name=${mongoServiceName3} --replication-set=rs --cluster=rs`)); + + // Adding extra replica set for restore + I.say(await I.verifyCommand(`docker exec rs101 pmm-admin add mongodb --username=pmm --password=pmmpass --port=27017 --service-name=${mongoExtraServiceName} --replication-set=rs1 --cluster=rs1`)); + I.say(await I.verifyCommand(`docker exec rs102 pmm-admin add mongodb --username=pmm --password=pmmpass --port=27017 --service-name=${mongoExtraServiceName2} --replication-set=rs1 --cluster=rs1`)); + I.say(await I.verifyCommand(`docker exec rs103 pmm-admin add mongodb --username=pmm --password=pmmpass --port=27017 --service-name=${mongoExtraServiceName3} --replication-set=rs1 --cluster=rs1`)); }); Before(async ({ @@ -69,6 +87,10 @@ Before(async ({ }); After(async ({ I }) => { + if (mongoClient) { + await mongoClient.close(); + } + await I.verifyCommand('docker exec rs101 systemctl start mongod'); }); @@ -166,7 +188,8 @@ restoreFromDifferentStorageLocationsTests.add([locationsAPI.storageType.s3, 'LOG restoreFromDifferentStorageLocationsTests.add([locationsAPI.storageType.localClient, 'LOGICAL']); Data(restoreFromDifferentStorageLocationsTests).Scenario( - '@PMM-T862 PMM-T1508 @PMM-T1393 @PMM-T1394 @PMM-T1508 @PMM-T1520 @PMM-T1452 PMM-T1583 PMM-T1674 PMM-T1675 Verify user is able to perform MongoDB restore from different storage locations @backup @bm-mongo @bm-fb', + '@PMM-T862 PMM-T1508 @PMM-T1393 @PMM-T1394 @PMM-T1508 @PMM-T1520 @PMM-T1452 PMM-T1583 PMM-T1674 PMM-T1675' + + ' Verify user is able to perform MongoDB restore from different storage locations @backup @bm-mongo @bm-fb', async ({ I, backupInventoryPage, backupAPI, inventoryAPI, restorePage, current, }) => { @@ -233,6 +256,56 @@ Data(restoreFromDifferentStorageLocationsTests).Scenario( }, ).retry(1); +const restoreToDifferentService = new DataTable(['backupType']); + +restoreToDifferentService.add(['LOGICAL']); +restoreToDifferentService.add(['PHYSICAL']); + +Data(restoreToDifferentService).Scenario( + '@PMM-T1773 Verify user is able to perform MongoDB restore to compatible service @backup @bm-mongo @bm-fb', + async ({ + I, backupInventoryPage, backupAPI, inventoryAPI, restorePage, current, + }) => { + const backupName = `mongo-restore-another-replica-${current.backupType}`; + const isLogical = current.backupType === 'LOGICAL'; + + const { service_id } = await inventoryAPI.apiGetNodeInfoByServiceName('MONGODB_SERVICE', mongoServiceName); + const artifactId = await backupAPI.startBackup(backupName, service_id, locationId, false, isLogical); + + await backupAPI.waitForBackupFinish(artifactId); + + I.refreshPage(); + I.waitForVisible(backupInventoryPage.elements.artifactName(backupName), 10); + backupInventoryPage.verifyBackupSucceeded(backupName); + + const artifactName = await I.grabTextFrom(backupInventoryPage.elements.artifactName(backupName)); + const artifact = await backupAPI.getArtifactByName(artifactName); + + if (current.storageType === locationsAPI.storageType.localClient) { + await I.verifyCommand('ls -la /tmp/backup_data/rs', artifact.metadata_list[0].name); + // TODO: add check if the folder is not empty + } + + backupInventoryPage.startRestoreCompatible(artifactName, mongoExtraServiceName); + + I.waitForVisible(restorePage.elements.targetServiceByName(artifactName), 10); + I.seeTextEquals(mongoExtraServiceName, restorePage.elements.targetServiceByName(artifactName)); + await restorePage.waitForRestoreSuccess(artifactName); + + // Wait 30 seconds to have all members restarted + if (current.backupType === 'PHYSICAL') { + I.wait(30); + } + + mongoClient = await I.getMongoClient(mongoConnectionReplica2); + const c = mongoClient.db('test').collection('test'); + + const record = await c.findOne({ number: 2, name: 'Anna' }); + + assert.ok(record === null, `Was expecting to not have a record ${JSON.stringify(record, null, 2)} after restore operation`); + }, +).retry(1); + Scenario( '@PMM-T910 @PMM-T911 Verify delete from storage is selected by default @backup @bm-mongo', async ({ @@ -272,11 +345,11 @@ Scenario( service_id: serviceId, location_id: locationId, cron_expression: '*/2 * * * *', - name: 'schedule_for_restore', + name: `schedule_for_restore_${faker.lorem.word()}`, mode: scheduledAPI.backupModes.snapshot, description: '', retry_interval: '10s', - retries: 1, + retries: 5, enabled: true, retention: 1, }; @@ -309,7 +382,7 @@ Scenario( assert.ok(record === null, `Was expecting to not have a record ${JSON.stringify(record, null, 2)} after restore operation`); }, -); +).retry(1); Scenario( '@PMM-T848 Verify service no longer exists error message during restore @backup @bm-mongo', @@ -481,7 +554,7 @@ Scenario( `Received unexpected logs: \n "${logsText}"`, ); }, -); +).retry(1); const deleteArtifactsTests = new DataTable(['storageType']); diff --git a/tests/backup/pages/api/locationsAPI.js b/tests/backup/pages/api/locationsAPI.js index 62ba65cc9..88be854fe 100644 --- a/tests/backup/pages/api/locationsAPI.js +++ b/tests/backup/pages/api/locationsAPI.js @@ -80,9 +80,9 @@ module.exports = { }; const resp = await I.sendPostRequest('v1/management/backup/Locations/Remove', body, headers); - I.assertEqual( - resp.status, 200, - `Failed to remove storage location with ID "${locationId}". Response message is "${resp.data.message}"`, - ); + // I.assertEqual( + // resp.status, 200, + // `Failed to remove storage location with ID "${locationId}". Response message is "${resp.data.message}"`, + // ); }, }; diff --git a/tests/backup/pages/inventoryPage.js b/tests/backup/pages/inventoryPage.js index b8173b7cc..52c41f58d 100644 --- a/tests/backup/pages/inventoryPage.js +++ b/tests/backup/pages/inventoryPage.js @@ -49,12 +49,14 @@ module.exports = { retryModeOption: (option) => locate('$retry-mode-selector').find('div').at(1).find('label') .withText(option), dataModel: '$dataModel-radio-button', + compatibleServicesLabel: locate('label').withText('Compatible services').inside('$modal-content'), }, fields: { backupName: '$backupName-text-input', vendor: '$vendor-text-input', description: '$description-textarea-input', serviceNameDropdown: locate('div[class$="-select-value-container"]').inside(locate('span').withChild('$service-select-label')), + serviceNameRestoreDropdown: locate('div[class$="-select-value-container"]').inside(locate('div').withChild('$service-select-label')), locationDropdown: '//label[@data-testid="location-field-label"]/parent::div/following-sibling::div[1]//div[contains(@class, "-select-value-container")]', }, messages: { @@ -111,6 +113,17 @@ module.exports = { I.click(this.buttons.modalRestore); }, + startRestoreCompatible(backupName, serviceName) { + I.click(this.buttons.actionsMenuByName(backupName)); + I.waitForVisible(this.buttons.restoreByName(backupName), 2); + I.click(this.buttons.restoreByName(backupName)); + I.waitForVisible(this.buttons.modalRestore, 10); + I.click(this.buttons.compatibleServicesLabel); + I.waitForVisible(this.fields.serviceNameRestoreDropdown, 10); + this.selectDropdownOption(this.fields.serviceNameRestoreDropdown, serviceName); + I.click(this.buttons.modalRestore); + }, + inputRandomBackupName(length = 10) { const backupName = faker.random.alpha(length); diff --git a/tests/ia/pages/ruleTemplatesPage.js b/tests/ia/pages/ruleTemplatesPage.js index c4848a637..2e6c19d2f 100644 --- a/tests/ia/pages/ruleTemplatesPage.js +++ b/tests/ia/pages/ruleTemplatesPage.js @@ -47,7 +47,7 @@ module.exports = { successfullyAdded: 'Alert rule template successfully added', successfullyEdited: 'Alert rule template successfully edited', successfullyDeleted: (name) => `Alert rule template "${name}" successfully deleted.`, - failedToParse: 'Failed to parse rule template.', + failedToParse: 'Failed to parse rule template', failedToDelete: (name) => `You can't delete the "${name}" rule template when it's being used by a rule.`, duplicateTemplate: (id) => `Template with name "${id}" already exists.`, }, @@ -114,9 +114,12 @@ module.exports = { I.click(this.buttons.editButtonByName(templateName)); }, - verifyRuleTemplateContent(content) { + async verifyRuleTemplateContent(content) { I.waitForVisible(this.fields.templateInput, 30); - I.seeInField(this.fields.templateInput, content); + const expected = content.replaceAll(/ +(?= )/g, ''); + const val = (await I.grabValueFrom(this.fields.templateInput)).replaceAll(/ +(?= )/g, ''); + + I.assertEqual(val, expected); }, openAddDialog(templateName) { diff --git a/tests/ia/ruleTemplates_test.js b/tests/ia/ruleTemplates_test.js index 760b92afe..e7c39a1cb 100644 --- a/tests/ia/ruleTemplates_test.js +++ b/tests/ia/ruleTemplates_test.js @@ -186,7 +186,7 @@ Scenario( ruleTemplatesPage.openRuleTemplatesTab(); ruleTemplatesPage.openEditDialog(templateName); - ruleTemplatesPage.verifyRuleTemplateContent(fileContent); + await ruleTemplatesPage.verifyRuleTemplateContent(fileContent); I.seeElementsDisabled(ruleTemplatesPage.buttons.editTemplate); I.clearField(ruleTemplatesPage.fields.templateInput); I.fillField(ruleTemplatesPage.fields.templateInput, updatedTemplateText); @@ -195,7 +195,7 @@ Scenario( I.click(ruleTemplatesPage.buttons.editTemplate); I.verifyPopUpMessage(ruleTemplatesPage.messages.successfullyEdited); ruleTemplatesPage.openEditDialog(newTemplateName); - ruleTemplatesPage.verifyRuleTemplateContent(updatedTemplateText); + await ruleTemplatesPage.verifyRuleTemplateContent(updatedTemplateText); // Checking Updated Rule template name in modal header ruleTemplatesPage.verifyEditModalHeaderAndWarning(newTemplateName); diff --git a/tests/ia/templates/template.yaml b/tests/ia/templates/template.yaml index 60431323a..3a646d9b1 100644 --- a/tests/ia/templates/template.yaml +++ b/tests/ia/templates/template.yaml @@ -1,4 +1,3 @@ ---- templates: - name: test_user_rule_yaml version: 1 diff --git a/tests/qa-integration/pmm_pgsm_integration_test.js b/tests/qa-integration/pmm_pgsm_integration_test.js index 6a0aad034..79cd4436a 100644 --- a/tests/qa-integration/pmm_pgsm_integration_test.js +++ b/tests/qa-integration/pmm_pgsm_integration_test.js @@ -21,7 +21,10 @@ const pgsm_service_name_socket = `socket_${container}_${version}_service`; const container_name = `${container}_${version}`; const percentageDiff = (a, b) => (a - b === 0 ? 0 : 100 * Math.abs((a - b) / b)); -const labels = [{ key: 'database', value: [`${database}`] }]; +const labels = [ + { key: 'database', value: [database] }, + { key: 'service_name', value: [pgsm_service_name] }, +]; const filters = new DataTable(['filterSection', 'filterToApply']); @@ -85,7 +88,7 @@ Scenario( // wait for pmm-agent to push the execution as part of next bucket to clickhouse await I.verifyCommand(`docker exec ${container_name} pmm-admin list | grep "postgresql_pgstatmonitor_agent" | grep "Running"`); - I.wait(5); + I.wait(30); let pgsm_output; if (version < 13) { @@ -136,7 +139,7 @@ Scenario( assert.ok(response.data.metrics.query_time.cnt === query_cnt, `Expected Total Query Count Metrics to be same for query ${query} with id as ${queryid} found in clickhouse as ${response.data.metrics.query_time.cnt} while pgsm has value as ${query_cnt}`); } }, -); +).retry(1); Data(filters).Scenario( 'PMM-T1261 - Verify the "Command type" filter for Postgres @not-ui-pipeline @pgsm-pmm-integration', @@ -191,9 +194,10 @@ Scenario( dashboardPage.verifyMetricsExistence(dashboardPage.postgresqlInstanceSummaryDashboard.metrics); await dashboardPage.verifyThereAreNoGraphsWithNA(); await dashboardPage.verifyThereAreNoGraphsWithoutData(1); - let log = await I.verifyCommand(`docker exec ${container_name} cat pmm-agent.log`); + const log = await I.verifyCommand(`docker exec ${container_name} cat pmm-agent.log`); + I.assertFalse(log.includes('Error opening connection to database \(postgres'), - 'The log wasn\'t supposed to contain errors regarding connection to postgress database but it does') + 'The log wasn\'t supposed to contain errors regarding connection to postgress database but it does'); }, ); @@ -221,7 +225,11 @@ Scenario( // wait for pmm-agent to push the execution as part of next bucket to clickhouse await I.verifyCommand(`docker exec ${container_name} pmm-admin list | grep "postgresql_pgstatmonitor_agent" | grep "Running"`); - const labels = [{ key: 'database', value: [`${db}`] }]; + const labels = [ + { key: 'database', value: [`${db}`] }, + { key: 'service_name', value: [pgsm_service_name] }, + ]; + const excluded_queries = [ 'SELECT version()', 'SELECT /* pmm-agent:pgstatmonitor */ version()', @@ -255,6 +263,8 @@ Scenario( 'BEGIN', 'COMMIT', ]; + + I.wait(30); let pgsm_output; if (version < 13) { @@ -306,7 +316,7 @@ Scenario( assert.ok(response.data.metrics.query_time.cnt === query_cnt, `Expected Total Query Count Metrics to be same for query ${query} with id as ${queryid} found in clickhouse as ${response.data.metrics.query_time.cnt} while pgsm has value as ${query_cnt}`); } }, -); +).retry(1); Scenario( 'PMM-T1063 - Verify Application Name with pg_stat_monitor @pgsm-pmm-integration @not-ui-pipeline',