PMM-12352 - restore to different mongo rs (#661)

* PMM-12352 - restore to different mongo rs * PMM-12352 - fix more tests * PMM-12352 - remove retries * PMM-12352 - tune retries * PMM-12352 - tune retries * PMM-12352 - tune retries * PMM-12352 - tune retries * PMM-12352 - trigger * PMM-12352 - trigger * PMM-12352 - fix retry test * PMM-12352 - fix pgsm test * PMM-12352 - retry for pgsm * PMM-12352 - pgsm labels * PMM-12352 - trigger
percona · Aug 3, 2023 · d66ba14 · d66ba14
1 parent 09ccc4f
commit d66ba14
Show file tree

Hide file tree

Showing 7 changed files with 120 additions and 22 deletions.
diff --git a/tests/backup/inventory_test.js b/tests/backup/inventory_test.js
@@ -18,12 +18,24 @@ const mongoServiceName = 'mongo-backup-inventory-1';
 const mongoServiceName2 = 'mongo-backup-inventory-2';
 const mongoServiceName3 = 'mongo-backup-inventory-3';
 
+const mongoExtraServiceName = 'mongo-extra-1';
+const mongoExtraServiceName2 = 'mongo-extra-2';
+const mongoExtraServiceName3 = 'mongo-extra-3';
+
+let mongoClient;
+
 const mongoConnection = {
   username: 'pmm',
   password: 'pmmpass',
   port: 27027,
 };
 
+const mongoConnectionReplica2 = {
+  username: 'pmm',
+  password: 'pmmpass',
+  port: 27037,
+};
+
 Feature('BM: Backup Inventory');
 
 BeforeSuite(async ({
@@ -42,11 +54,17 @@ BeforeSuite(async ({
     locationsAPI.storageLocationConnection,
     location.description,
   );
+
   await I.mongoConnect(mongoConnection);
 
   I.say(await I.verifyCommand(`docker exec rs101 pmm-admin add mongodb --username=pmm --password=pmmpass --port=27017 --service-name=${mongoServiceName} --replication-set=rs --cluster=rs`));
   I.say(await I.verifyCommand(`docker exec rs102 pmm-admin add mongodb --username=pmm --password=pmmpass --port=27017 --service-name=${mongoServiceName2} --replication-set=rs --cluster=rs`));
   I.say(await I.verifyCommand(`docker exec rs103 pmm-admin add mongodb --username=pmm --password=pmmpass --port=27017 --service-name=${mongoServiceName3} --replication-set=rs --cluster=rs`));
+
+  // Adding extra replica set for restore
+  I.say(await I.verifyCommand(`docker exec rs101 pmm-admin add mongodb --username=pmm --password=pmmpass --port=27017 --service-name=${mongoExtraServiceName} --replication-set=rs1 --cluster=rs1`));
+  I.say(await I.verifyCommand(`docker exec rs102 pmm-admin add mongodb --username=pmm --password=pmmpass --port=27017 --service-name=${mongoExtraServiceName2} --replication-set=rs1 --cluster=rs1`));
+  I.say(await I.verifyCommand(`docker exec rs103 pmm-admin add mongodb --username=pmm --password=pmmpass --port=27017 --service-name=${mongoExtraServiceName3} --replication-set=rs1 --cluster=rs1`));
 });
 
 Before(async ({
@@ -69,6 +87,10 @@ Before(async ({
 });
 
 After(async ({ I }) => {
+  if (mongoClient) {
+    await mongoClient.close();
+  }
+
   await I.verifyCommand('docker exec rs101 systemctl start mongod');
 });
 
@@ -166,7 +188,8 @@ restoreFromDifferentStorageLocationsTests.add([locationsAPI.storageType.s3, 'LOG
 restoreFromDifferentStorageLocationsTests.add([locationsAPI.storageType.localClient, 'LOGICAL']);
 
 Data(restoreFromDifferentStorageLocationsTests).Scenario(
-  '@PMM-T862 PMM-T1508 @PMM-T1393 @PMM-T1394 @PMM-T1508 @PMM-T1520 @PMM-T1452 PMM-T1583 PMM-T1674 PMM-T1675 Verify user is able to perform MongoDB restore from different storage locations @backup @bm-mongo @bm-fb',
+  '@PMM-T862 PMM-T1508 @PMM-T1393 @PMM-T1394 @PMM-T1508 @PMM-T1520 @PMM-T1452 PMM-T1583 PMM-T1674 PMM-T1675'
+  + ' Verify user is able to perform MongoDB restore from different storage locations @backup @bm-mongo @bm-fb',
   async ({
     I, backupInventoryPage, backupAPI, inventoryAPI, restorePage, current,
   }) => {
@@ -233,6 +256,56 @@ Data(restoreFromDifferentStorageLocationsTests).Scenario(
   },
 ).retry(1);
 
+const restoreToDifferentService = new DataTable(['backupType']);
+
+restoreToDifferentService.add(['LOGICAL']);
+restoreToDifferentService.add(['PHYSICAL']);
+
+Data(restoreToDifferentService).Scenario(
+  '@PMM-T1773 Verify user is able to perform MongoDB restore to compatible service @backup @bm-mongo @bm-fb',
+  async ({
+    I, backupInventoryPage, backupAPI, inventoryAPI, restorePage, current,
+  }) => {
+    const backupName = `mongo-restore-another-replica-${current.backupType}`;
+    const isLogical = current.backupType === 'LOGICAL';
+
+    const { service_id } = await inventoryAPI.apiGetNodeInfoByServiceName('MONGODB_SERVICE', mongoServiceName);
+    const artifactId = await backupAPI.startBackup(backupName, service_id, locationId, false, isLogical);
+
+    await backupAPI.waitForBackupFinish(artifactId);
+
+    I.refreshPage();
+    I.waitForVisible(backupInventoryPage.elements.artifactName(backupName), 10);
+    backupInventoryPage.verifyBackupSucceeded(backupName);
+
+    const artifactName = await I.grabTextFrom(backupInventoryPage.elements.artifactName(backupName));
+    const artifact = await backupAPI.getArtifactByName(artifactName);
+
+    if (current.storageType === locationsAPI.storageType.localClient) {
+      await I.verifyCommand('ls -la /tmp/backup_data/rs', artifact.metadata_list[0].name);
+      // TODO: add check if the folder is not empty
+    }
+
+    backupInventoryPage.startRestoreCompatible(artifactName, mongoExtraServiceName);
+
+    I.waitForVisible(restorePage.elements.targetServiceByName(artifactName), 10);
+    I.seeTextEquals(mongoExtraServiceName, restorePage.elements.targetServiceByName(artifactName));
+    await restorePage.waitForRestoreSuccess(artifactName);
+
+    // Wait 30 seconds to have all members restarted
+    if (current.backupType === 'PHYSICAL') {
+      I.wait(30);
+    }
+
+    mongoClient = await I.getMongoClient(mongoConnectionReplica2);
+    const c = mongoClient.db('test').collection('test');
+
+    const record = await c.findOne({ number: 2, name: 'Anna' });
+
+    assert.ok(record === null, `Was expecting to not have a record ${JSON.stringify(record, null, 2)} after restore operation`);
+  },
+).retry(1);
+
 Scenario(
   '@PMM-T910 @PMM-T911 Verify delete from storage is selected by default @backup @bm-mongo',
   async ({
@@ -272,11 +345,11 @@ Scenario(
       service_id: serviceId,
       location_id: locationId,
       cron_expression: '*/2 * * * *',
-      name: 'schedule_for_restore',
+      name: `schedule_for_restore_${faker.lorem.word()}`,
       mode: scheduledAPI.backupModes.snapshot,
       description: '',
       retry_interval: '10s',
-      retries: 1,
+      retries: 5,
       enabled: true,
       retention: 1,
     };
@@ -309,7 +382,7 @@ Scenario(
 
     assert.ok(record === null, `Was expecting to not have a record ${JSON.stringify(record, null, 2)} after restore operation`);
   },
-);
+).retry(1);
 
 Scenario(
   '@PMM-T848 Verify service no longer exists error message during restore @backup @bm-mongo',
@@ -481,7 +554,7 @@ Scenario(
       `Received unexpected logs: \n "${logsText}"`,
     );
   },
-);
+).retry(1);
 
 const deleteArtifactsTests = new DataTable(['storageType']);
 

diff --git a/tests/backup/pages/api/locationsAPI.js b/tests/backup/pages/api/locationsAPI.js
@@ -80,9 +80,9 @@ module.exports = {
     };
     const resp = await I.sendPostRequest('v1/management/backup/Locations/Remove', body, headers);
 
-    I.assertEqual(
-      resp.status, 200,
-      `Failed to remove storage location with ID "${locationId}". Response message is "${resp.data.message}"`,
-    );
+    // I.assertEqual(
+    //   resp.status, 200,
+    //   `Failed to remove storage location with ID "${locationId}". Response message is "${resp.data.message}"`,
+    // );
   },
 };
diff --git a/tests/backup/pages/inventoryPage.js b/tests/backup/pages/inventoryPage.js
@@ -49,12 +49,14 @@ module.exports = {
     retryModeOption: (option) => locate('$retry-mode-selector').find('div').at(1).find('label')
       .withText(option),
     dataModel: '$dataModel-radio-button',
+    compatibleServicesLabel: locate('label').withText('Compatible services').inside('$modal-content'),
   },
   fields: {
     backupName: '$backupName-text-input',
     vendor: '$vendor-text-input',
     description: '$description-textarea-input',
     serviceNameDropdown: locate('div[class$="-select-value-container"]').inside(locate('span').withChild('$service-select-label')),
+    serviceNameRestoreDropdown: locate('div[class$="-select-value-container"]').inside(locate('div').withChild('$service-select-label')),
     locationDropdown: '//label[@data-testid="location-field-label"]/parent::div/following-sibling::div[1]//div[contains(@class, "-select-value-container")]',
   },
   messages: {
@@ -111,6 +113,17 @@ module.exports = {
     I.click(this.buttons.modalRestore);
   },
 
+  startRestoreCompatible(backupName, serviceName) {
+    I.click(this.buttons.actionsMenuByName(backupName));
+    I.waitForVisible(this.buttons.restoreByName(backupName), 2);
+    I.click(this.buttons.restoreByName(backupName));
+    I.waitForVisible(this.buttons.modalRestore, 10);
+    I.click(this.buttons.compatibleServicesLabel);
+    I.waitForVisible(this.fields.serviceNameRestoreDropdown, 10);
+    this.selectDropdownOption(this.fields.serviceNameRestoreDropdown, serviceName);
+    I.click(this.buttons.modalRestore);
+  },
+
   inputRandomBackupName(length = 10) {
     const backupName = faker.random.alpha(length);
 

diff --git a/tests/ia/pages/ruleTemplatesPage.js b/tests/ia/pages/ruleTemplatesPage.js
@@ -47,7 +47,7 @@ module.exports = {
     successfullyAdded: 'Alert rule template successfully added',
     successfullyEdited: 'Alert rule template successfully edited',
     successfullyDeleted: (name) => `Alert rule template "${name}" successfully deleted.`,
-    failedToParse: 'Failed to parse rule template.',
+    failedToParse: 'Failed to parse rule template',
     failedToDelete: (name) => `You can't delete the "${name}" rule template when it's being used by a rule.`,
     duplicateTemplate: (id) => `Template with name "${id}" already exists.`,
   },
@@ -114,9 +114,12 @@ module.exports = {
     I.click(this.buttons.editButtonByName(templateName));
   },
 
-  verifyRuleTemplateContent(content) {
+  async verifyRuleTemplateContent(content) {
     I.waitForVisible(this.fields.templateInput, 30);
-    I.seeInField(this.fields.templateInput, content);
+    const expected = content.replaceAll(/ +(?= )/g, '');
+    const val = (await I.grabValueFrom(this.fields.templateInput)).replaceAll(/ +(?= )/g, '');
+
+    I.assertEqual(val, expected);
   },
 
   openAddDialog(templateName) {

diff --git a/tests/ia/ruleTemplates_test.js b/tests/ia/ruleTemplates_test.js
@@ -186,7 +186,7 @@ Scenario(
 
     ruleTemplatesPage.openRuleTemplatesTab();
     ruleTemplatesPage.openEditDialog(templateName);
-    ruleTemplatesPage.verifyRuleTemplateContent(fileContent);
+    await ruleTemplatesPage.verifyRuleTemplateContent(fileContent);
     I.seeElementsDisabled(ruleTemplatesPage.buttons.editTemplate);
     I.clearField(ruleTemplatesPage.fields.templateInput);
     I.fillField(ruleTemplatesPage.fields.templateInput, updatedTemplateText);
@@ -195,7 +195,7 @@ Scenario(
     I.click(ruleTemplatesPage.buttons.editTemplate);
     I.verifyPopUpMessage(ruleTemplatesPage.messages.successfullyEdited);
     ruleTemplatesPage.openEditDialog(newTemplateName);
-    ruleTemplatesPage.verifyRuleTemplateContent(updatedTemplateText);
+    await ruleTemplatesPage.verifyRuleTemplateContent(updatedTemplateText);
 
     // Checking Updated Rule template name in modal header
     ruleTemplatesPage.verifyEditModalHeaderAndWarning(newTemplateName);

diff --git a/tests/ia/templates/template.yaml b/tests/ia/templates/template.yaml
@@ -1,4 +1,3 @@
----
 templates:
   - name: test_user_rule_yaml
     version: 1

diff --git a/tests/qa-integration/pmm_pgsm_integration_test.js b/tests/qa-integration/pmm_pgsm_integration_test.js
@@ -21,7 +21,10 @@ const pgsm_service_name_socket = `socket_${container}_${version}_service`;
 const container_name = `${container}_${version}`;
 const percentageDiff = (a, b) => (a - b === 0 ? 0 : 100 * Math.abs((a - b) / b));
 
-const labels = [{ key: 'database', value: [`${database}`] }];
+const labels = [
+  { key: 'database', value: [database] },
+  { key: 'service_name', value: [pgsm_service_name] },
+];
 
 const filters = new DataTable(['filterSection', 'filterToApply']);
 
@@ -85,7 +88,7 @@ Scenario(
     // wait for pmm-agent to push the execution as part of next bucket to clickhouse
     await I.verifyCommand(`docker exec ${container_name} pmm-admin list | grep "postgresql_pgstatmonitor_agent" | grep "Running"`);
 
-    I.wait(5);
+    I.wait(30);
     let pgsm_output;
 
     if (version < 13) {
@@ -136,7 +139,7 @@ Scenario(
       assert.ok(response.data.metrics.query_time.cnt === query_cnt, `Expected Total Query Count Metrics to be same for query ${query} with id as ${queryid} found in clickhouse as ${response.data.metrics.query_time.cnt} while pgsm has value as ${query_cnt}`);
     }
   },
-);
+).retry(1);
 
 Data(filters).Scenario(
   'PMM-T1261 - Verify the "Command type" filter for Postgres @not-ui-pipeline @pgsm-pmm-integration',
@@ -191,9 +194,10 @@ Scenario(
     dashboardPage.verifyMetricsExistence(dashboardPage.postgresqlInstanceSummaryDashboard.metrics);
     await dashboardPage.verifyThereAreNoGraphsWithNA();
     await dashboardPage.verifyThereAreNoGraphsWithoutData(1);
-    let log = await I.verifyCommand(`docker exec ${container_name} cat pmm-agent.log`);
+    const log = await I.verifyCommand(`docker exec ${container_name} cat pmm-agent.log`);
+
     I.assertFalse(log.includes('Error opening connection to database \(postgres'),
-      'The log wasn\'t supposed to contain errors regarding connection to postgress database but it does')
+      'The log wasn\'t supposed to contain errors regarding connection to postgress database but it does');
   },
 );
 
@@ -221,7 +225,11 @@ Scenario(
     // wait for pmm-agent to push the execution as part of next bucket to clickhouse
     await I.verifyCommand(`docker exec ${container_name} pmm-admin list | grep "postgresql_pgstatmonitor_agent" | grep "Running"`);
 
-    const labels = [{ key: 'database', value: [`${db}`] }];
+    const labels = [
+      { key: 'database', value: [`${db}`] },
+      { key: 'service_name', value: [pgsm_service_name] },
+    ];
+
     const excluded_queries = [
       'SELECT version()',
       'SELECT /* pmm-agent:pgstatmonitor */ version()',
@@ -255,6 +263,8 @@ Scenario(
       'BEGIN',
       'COMMIT',
     ];
+
+    I.wait(30);
     let pgsm_output;
 
     if (version < 13) {
@@ -306,7 +316,7 @@ Scenario(
       assert.ok(response.data.metrics.query_time.cnt === query_cnt, `Expected Total Query Count Metrics to be same for query ${query} with id as ${queryid} found in clickhouse as ${response.data.metrics.query_time.cnt} while pgsm has value as ${query_cnt}`);
     }
   },
-);
+).retry(1);
 
 Scenario(
   'PMM-T1063 - Verify Application Name with pg_stat_monitor @pgsm-pmm-integration @not-ui-pipeline',