Refactor BaseTest and ScraperTestHelper to streamline index UID handling

- Updated BaseTest to generate and manage a random index UID internally. - Modified ScraperTestHelper to accept the index UID in the constructor and removed redundant parameters from methods. - Adjusted test cases to reflect changes in method signatures and ensure proper functionality without passing the index UID explicitly. This refactor enhances code clarity and reduces the risk of errors related to index UID management.
meilisearch · Nov 30, 2024 · 6ec776c · 6ec776c
1 parent 36bde9e
commit 6ec776c
Show file tree

Hide file tree

Showing 3 changed files with 63 additions and 54 deletions.
diff --git a/test/helpers/BaseTest.ts b/test/helpers/BaseTest.ts
@@ -12,11 +12,13 @@ export interface TestMetrics {
 
 export class BaseTest {
   public helper: ScraperTestHelper;
-  public currentIndexUid: string | null = null;
+  public currentIndexUid: string;
   private metrics: TestMetrics | null = null;
 
   constructor() {
-    this.helper = new ScraperTestHelper();
+    this.currentIndexUid = this.generateRandomIndexUid();
+
+    this.helper = new ScraperTestHelper(this.currentIndexUid);
   }
 
   async setup() {
@@ -29,7 +31,7 @@ export class BaseTest {
       if (this.metrics) {
         await this.saveMetrics();
       }
-      await this.helper.deleteIndex(this.currentIndexUid);
+      await this.helper.deleteIndex();
     }
   }
 
@@ -47,18 +49,16 @@ export class BaseTest {
   public async runScraper(config: Config, useAsync = false) {
     const startTime = Date.now();
 
-    // Ensure we have a random index UID
-    config.meilisearch_index_uid = this.generateRandomIndexUid();
-    this.currentIndexUid = config.meilisearch_index_uid;
+    config.meilisearch_index_uid = this.currentIndexUid;
 
     await this.helper.startScraping(config, useAsync);
 
     if (useAsync) {
-      await this.helper.waitForScrapingToComplete(this.currentIndexUid);
+      await this.helper.waitForScrapingToComplete();
     }
 
     const endTime = Date.now();
-    const stats = await this.helper.getStats(this.currentIndexUid);
+    const stats = await this.helper.getStats();
 
     this.metrics = {
       testName: expect.getState().currentTestName || "unknown",

diff --git a/test/helpers/ScraperTestHelper.ts b/test/helpers/ScraperTestHelper.ts
@@ -5,8 +5,10 @@ import { MeiliSearch } from "meilisearch";
 export class ScraperTestHelper {
   private meiliClient: MeiliSearch;
   private scraperUrl: string;
+  private indexUid: string;
 
-  constructor() {
+  constructor(indexUid: string) {
+    this.indexUid = indexUid;
     this.meiliClient = new MeiliSearch({
       host: process.env.MEILI_HOST || "http://localhost:7700",
       apiKey: process.env.MEILI_MASTER_KEY || "masterKey",
@@ -44,14 +46,11 @@ export class ScraperTestHelper {
     }
   }
 
-  async waitForScrapingToComplete(
-    indexUid: string,
-    timeoutMs = 30000
-  ): Promise<void> {
+  async waitForScrapingToComplete(timeoutMs = 30000): Promise<void> {
     const startTime = Date.now();
 
     while (Date.now() - startTime < timeoutMs) {
-      const tasks = await this.meiliClient.index(indexUid).getTasks();
+      const tasks = await this.meiliClient.index(this.indexUid).getTasks();
       const pendingTasks = tasks.results.filter(
         (task) => task.status !== "succeeded" && task.status !== "failed"
       );
@@ -67,64 +66,73 @@ export class ScraperTestHelper {
   }
 
   async getSearchResults(
-    indexUid: string,
-    query: string,
+    query: string = "",
+    options: Record<string, any> | undefined = undefined,
     maxAttempts = 10,
     delayMs = 1000
   ): Promise<any> {
     return await this.retryOperation(
       async () => {
-        const response = await this.meiliClient.index(indexUid).search(query);
-        return response;
+        const response = await this.meiliClient
+          .index(this.indexUid)
+          .search(query, options);
+
+        if (response.hits.length === 0) {
+          throw new Error("No search results found");
+        } else {
+          return response.hits;
+        }
       },
       maxAttempts,
       delayMs
     );
   }
 
-  async debugAllIndexes() {
-    const indexes = await this.meiliClient.getIndexes();
-    console.log("Indexes:", indexes);
+  async debugStats() {
+    const stats = await this.getStats();
+    console.log("Stats:", stats);
   }
 
-  async debugAllStats() {
-    const stats = await this.meiliClient.getStats();
-    console.log("Stats:", stats);
+  async debugSearchResults(
+    query: string = "",
+    options: Record<string, any> | undefined = undefined
+  ) {
+    const searchResults = await this.getSearchResults(query, options);
+    console.log("Search Results:", searchResults);
   }
 
-  async getStats(
-    indexUid: string,
-    maxAttempts = 10,
-    delayMs = 1000
-  ): Promise<any> {
+  async debugSettings() {
+    const settings = await this.getSettings();
+    console.log("Settings:", settings);
+  }
+
+  async getStats(maxAttempts = 10, delayMs = 1000): Promise<any> {
     return await this.retryOperation(
       async () => {
-        const stats = await this.meiliClient.index(indexUid).getStats();
+        const stats = await this.meiliClient.index(this.indexUid).getStats();
         return stats;
       },
       maxAttempts,
       delayMs
     );
   }
 
-  async getSettings(
-    indexUid: string,
-    maxAttempts = 10,
-    delayMs = 1000
-  ): Promise<any> {
+  async getSettings(maxAttempts = 10, delayMs = 1000): Promise<any> {
     return await this.retryOperation(
       async () => {
-        const settings = await this.meiliClient.index(indexUid).getSettings();
+        const settings = await this.meiliClient
+          .index(this.indexUid)
+          .getSettings();
         return settings;
       },
       maxAttempts,
       delayMs
     );
   }
 
-  async deleteIndex(indexUid: string) {
+  async deleteIndex() {
     try {
-      await this.meiliClient.deleteIndex(indexUid);
+      await this.meiliClient.deleteIndex(this.indexUid);
     } catch (error) {
       // Ignore if index doesn't exist
     }

diff --git a/test/integration/blog-scraping.test.ts b/test/integration/blog-scraping.test.ts
@@ -13,33 +13,34 @@ describe("Blog Scraping", () => {
     return testInstance.teardown();
   });
 
-  const runScrapingTest = async (useAsync = false) => {
+  it("scrap blog posts with no options", async () => {
     const config = createTestConfig({
       start_urls: ["http://playground:3000/blog"],
     });
 
-    await testInstance.runScraper(config, useAsync);
-
-    await testInstance.helper.debugAllIndexes();
-    await testInstance.helper.debugAllStats();
+    await testInstance.runScraper(config);
 
     // Test search results
-    const searchResults = await testInstance.helper.getSearchResults(
-      testInstance.currentIndexUid!,
-      ""
-    );
-    expect(searchResults?.hits).toHaveSearchResult({
+    const searchResults = await testInstance.helper.getSearchResults();
+    expect(searchResults).toHaveSearchResult({
       h3: "The Art and History of Camembert Cheese",
     });
 
     // Test stats
-    const stats = await testInstance.helper.getStats(
-      testInstance.currentIndexUid!
-    );
+    const stats = await testInstance.helper.getStats();
     expect(stats).toHaveDocumentCount(3); // Assuming 3 blog posts
-  };
+  });
+
+  it("scrap blog posts with strategy schema", async () => {
+    const config = createTestConfig({
+      start_urls: ["http://playground:3000/blog"],
+      strategy: "schema",
+    });
+
+    await testInstance.runScraper(config);
 
-  it("should scrape blog posts correctly", async () => {
-    await runScrapingTest();
+    await testInstance.helper.debugStats();
+    await testInstance.helper.debugSearchResults();
+    await testInstance.helper.debugSettings();
   });
 });