From b7b19314a4c372b61350184795b29eae72da294c Mon Sep 17 00:00:00 2001 From: alallema Date: Tue, 30 May 2023 12:19:21 +0200 Subject: [PATCH] Adding qualified version for analytics --- .github/scripts/check-release.sh | 13 +++++++ .github/workflows/publish-docker-image.yml | 3 ++ scraper/src/config/config_loader.py | 3 +- scraper/src/config/version.py | 9 +++++ scraper/src/meilisearch_helper.py | 4 +- tests/config_loader/basic_test.py | 12 ++++++ tests/meilisearch_helper/__init__.py | 0 tests/meilisearch_helper/common.py | 21 +++++++++++ .../meilisearch_helper_test.py | 37 +++++++++++++++++++ tests/meilisearch_helper/version_test.py | 9 +++++ 10 files changed, 109 insertions(+), 2 deletions(-) create mode 100644 .github/scripts/check-release.sh create mode 100644 scraper/src/config/version.py create mode 100644 tests/meilisearch_helper/__init__.py create mode 100644 tests/meilisearch_helper/common.py create mode 100644 tests/meilisearch_helper/meilisearch_helper_test.py create mode 100644 tests/meilisearch_helper/version_test.py diff --git a/.github/scripts/check-release.sh b/.github/scripts/check-release.sh new file mode 100644 index 0000000..367cb6b --- /dev/null +++ b/.github/scripts/check-release.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +# Checking if current tag matches the package version +current_tag=$(echo $GITHUB_REF | cut -d '/' -f 3 | sed -r 's/^v//') +file_tag=$(grep '__version__ =' meilisearch/version.py | cut -d '=' -f 2- | tr -d ' ' | tr -d '"' | tr -d ',') +if [ "$current_tag" != "$file_tag" ]; then + echo "Error: the current tag does not match the version in package file(s)." + echo "$current_tag vs $file_tag" + exit 1 +fi + +echo 'OK' +exit 0 diff --git a/.github/workflows/publish-docker-image.yml b/.github/workflows/publish-docker-image.yml index 00c5dbf..4088239 100644 --- a/.github/workflows/publish-docker-image.yml +++ b/.github/workflows/publish-docker-image.yml @@ -13,6 +13,9 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 + - name: Check release validity + run: sh .github/scripts/check-release.sh + - name: Set up QEMU uses: docker/setup-qemu-action@v2 diff --git a/scraper/src/config/config_loader.py b/scraper/src/config/config_loader.py index 4a157d9..6f81490 100644 --- a/scraper/src/config/config_loader.py +++ b/scraper/src/config/config_loader.py @@ -10,6 +10,7 @@ import os import copy +from .version import qualified_version from .config_validator import ConfigValidator from .urls_parser import UrlsParser from .selectors_parser import SelectorsParser @@ -44,7 +45,7 @@ class ConfigLoader: strict_redirect = True strip_chars = ".,;:§¶" use_anchors = False - user_agent = 'Meilisearch docs-scraper' + user_agent = qualified_version() only_content_level = False query_rules = [] diff --git a/scraper/src/config/version.py b/scraper/src/config/version.py new file mode 100644 index 0000000..95acfe3 --- /dev/null +++ b/scraper/src/config/version.py @@ -0,0 +1,9 @@ +from __future__ import annotations + +__version__ = "0.12.9" + + +def qualified_version() -> str: + """Get the qualified version of this module.""" + + return f"Meilisearch DocsScraper (v{__version__})" diff --git a/scraper/src/meilisearch_helper.py b/scraper/src/meilisearch_helper.py index cf9ee1f..a9c058c 100644 --- a/scraper/src/meilisearch_helper.py +++ b/scraper/src/meilisearch_helper.py @@ -4,6 +4,8 @@ import meilisearch from builtins import range +from .config.version import qualified_version + def remove_bad_encoding(value): return value.replace(''', "'") @@ -99,7 +101,7 @@ class MeiliSearchHelper: } def __init__(self, host_url, api_key, index_uid, custom_settings): - self.meilisearch_client = meilisearch.Client(host_url, api_key) + self.meilisearch_client = meilisearch.Client(host_url, api_key, client_agents=(qualified_version(),)) self.meilisearch_index = self.meilisearch_client.index(index_uid) self.delete_index() self.add_settings(MeiliSearchHelper.SETTINGS, custom_settings) diff --git a/tests/config_loader/basic_test.py b/tests/config_loader/basic_test.py index 4bddbed..195f039 100644 --- a/tests/config_loader/basic_test.py +++ b/tests/config_loader/basic_test.py @@ -1,4 +1,6 @@ # coding: utf-8 +import re +from scraper.src.config.version import __version__, qualified_version from scraper.src.config.config_loader import ConfigLoader from .abstract import config import pytest @@ -48,3 +50,13 @@ def test_excpetion_when_shadowing_(self): with pytest.raises(Exception): ConfigLoader(c) + + def test_get_qualified_version(self): + """ Old variable scrap_url must be spread to scrape_url. If one is defined, the previous one must be used""" + c = config({ + 'user_agent': qualified_version() + }) + + config_loaded = ConfigLoader(c) + + assert config_loaded.user_agent == f"Meilisearch DocsScraper (v{__version__})" diff --git a/tests/meilisearch_helper/__init__.py b/tests/meilisearch_helper/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/meilisearch_helper/common.py b/tests/meilisearch_helper/common.py new file mode 100644 index 0000000..b3f40ee --- /dev/null +++ b/tests/meilisearch_helper/common.py @@ -0,0 +1,21 @@ +MASTER_KEY = "masterKey" +BASE_URL = "http://127.0.0.1:7700" + +DEFAULT_INDEX = 'index_uid' + +DEFAULT_DATA_DELETE = { + "taskUid": 1, + "indexUid": DEFAULT_INDEX, + "status": "enqueued", + "type": "indexDeletion", + "enqueuedAt": "2023-03-30T13:24:01.789654093Z" + } +DEFAULT_DATA_PATCH = { + "taskUid": 1, + "indexUid": DEFAULT_INDEX, + "status": "enqueued", + "type": "settingsUpdate", + "enqueuedAt": "2023-03-30T13:24:01.789654093Z" + } + +DEFAULT_ACCEPTED_STATUS = 202 diff --git a/tests/meilisearch_helper/meilisearch_helper_test.py b/tests/meilisearch_helper/meilisearch_helper_test.py new file mode 100644 index 0000000..6c633a8 --- /dev/null +++ b/tests/meilisearch_helper/meilisearch_helper_test.py @@ -0,0 +1,37 @@ +# coding: utf-8 +import json +from unittest.mock import patch + +import requests + +from scraper.src.meilisearch_helper import MeiliSearchHelper +from scraper.src.config.version import __version__ +from tests.meilisearch_helper import common + + +class TestMeilisearchHelper: + @patch("requests.delete") + @patch("requests.patch") + def test_use_meilisearch_default(self, mock_delete, mock_patch): + mock_delete.configure_mock(__name__="delete") + mock_response = requests.models.Response() + mock_response.status_code = common.DEFAULT_ACCEPTED_STATUS + mock_response._content = json.dumps(common.DEFAULT_DATA_DELETE).encode('utf-8') + mock_delete.return_value = mock_response + + mock_patch.configure_mock(__name__="patch") + mock_response = requests.models.Response() + mock_response.status_code = common.DEFAULT_ACCEPTED_STATUS + mock_response._content = json.dumps(common.DEFAULT_DATA_DELETE).encode('utf-8') + mock_patch.return_value = mock_response + """ Should set the `User-Agent` doscraper by default """ + # When + actual = MeiliSearchHelper( + common.BASE_URL, + common.MASTER_KEY, + common.DEFAULT_INDEX, + MeiliSearchHelper.SETTINGS + ) + + # Then + assert actual.meilisearch_client.http.headers['User-Agent'] == f"Meilisearch Python (v0.27.0);Meilisearch DocsScraper (v{__version__})" diff --git a/tests/meilisearch_helper/version_test.py b/tests/meilisearch_helper/version_test.py new file mode 100644 index 0000000..8404a72 --- /dev/null +++ b/tests/meilisearch_helper/version_test.py @@ -0,0 +1,9 @@ +# coding: utf-8 +import re + +from scraper.src.config.version import __version__ + + +class TestInit: + def test_get_version(self): + assert re.match(r"^(\d+\.)?(\d+\.)?(\*|\d+)$", __version__)