Merge pull request #10 from maxbachmann/master

replace python-Levenshtein with rapidfuzz

.github/workflows/ci.yml

@@ -11,8 +11,6 @@ jobs:
         python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
         test-cmd: [pytest]
         include:
-          #- python-version: pyp-y3.8
-          #  test-cmd: pytest test_thefuzz.py test_thefuzz_pytest.py
           - python-version: "3.7"
             test-cmd: python setup.py check --restructuredtext --strict --metadata
           - python-version: "3.10"
@@ -26,7 +24,10 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip setuptools wheel
-          pip install pytest pycodestyle docutils Pygments hypothesis python-Levenshtein
+          pip install pytest pycodestyle docutils Pygments hypothesis
+
+      - name: Install project
+        run: pip install .
+
       - name: Test with pytest
-        run: |
-          ${{ matrix.test-cmd }}
+        run: ${{ matrix.test-cmd }}

README.rst

@@ -10,9 +10,7 @@ Requirements
 ============
 
 -  Python 3.7 or higher
--  difflib
--  `python-Levenshtein <https://github.com/ztane/python-Levenshtein/>`_ (optional, provides a 4-10x speedup in String
-   Matching, though may result in `differing results for certain cases <https://github.com/seatgeek/fuzzywuzzy/issues/128>`_)
+-  `rapidfuzz <https://github.com/maxbachmann/RapidFuzz/>`_
 
 For testing
 ~~~~~~~~~~~
@@ -29,12 +27,6 @@ Using PIP via PyPI
 
     pip install thefuzz
 
-or the following to install `python-Levenshtein` too
-
-.. code:: bash
-
-    pip install thefuzz[speedup]
-
 
 Using PIP via Github
 
@@ -110,7 +102,7 @@ Partial Token Sort Ratio
         84
     >>> fuzz.partial_token_sort_ratio("fuzzy was a bear", "wuzzy fuzzy was a bear")
         100
-        
+
 Process
 ~~~~~~~
 

benchmarks.py

@@ -39,7 +39,6 @@
 ]
 
 common_setup = "from thefuzz import fuzz, utils; "
-basic_setup = "from thefuzz.string_processing import StringProcessor;"
 
 
 def print_result_from_timeit(stmt='pass', setup='pass', number=1000000):
@@ -55,48 +54,42 @@ def print_result_from_timeit(stmt='pass', setup='pass', number=1000000):
         duration, avg_duration * (1000 ** -thousands), units[-thousands]))
 
 
-for s in choices:
-    print('Test validate_string for: "%s"' % s)
-    print_result_from_timeit('utils.validate_string(\'%s\')' % s, common_setup, number=iterations)
-
-print('')
-
 for s in mixed_strings + cirque_strings + choices:
     print('Test full_process for: "%s"' % s)
     print_result_from_timeit('utils.full_process(u\'%s\')' % s,
-                             common_setup + basic_setup, number=iterations)
+                             common_setup, number=iterations)
 
 # benchmarking the core matching methods...
 
 for s in cirque_strings:
     print('Test fuzz.ratio for string: "%s"' % s)
     print('-------------------------------')
     print_result_from_timeit('fuzz.ratio(u\'cirque du soleil\', u\'%s\')' % s,
-                             common_setup + basic_setup, number=iterations / 100)
+                             common_setup, number=iterations / 100)
 
 for s in cirque_strings:
     print('Test fuzz.partial_ratio for string: "%s"' % s)
     print('-------------------------------')
     print_result_from_timeit('fuzz.partial_ratio(u\'cirque du soleil\', u\'%s\')'
-                             % s, common_setup + basic_setup, number=iterations / 100)
+                             % s, common_setup, number=iterations / 100)
 
 for s in cirque_strings:
     print('Test fuzz.WRatio for string: "%s"' % s)
     print('-------------------------------')
     print_result_from_timeit('fuzz.WRatio(u\'cirque du soleil\', u\'%s\')' % s,
-                             common_setup + basic_setup, number=iterations / 100)
+                             common_setup, number=iterations / 100)
 
 print('Test process.extract(scorer =  fuzz.QRatio) for string: "%s"' % s)
 print('-------------------------------')
 print_result_from_timeit('process.extract(u\'cirque du soleil\', choices, scorer =  fuzz.QRatio)',
-                             common_setup + basic_setup + " from thefuzz import process; import string,random; random.seed(18);"
+                             common_setup + " from thefuzz import process; import string,random; random.seed(18);"
                              " choices = [\'\'.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(30)) for s in range(5000)]",
                               number=10)
 
 print('Test process.extract(scorer =  fuzz.WRatio) for string: "%s"' % s)
 print('-------------------------------')
 print_result_from_timeit('process.extract(u\'cirque du soleil\', choices, scorer =  fuzz.WRatio)',
-                             common_setup + basic_setup + " from thefuzz import process; import string,random; random.seed(18);"
+                             common_setup + " from thefuzz import process; import string,random; random.seed(18);"
                              " choices = [\'\'.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(30)) for s in range(5000)]",
                               number=10)
 
@@ -114,6 +107,4 @@ def print_result_from_timeit(stmt='pass', setup='pass', number=1000000):
 print('-------------------------------')
 test += 'prepared_ratio = functools.partial(fuzz.ratio, "%s")\n' % s
 test += 'titles.sort(key=prepared_ratio)\n'
-print_result_from_timeit(test,
-                         common_setup + basic_setup,
-                         number=100)
+print_result_from_timeit(test, common_setup, number=100)

setup.py

@@ -5,25 +5,20 @@
 # This file is part of thefuzz.
 
 from thefuzz import __version__
-import os
-
-try:
-    from setuptools import setup
-except ImportError:
-    from distutils.core import setup
-
-
-def open_file(fname):
-    return open(os.path.join(os.path.dirname(__file__), fname))
+from setuptools import setup
 
+with open('README.rst') as f:
+    long_description = f.read()
 
 setup(
     name='thefuzz',
     version=__version__,
     author='Adam Cohen',
     author_email='[email protected]',
     packages=['thefuzz'],
-    extras_require={'speedup': ['python-levenshtein>=0.12']},
+    # keep for backwards compatibility of projects depending on `thefuzz[speedup]`
+    extras_require={'speedup': []},
+    install_requires= ['rapidfuzz>=3.0.0, < 4.0.0'],
     url='https://github.com/seatgeek/thefuzz',
     license="GPLv2",
     classifiers=[
@@ -39,6 +34,7 @@ def open_file(fname):
         'Programming Language :: Python :: 3 :: Only',
     ],
     description='Fuzzy string matching in python',
-    long_description=open_file('README.rst').read(),
+    long_description=long_description,
     zip_safe=True,
+    python_requires='>=3.7'
 )

test_thefuzz.py

@@ -5,25 +5,40 @@
 from thefuzz import fuzz
 from thefuzz import process
 from thefuzz import utils
-from thefuzz.string_processing import StringProcessor
 
+scorers = [
+    fuzz.ratio,
+    fuzz.partial_ratio,
+    fuzz.token_sort_ratio,
+    fuzz.token_set_ratio,
+    fuzz.partial_token_sort_ratio,
+    fuzz.partial_token_set_ratio,
+    fuzz.QRatio,
+    fuzz.UQRatio,
+    fuzz.WRatio,
+    fuzz.UWRatio,
+]
 
 class StringProcessingTest(unittest.TestCase):
     def test_replace_non_letters_non_numbers_with_whitespace(self):
         strings = ["new york mets - atlanta braves", "Cães danados",
                    "New York //// Mets $$$", "Ça va?"]
         for string in strings:
-            proc_string = StringProcessor.replace_non_letters_non_numbers_with_whitespace(string)
+            proc_string = utils.full_process(string)
             regex = re.compile(r"(?ui)[\W]")
             for expr in regex.finditer(proc_string):
                 self.assertEqual(expr.group(), " ")
 
     def test_dont_condense_whitespace(self):
         s1 = "new york mets - atlanta braves"
         s2 = "new york mets atlanta braves"
-        p1 = StringProcessor.replace_non_letters_non_numbers_with_whitespace(s1)
-        p2 = StringProcessor.replace_non_letters_non_numbers_with_whitespace(s2)
-        self.assertNotEqual(p1, p2)
+        s3 = "new york mets   atlanta braves"
+        p1 = utils.full_process(s1)
+        p2 = utils.full_process(s2)
+        p3 = utils.full_process(s3)
+        self.assertEqual(p1, s3)
+        self.assertEqual(p2, s2)
+        self.assertEqual(p3, s3)
 
 
 class UtilsTest(unittest.TestCase):
@@ -120,7 +135,8 @@ def testPartialTokenSortRatio(self):
         self.assertEqual(fuzz.partial_token_sort_ratio(self.s8, self.s8a, full_process=False), 100)
         self.assertEqual(fuzz.partial_token_sort_ratio(self.s9, self.s9a, full_process=True), 100)
         self.assertEqual(fuzz.partial_token_sort_ratio(self.s9, self.s9a, full_process=False), 100)
-        self.assertEqual(fuzz.partial_token_sort_ratio(self.s10, self.s10a, full_process=False), 50)
+        self.assertEqual(fuzz.partial_token_sort_ratio(self.s10, self.s10a, full_process=False), 67)
+        self.assertEqual(fuzz.partial_token_sort_ratio(self.s10a, self.s10, full_process=False), 67)
 
     def testTokenSetRatio(self):
         self.assertEqual(fuzz.token_set_ratio(self.s4, self.s5), 100)
@@ -243,58 +259,44 @@ def testQRatioForceAscii(self):
         score = fuzz.WRatio(s1, s2, force_ascii=False)
         self.assertLess(score, 100)
 
-    def testTokenSetForceAscii(self):
+    def testPartialTokenSetRatioForceAscii(self):
         s1 = "ABCD\u00C1 HELP\u00C1"
         s2 = "ABCD HELP"
 
-        score = fuzz._token_set(s1, s2, force_ascii=True)
+        score = fuzz.partial_token_set_ratio(s1, s2, force_ascii=True)
         self.assertEqual(score, 100)
 
-        score = fuzz._token_set(s1, s2, force_ascii=False)
+        score = fuzz.partial_token_set_ratio(s1, s2, force_ascii=False)
         self.assertLess(score, 100)
 
-    def testTokenSortForceAscii(self):
+    def testPartialTokenSortRatioForceAscii(self):
         s1 = "ABCD\u00C1 HELP\u00C1"
         s2 = "ABCD HELP"
 
-        score = fuzz._token_sort(s1, s2, force_ascii=True)
+        score = fuzz.partial_token_sort_ratio(s1, s2, force_ascii=True)
         self.assertEqual(score, 100)
 
-        score = fuzz._token_sort(s1, s2, force_ascii=False)
+        score = fuzz.partial_token_sort_ratio(s1, s2, force_ascii=False)
         self.assertLess(score, 100)
 
-
-class ValidatorTest(unittest.TestCase):
-    def setUp(self):
-        self.testFunc = lambda *args, **kwargs: (args, kwargs)
-
     def testCheckForNone(self):
-        invalid_input = [
-            (None, None),
-            ('Some', None),
-            (None, 'Some')
-        ]
-        decorated_func = utils.check_for_none(self.testFunc)
-        for i in invalid_input:
-            self.assertEqual(decorated_func(*i), 0)
+        for scorer in scorers:
+            self.assertEqual(scorer(None, None), 0)
+            self.assertEqual(scorer('Some', None), 0)
+            self.assertEqual(scorer(None, 'Some'), 0)
 
-        valid_input = ('Some', 'Some')
-        actual = decorated_func(*valid_input)
-        self.assertNotEqual(actual, 0)
+            self.assertNotEqual(scorer('Some', 'Some'), 0)
 
     def testCheckEmptyString(self):
-        invalid_input = [
-            ('', ''),
-            ('Some', ''),
-            ('', 'Some')
-        ]
-        decorated_func = utils.check_empty_string(self.testFunc)
-        for i in invalid_input:
-            self.assertEqual(decorated_func(*i), 0)
-
-        valid_input = ('Some', 'Some')
-        actual = decorated_func(*valid_input)
-        self.assertNotEqual(actual, 0)
+        for scorer in scorers:
+            if scorer in {fuzz.token_set_ratio, fuzz.partial_token_set_ratio, fuzz.WRatio, fuzz.UWRatio, fuzz.QRatio, fuzz.UQRatio}:
+                self.assertEqual(scorer('', ''), 0)
+            else:
+                self.assertEqual(scorer('', ''), 100)
+
+            self.assertEqual(scorer('Some', ''), 0)
+            self.assertEqual(scorer('', 'Some'), 0)
+            self.assertNotEqual(scorer('Some', 'Some'), 0)
 
 
 class ProcessTest(unittest.TestCase):

test_thefuzz_hypothesis.py

@@ -2,7 +2,7 @@
 from functools import partial
 from string import ascii_letters, digits, punctuation
 
-from hypothesis import given, assume, settings
+from hypothesis import given, assume, settings, HealthCheck
 import hypothesis.strategies as st
 import pytest
 
@@ -62,7 +62,7 @@ def full_scorers_processors():
 @pytest.mark.parametrize('scorer,processor',
                          scorers_processors())
 @given(data=st.data())
-@settings(max_examples=20, deadline=5000)
+@settings(max_examples=20, deadline=5000, suppress_health_check=[HealthCheck.data_too_large])
 def test_identical_strings_extracted(scorer, processor, data):
     """
     Test that identical strings will always return a perfect match.