Check for dropped tokens

andrewdalpino · Oct 15, 2024 · a453675 · a453675
1 parent 389066a
commit a453675
Showing 1 changed file with 6 additions and 1 deletion.
diff --git a/tests/test_tokenizers.py b/tests/test_tokenizers.py
@@ -13,6 +13,8 @@ def test_tokenize(self):
         for i, token in enumerate(tokens):
             self.assertEqual(token, expected[i])
 
+        self.assertEqual(tokenizer.dropped, 6)
+
 class TestCanonical(unittest.TestCase):
     def test_tokenize(self):
         tokenizer = tokenizers.Canonical(tokenizers.Kmer(k=6))
@@ -33,4 +35,7 @@ def test_tokenize(self):
         expected = ['CGGT', 'TCAG', 'TAAT']
 
         for i, token in enumerate(tokens):
-            self.assertEqual(token, expected[i])
+            self.assertEqual(token, expected[i])
+
+        self.assertEqual(tokenizer.dropped, 1)
+