remove empty domain

rom1504 · Dec 8, 2023 · 076358e · 076358e
1 parent 9af3383
commit 076358e
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/cc2dataset/main.py b/cc2dataset/main.py
@@ -106,14 +106,15 @@ def extract_domain(url):
 for extractor in FILTERED_EXTRACTORS.values():
     for url in extract_test(extractor):
         domain = extract_domain(url)
+        if domain == "":
+            continue
         if domain in DOMAIN_IES_DICT:
             if extractor not in DOMAIN_IES_DICT[domain]:
                 DOMAIN_IES_DICT[domain] = DOMAIN_IES_DICT[domain] + [extractor]
         else:
             DOMAIN_IES_DICT[domain] = [extractor]
 
 
-
 def is_link_suitable(link, extractors):
     """Check if link is valid given an extractor."""
     try: