Upgrade Tika to 3.0.0 and lighten TIKA dependency tree

HHN · Oct 22, 2024 · 38ac2b1 · 38ac2b1
1 parent fcfcda9
commit 38ac2b1
Show file tree

Hide file tree

Showing 5 changed files with 48 additions and 44 deletions.
diff --git a/README.md b/README.md
@@ -351,6 +351,18 @@ An example on how to configure form authentication in its most simple form:
 
 Overriding `FormAuthInfo.doFormLogin(...)` allows implementing more dynamic form authentication (entering more fields, fetching dynamic login forms, ...).
 
+## Binary Content
+
+To lighten our dependencies, we replaced the default Apache Tika standard package with the HTML module only.
+If you need more parsing capabilities, just add the related dependencies:
+```xml
+<dependency>
+   <groupId>org.apache.tika</groupId>
+   <artifactId>tika-parsers-standard</artifactId>
+   <version>VERSION</version>
+</dependency>
+```
+
 ## High-level design diagrams
 
 Below activity diagram highlights the most important steps and vocabulary to know as user of the library.

diff --git a/crawler4j-core/pom.xml b/crawler4j-core/pom.xml
@@ -161,7 +161,7 @@
         </dependency>
         <dependency>
             <groupId>org.apache.tika</groupId>
-            <artifactId>tika-parsers-standard-package</artifactId>
+            <artifactId>tika-parser-html-module</artifactId>
             <version>${apache.tika.version}</version>
             <exclusions>
                 <exclusion>
@@ -186,19 +186,19 @@
                 </exclusion>
             </exclusions>
         </dependency>
-					<!-- Test dependencies -->
-					<dependency>
-						<groupId>org.junit.jupiter</groupId>
-						<artifactId>junit-jupiter</artifactId>
-						<version>${junit.version}</version>
-						<scope>test</scope>
-					</dependency>
-					<dependency>
-						<groupId>org.assertj</groupId>
-						<artifactId>assertj-core</artifactId>
-						<version>${assertj.version}</version>
-						<scope>test</scope>
-					</dependency>
+        <!-- Test dependencies -->
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter</artifactId>
+            <version>${junit.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.assertj</groupId>
+            <artifactId>assertj-core</artifactId>
+            <version>${assertj.version}</version>
+            <scope>test</scope>
+        </dependency>
         <!-- Test Dependencies -->
         <dependency>
             <groupId>de.hs-heilbronn.mi</groupId>

diff --git a/crawler4j-core/src/main/java/edu/uci/ics/crawler4j/parser/TikaHtmlParser.java b/crawler4j-core/src/main/java/edu/uci/ics/crawler4j/parser/TikaHtmlParser.java
@@ -34,8 +34,9 @@
 import org.apache.tika.metadata.DublinCore;
 import org.apache.tika.metadata.Metadata;
 import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.Parser;
 import org.apache.tika.parser.html.HtmlMapper;
-import org.apache.tika.parser.html.HtmlParser;
+import org.apache.tika.parser.html.JSoupParser;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -51,7 +52,7 @@ public class TikaHtmlParser implements edu.uci.ics.crawler4j.parser.HtmlParser {
     private final CrawlConfig config;
     private final TLDList tldList;
 
-    private final HtmlParser htmlParser;
+    private final Parser htmlParser;
     private final ParseContext parseContext;
     private final WebURLFactory factory;
     private final BasicURLNormalizer normalizer;
@@ -61,7 +62,7 @@ public TikaHtmlParser(CrawlConfig config, BasicURLNormalizer normalizer, TLDList
         this.tldList = tldList;
         this.normalizer = normalizer;
 
-        htmlParser = new HtmlParser();
+        htmlParser = new JSoupParser();
         parseContext = new ParseContext();
         parseContext.set(HtmlMapper.class, new AllTagMapper());
         this.factory = webURLFactory;

diff --git a/...t/java/edu/uci/ics/crawler4j/tests/fetcher/politeness/SimplePolitenessServerTestCase.java b/...t/java/edu/uci/ics/crawler4j/tests/fetcher/politeness/SimplePolitenessServerTestCase.java
@@ -20,6 +20,7 @@
 package edu.uci.ics.crawler4j.tests.fetcher.politeness;
 
 import org.assertj.core.api.Assertions;
+import org.awaitility.Awaitility;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
@@ -29,6 +30,8 @@
 import edu.uci.ics.crawler4j.test.SimpleWebURL;
 import edu.uci.ics.crawler4j.url.WebURL;
 
+import java.util.concurrent.TimeUnit;
+
 public class SimplePolitenessServerTestCase {
 
     private edu.uci.ics.crawler4j.PolitenessServer simplePolitenessServer;
@@ -54,8 +57,8 @@ public void testApplyPoliteness1() {
         politenessDelay = simplePolitenessServer.applyPoliteness(webUrl);
 
         Assertions.assertThat(politenessDelay).isBetween(//
-        		Long.valueOf(config.getPolitenessDelay() - 10)//
-        		, Long.valueOf(config.getPolitenessDelay()));
+                (long) (config.getPolitenessDelay() - 10)//
+        		, (long) config.getPolitenessDelay());
 
     }
 
@@ -74,15 +77,13 @@ public void testApplyPoliteness2() {
         politenessDelay = simplePolitenessServer.applyPoliteness(webUrl);
 
         Assertions.assertThat(politenessDelay).isBetween(//
-        		Long.valueOf(config.getPolitenessDelay() - 10)//
-        		, Long.valueOf(config.getPolitenessDelay()));
-
-        //let's wait some time, it should not be listed anymore
-        sleep(1000);
-
-        politenessDelay = simplePolitenessServer.applyPoliteness(webUrl);
+                (long) (config.getPolitenessDelay() - 10)//
+        		, (long) config.getPolitenessDelay());
 
-        Assertions.assertThat(politenessDelay).isEqualTo(CachedPolitenessServer.NO_POLITENESS_APPLIED);
+        Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> {
+            final long delay = simplePolitenessServer.applyPoliteness(webUrl);
+            Assertions.assertThat(delay).isEqualTo(CachedPolitenessServer.NO_POLITENESS_APPLIED);
+        });
 
     }
 
@@ -101,30 +102,20 @@ public void testApplyPoliteness3() {
         politenessDelay = simplePolitenessServer.applyPoliteness(webUrl);
 
         Assertions.assertThat(politenessDelay).isBetween(//
-        		Long.valueOf(config.getPolitenessDelay() - 10)//
-        		, Long.valueOf(config.getPolitenessDelay()));
+                (long) (config.getPolitenessDelay() - 10)//
+        		, (long) config.getPolitenessDelay());
 
         webUrl.setURL("https://github.com/yasserg/crawler4j/blob/master/pom.xml");
 
         politenessDelay = simplePolitenessServer.applyPoliteness(webUrl);
 
         Assertions.assertThat(politenessDelay).isEqualTo(config.getPolitenessDelay());
 
-        //let's wait some time, it should not be listed anymore
-        sleep(3000);
-
-        politenessDelay = simplePolitenessServer.applyPoliteness(webUrl);
-
-        Assertions.assertThat(politenessDelay).isEqualTo(CachedPolitenessServer.NO_POLITENESS_APPLIED);
-
-    }
+        Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> {
+            final long delay = simplePolitenessServer.applyPoliteness(webUrl);
+            Assertions.assertThat(delay).isEqualTo(CachedPolitenessServer.NO_POLITENESS_APPLIED);
+        });
 
-    private void sleep(int i) {
-        try {
-            Thread.sleep(i);
-        } catch (InterruptedException e) {
-            //nothing to do here
-        }
     }
 
 }
diff --git a/pom.xml b/pom.xml
@@ -112,7 +112,7 @@
         <apache.http.core.version>5.3</apache.http.core.version>
         <apache.http.core.h2.version>5.3</apache.http.core.h2.version>
 
-        <apache.tika.version>2.9.1</apache.tika.version>
+        <apache.tika.version>3.0.0</apache.tika.version>
         <!-- XXX replace with crawler commons -->
         <url-detector.version>0.1.23</url-detector.version>
         <crawler-commons.version>1.3</crawler-commons.version>