fix of the partitioner bug for NUTCH-2455

apache · Dec 27, 2017 · 16f26f1 · 16f26f1
1 parent e20973c
commit 16f26f1
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 7 deletions.
diff --git a/src/java/org/apache/nutch/crawl/Generator.java b/src/java/org/apache/nutch/crawl/Generator.java
@@ -226,7 +226,7 @@ public int compareTo(FloatTextPair tp) {
 		}
 
 	}
-  //The comparator is made to "merge" hostdb data and crawldd data. See  NUTCH-2455
+  //The comparator is made to "merge" hostdb data and crawldb data. See  NUTCH-2455
   //TODO : Implement RawComparator
   public static class ScoreHostKeyComparator extends WritableComparator {
 	    protected ScoreHostKeyComparator() {
@@ -456,7 +456,6 @@ public void reduce(FloatTextPair key, Iterator<SelectorEntry> values,
         OutputCollector<FloatWritable, SelectorEntry> output, Reporter reporter)
         throws IOException {
 
-      String hostname = null;
       LongWritable variableFetchDelayWritable = null; // in millis
       Text variableFetchDelayKey = new Text("_variableFetchDelay_");
       int maxCount = this.maxCount;
@@ -832,7 +831,7 @@ public Path[] generate(Path dbDir, Path segments, int numLists, long topN,
     job.setOutputKeyComparatorClass(ScoreHostKeyComparator.class);
     job.setPartitionerClass(Selector.class);
     job.setReducerClass(Selector.class);
-    
+
     FileOutputFormat.setOutputPath(job, tempDir);
     job.setOutputFormat(SequenceFileOutputFormat.class);
     job.setOutputKeyClass(FloatWritable.class);

diff --git a/src/java/org/apache/nutch/crawl/URLPartitioner.java b/src/java/org/apache/nutch/crawl/URLPartitioner.java
@@ -65,7 +65,6 @@ public void close() {
 
   public String getNormalizedURLRoot(Text key){
 	  URLNormalizers normalizers = this.normalizers;
-	  String rootMode = mode;
 
 	  String urlString = key.toString();
 	  URL url = null;
@@ -86,11 +85,19 @@ public String getNormalizedURLRoot(Text key){
 		return urlString;
   }
 
+
   /** Hash by host or domain name or IP address. */
   public int getPartition(Text key, Writable value, int numReduceTasks) {
-    String urlString = key.toString();
-    URL url = null;
-    int hashCode = getNormalizedURLRoot(key).hashCode();
+	String partitionKey;
+
+	if(key.toString().isEmpty()){
+		partitionKey = value.toString();
+	} else
+	{
+		partitionKey = getNormalizedURLRoot(key);
+	}
+
+	int hashCode = partitionKey.hashCode();
     // make hosts wind up in different partitions on different runs
     hashCode ^= seed;