NUTCH-1011 Remove duplicate slashes from URLs

git-svn-id: https://svn.apache.org/repos/asf/nutch/branches/branch-1.4@1143467 13f79535-47bb-0310-9956-ffa450edef68
w2ogroup · Jul 6, 2011 · 60f0dbf · 60f0dbf
1 parent 32f7caf
commit 60f0dbf
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 0 deletions.
diff --git a/CHANGES.txt b/CHANGES.txt
@@ -2,6 +2,8 @@ Nutch Change Log
 
 Release 1.4 - Current development
 
+* NUTCH-1011 Normalize duplicate slashes in URL's (markus)
+
 * NUTCH-993 NullPointerException at FetcherOutputFormat.checkOutputSpecs (Christian Guegi via jnioche)
 
 * NUTCH-1013 Migrate RegexURLNormalizer from Apache ORO to java.util.regex (markus)

diff --git a/conf/regex-normalize.xml.template b/conf/regex-normalize.xml.template
@@ -63,4 +63,10 @@
   <substitution></substitution>
 </regex>
 
+<!-- removes duplicate slashes -->
+<regex>
+  <pattern>(?&lt;!:)/{2,}</pattern>
+  <substitution>/</substitution>
+</regex>
+
 </regex-normalize>
diff --git a/src/test/org/apache/nutch/net/TestURLNormalizers.java b/src/test/org/apache/nutch/net/TestURLNormalizers.java
@@ -39,6 +39,15 @@ public void testURLNormalizers() {
     } catch (MalformedURLException mue) {
       fail(mue.toString());
     }
+
+    // NUTCH-1011 - Get rid of superfluous slashes
+    try {
+      String normalizedSlashes = normalizers.normalize("http://www.example.org//path/to//somewhere.html", URLNormalizers.SCOPE_DEFAULT);
+      assertEquals(normalizedSlashes, "http://www.example.org/path/to/somewhere.html");
+    } catch (MalformedURLException mue) {
+      fail(mue.toString());
+    }
+
     // check the order
     int pos1 = -1, pos2 = -1;
     URLNormalizer[] impls = normalizers.getURLNormalizers(URLNormalizers.SCOPE_DEFAULT);