Merge pull request #859 from TamimEhsan/NUTCH-3122

[NUTCH-3122] Make SpellCheckedMetadata case-insensitive for all Metadata names

Author: sebastian-nagel

src/java/org/apache/nutch/metadata/SpellCheckedMetadata.java

@@ -26,10 +26,10 @@
 /**
  * A decorator to Metadata that adds spellchecking capabilities to property
  * names. Currently used spelling vocabulary contains just the HTTP headers from
- * {@link HttpHeaders} class.
+ * {@link HttpHeaders} class. Other names are case insensitive.
  * 
  */
-public class SpellCheckedMetadata extends Metadata {
+public class SpellCheckedMetadata extends CaseInsensitiveMetadata {
 
   /**
    * Threshold divider to calculate max. Levenshtein distance for misspelled

src/test/org/apache/nutch/metadata/TestSpellCheckedMetadata.java

@@ -21,6 +21,7 @@
 import java.io.DataInputStream;
 import java.io.DataOutputStream;
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.Properties;
 
 import org.junit.Assert;
@@ -111,6 +112,33 @@ public void testSet() {
     Assert.assertEquals("new value 2", values[1]);
   }
 
+  /** Test for the <code>set(String, String)</code> method. */
+  @Test
+  public void testSetCaseInsensitive() {
+    String[] values = null;
+    SpellCheckedMetadata meta = new SpellCheckedMetadata();
+
+    values = meta.getValues("name-one");
+    Assert.assertEquals(0, values.length);
+
+    meta.set("name-one", "value1");
+    values = meta.getValues("name-one");
+    Assert.assertEquals(1, values.length);
+    Assert.assertEquals("value1", values[0]);
+
+    meta.set("naMe-OnE", "value2");
+    values = meta.getValues("name-one");
+    Assert.assertEquals(1, values.length);
+    Assert.assertEquals("value2", values[0]);
+
+    meta.set("nAme-One", "new value 1");
+    meta.add("NamE-oNe", "new value 2");
+    values = meta.getValues("namE-OnE");
+    Assert.assertEquals(2, values.length);
+    Assert.assertEquals("new value 1", values[0]);
+    Assert.assertEquals("new value 2", values[1]);
+  }
+
   /** Test for <code>setAll(Properties)</code> method. */
   @Test
   public void testSetProperties() {
@@ -151,6 +179,19 @@ public void testGet() {
     Assert.assertEquals("value-1", meta.get("a-name"));
   }
 
+  /** Test for <code>get(String)</code> method. */
+  @Test
+  public void testGetCaseInsensitive() {
+    SpellCheckedMetadata meta = new SpellCheckedMetadata();
+    Assert.assertNull(meta.get("a-name"));
+
+    meta.add("a-name", "value-1");
+    Assert.assertEquals("value-1", meta.get("a-name"));
+    
+    Assert.assertNotNull(meta.get("a-NamE"));
+    Assert.assertEquals("value-1", meta.get("a-NamE"));
+  }
+
   /** Test for <code>isMultiValued()</code> method. */
   @Test
   public void testIsMultiValued() {
@@ -252,6 +293,51 @@ public void testWritable() {
     Assert.assertEquals("text/html", result.get(Metadata.CONTENT_TYPE));
   }
 
+  /** Test for <code>Writable</code> implementation. */
+  @Test
+  public void testWritableBackwardCompatibility() {
+    // Entries added with old instances of SpellCheckedMetadata
+    // should be readable by new instances of SpellCheckedMetadata
+    SpellCheckedMetadata result = null;
+    CaseSensitiveSpellCheckedMetadata meta = new CaseSensitiveSpellCheckedMetadata();
+
+    result = writeRead(meta);
+    Assert.assertEquals(0, result.size());
+
+    meta.add("name-One", "value-1.1");
+    // Check that the original case is kept for old Metadata class
+    Assert.assertEquals(0, result.getValues("naMe-one").length);
+
+    // Check that the values written by old instances can be
+    // read by new instances of SpellCheckedMetadata
+    result = writeRead(meta);
+    Assert.assertEquals(1, result.size());
+    Assert.assertEquals(1, result.getValues("naMe-one").length);
+    Assert.assertEquals("value-1.1", result.get("nAme-oNe"));
+
+    meta.add("Contenttype", "text/html");
+    meta.add("name-Two", "value-2.1");
+    meta.add("namE-two", "value-2.2");
+    result = writeRead(meta);
+    Assert.assertEquals(3, result.size());
+    Assert.assertEquals(1, result.getValues("name-onE").length);
+    Assert.assertEquals("value-1.1", result.getValues("namE-one")[0]);
+    Assert.assertEquals(2, result.getValues("name-two").length);
+    Assert.assertEquals("value-2.1", result.getValues("nAme-tWo")[0]);
+    Assert.assertEquals("value-2.2", result.getValues("namE-Two")[1]);
+    Assert.assertEquals("text/html", result.get(Metadata.CONTENT_TYPE));
+  }
+
+  /** 
+   * Mock class to test backward compatibility of SpellCheckedMetadata
+   * after changing the internal storage to be case insensitive.
+   */
+  private static class CaseSensitiveSpellCheckedMetadata extends SpellCheckedMetadata {
+    public CaseSensitiveSpellCheckedMetadata() {
+      metadata = new HashMap<>();
+    }
+  }
+
   /**
    * IO Test method, usable only when you plan to do changes in metadata to
    * measure relative performance impact.