Merge pull request #870 from igiguere/NUTCH-2971

Fix for NUTCH-2971: Unt tests fail with JDK 17

Author: sebastian-nagel

build.xml

@@ -86,6 +86,10 @@
     <ivy:dependencytree />
   </target>
 
+  <target name="dependencytests" depends="resolve-test" description="Show unit tests dependency tree">
+	<ivy:dependencytree />
+  </target>
+
   <!-- ====================================================== -->
   <!-- Stuff needed by all targets                            -->
   <!-- ====================================================== -->

ivy/ivy.xml

@@ -122,13 +122,6 @@
         <!-- Required for JUnit 5 (Jupiter) test execution -->
         <dependency org="org.junit.jupiter" name="junit-jupiter-engine" rev="5.13.4" conf="test->default"/>
         <dependency org="org.junit.jupiter" name="junit-jupiter-api" rev="5.13.4" conf="test->default"/>
-		<dependency org="org.apache.mrunit" name="mrunit" rev="1.1.0" conf="test->default">
-			<artifact name="mrunit" ns0:classifier="hadoop2" />
-			<exclude org="log4j" module="log4j" />
-			<exclude org="junit" module="junit" />
-			<exclude org="org.powermock" module="powermock-module-junit4" />
-			<exclude org="com.google.guava" name="guava" />
-		</dependency>
 
 		<!-- Jetty used to serve test pages for unit tests, but is also provided as dependency of Hadoop -->
 		<dependency org="org.eclipse.jetty" name="jetty-server" rev="10.0.25" conf="test->default">

src/plugin/build-plugin.xml

@@ -282,4 +282,7 @@
   <target name="dependencytree" depends="resolve-default" description="Show dependency tree">
     <ivy:dependencytree />
   </target>
+  <target name="dependencytests" depends="resolve-test" description="Show unit tests dependency tree">
+    <ivy:dependencytree />
+  </target>
 </project>

src/plugin/protocol-http/src/test/org/apache/nutch/protocol/http/TestProtocolHttpByProxy.java

@@ -22,6 +22,7 @@
 import org.apache.nutch.protocol.Content;
 import org.apache.nutch.protocol.ProtocolOutput;
 import org.apache.nutch.protocol.ProtocolStatus;
+import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.littleshoot.proxy.HttpProxyServer;
@@ -30,26 +31,43 @@
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
+import java.io.IOException;
+import java.net.ServerSocket;
+
 /**
  * Test cases for protocol-http by proxy
  */
 public class TestProtocolHttpByProxy extends AbstractHttpProtocolPluginTest {
 
   public static final String PROXY_HOST = "localhost";
-  public static final Integer PROXY_PORT = 8888;
+  public Integer proxyPort = 8888;
 
   public static final String TARGET_HOST = "www.baidu.com";
   public static final Integer TARGET_PORT = 443;
+  
+  private HttpProxyServer server;
 
   @BeforeEach
   public void setUp() throws Exception {
     super.setUp();
+    proxyPort = findOpenPort();
     conf.set("http.proxy.host", PROXY_HOST);
-    conf.set("http.proxy.port", PROXY_PORT.toString());
+    conf.set("http.proxy.port", proxyPort.toString());
     http.setConf(conf);
 
-    HttpProxyServer server = DefaultHttpProxyServer.bootstrap()
-        .withPort(PROXY_PORT).start();
+    server = DefaultHttpProxyServer.bootstrap()
+        .withPort(proxyPort).start();
+  }
+  
+  private Integer findOpenPort() throws IOException {
+    try (ServerSocket socket = new ServerSocket(0)) {
+      return socket.getLocalPort();
+    }
+  }
+  
+  @AfterEach
+  public void tearDown() {
+    server.stop();
   }
 
   @Override

src/test/org/apache/nutch/crawl/CrawlDbUpdateTestDriver.java

@@ -23,6 +23,7 @@
 import org.apache.hadoop.util.StringUtils;
 import org.apache.nutch.scoring.ScoringFilterException;
 import org.apache.nutch.scoring.ScoringFilters;
+import org.apache.nutch.util.ReducerContextWrapper;
 import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -31,8 +32,10 @@
 import java.lang.invoke.MethodHandles;
 import java.util.ArrayList;
 import java.util.Date;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 
 import static org.apache.nutch.crawl.CrawlDatum.*;
 import static org.junit.jupiter.api.Assertions.fail;
@@ -196,13 +199,13 @@ public void testCrawlDbStateTransitionMatrix() {
    * already in CrawlDb. Newly injected elements have status "db_unfetched".
    * Inject is simulated by calling {@link Injector.InjectReducer#reduce()}.
    */
+  @SuppressWarnings({ "unchecked", "rawtypes" })
   @Test
   public void testCrawlDbStatTransitionInject() {
     LOG.info("Test CrawlDatum states in Injector after inject");
     Configuration conf = CrawlDBTestUtil.createContext().getConfiguration();
     Injector.InjectReducer injector = new Injector.InjectReducer();
-    CrawlDbUpdateTestDriver<Injector.InjectReducer> injectDriver =
-        new CrawlDbUpdateTestDriver<Injector.InjectReducer>(injector, conf);
+
     ScoringFilters scfilters = new ScoringFilters(conf);
     for (String sched : schedules) {
       LOG.info("Testing inject with {}", sched);
@@ -234,12 +237,29 @@ public void testCrawlDbStatTransitionInject() {
           LOG.error(StringUtils.stringifyException(e));
         }
         values.add(injected);
-        List<CrawlDatum> res = injectDriver.update(values);
-        if (res.size() != 1) {
+        
+        List<CrawlDatum> result = new ArrayList<CrawlDatum>();
+        Map<Text, CrawlDatum> res = new HashMap<>(); 
+        ReducerContextWrapper contextWrapper = new ReducerContextWrapper(injector, conf, res);
+        try {
+          injector.setup(contextWrapper.getContext());
+          // test
+          injector.reduce(CrawlDbUpdateUtil.dummyURL, values, contextWrapper.getContext());
+
+          for (Map.Entry<Text, CrawlDatum> e : res.entrySet()) {
+            if (e.getKey().equals(CrawlDbUpdateUtil.dummyURL)) {
+              result.add(e.getValue());
+            }
+          }
+        } catch (IOException | InterruptedException e) {
+          LOG.error(StringUtils.stringifyException(e));
+        }
+
+        if (result.size() != 1) {
           fail("Inject didn't result in one single CrawlDatum per URL");
           continue;
         }
-        byte status = res.get(0).getStatus();
+        byte status = result.get(0).getStatus();
         if (status != toDbStatus) {
           fail("Inject for "
               + (fromDbStatus == -1 ? "" : getStatusName(fromDbStatus)

src/test/org/apache/nutch/crawl/TestCrawlDbStates.java

@@ -19,9 +19,6 @@
 import org.apache.commons.codec.binary.Base64;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
-import org.apache.hadoop.mrunit.types.Pair;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.nutch.crawl.CrawlDatum;
 import org.apache.nutch.crawl.NutchWritable;
@@ -33,6 +30,7 @@
 import org.apache.nutch.parse.ParseText;
 import org.apache.nutch.protocol.Content;
 import org.apache.nutch.util.NutchConfiguration;
+import org.apache.nutch.util.ReducerContextWrapper;
 import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -42,7 +40,9 @@
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -89,8 +89,8 @@ public class TestIndexerMapReduce {
   public static CrawlDatum crawlDatumFetchSuccess = new CrawlDatum(
       CrawlDatum.STATUS_FETCH_SUCCESS, 60 * 60 * 24);
 
-  private Reducer<Text, NutchWritable, Text, NutchIndexAction> reducer = new IndexerMapReduce.IndexerReducer();
-  private ReduceDriver<Text, NutchWritable, Text, NutchIndexAction> reduceDriver;
+  private IndexerMapReduce.IndexerReducer reducer = new IndexerMapReduce.IndexerReducer();
+
   private Configuration configuration;
 
 
@@ -101,6 +101,9 @@ public class TestIndexerMapReduce {
   public void testBinaryContentBase64() {
     configuration = NutchConfiguration.create();
     configuration.setBoolean(IndexerMapReduce.INDEXER_BINARY_AS_BASE64, true);
+    
+    // unrelated issue with "index.jexl.filter", don't use all plugins.  Ref: src/test/nutch-site.xml
+    configuration.set("plugin.includes", "protocol-http|urlfilter-regex|parse-(html|tika)|index-(basic|anchor)|indexer-csv|scoring-opic|urlnormalizer-(pass|regex|basic)");
 
     Charset[] testCharsets = { StandardCharsets.UTF_8,
         Charset.forName("iso-8859-1"), Charset.forName("iso-8859-2") };
@@ -155,7 +158,10 @@ public void testBinaryContentBase64() {
    * @param content
    *          (optional, if index binary content) protocol content
    * @return &quot;indexed&quot; document
+   * @throws InterruptedException 
+   * @throws IOException 
    */
+  @SuppressWarnings({ "unchecked", "rawtypes" })
   public NutchDocument runIndexer(CrawlDatum dbDatum, CrawlDatum fetchDatum,
       ParseText parseText, ParseData parseData, Content content) {
     List<NutchWritable> values = new ArrayList<NutchWritable>();
@@ -164,19 +170,20 @@ public NutchDocument runIndexer(CrawlDatum dbDatum, CrawlDatum fetchDatum,
     values.add(new NutchWritable(parseText));
     values.add(new NutchWritable(parseData));
     values.add(new NutchWritable(content));
-    reduceDriver = ReduceDriver.newReduceDriver(reducer);
-    reduceDriver.getConfiguration().addResource(configuration);
-    reduceDriver.withInput(testUrlText, values);
-    List<Pair<Text, NutchIndexAction>> reduceResult;
+    Map<Text, NutchIndexAction> reduceResult = new HashMap<>();  
+    ReducerContextWrapper contextWrapper = new ReducerContextWrapper(reducer, configuration, reduceResult);
     NutchDocument doc = null;
-    try {
-      reduceResult = reduceDriver.run();
-      for (Pair<Text, NutchIndexAction> p : reduceResult) {
-        if (p.getSecond().action != NutchIndexAction.DELETE) {
-          doc = p.getSecond().doc;
+    try {      
+      reducer.setup(contextWrapper.getContext());
+      // test
+      reducer.reduce(testUrlText, values, contextWrapper.getContext());
+      
+      for (Map.Entry<Text, NutchIndexAction> e : reduceResult.entrySet()) {
+        if (e.getValue().action != NutchIndexAction.DELETE) {
+          doc = e.getValue().doc;
         }
       }
-    } catch (IOException e) {
+    } catch (IOException | InterruptedException e) {
       LOG.error(StringUtils.stringifyException(e));
     }
     return doc;