Require Java 8 and include some Java 8 dependencies. (apache#3914)

* Require Java 8 and include some Java 8 dependencies. - Upgrade Jetty to 9.3.16.v20170120. - Upgrade DataSketches to 0.8.4. - Bundle caffeine-cache by default. - Still target Java 7 when compiling base Druid classes. * Update cluster, quickstart docs. * Remove oraclejdk7 from travis.yml.
Gamehu · Feb 14, 2017 · 78b0d13 · 78b0d13
1 parent 28d8570
commit 78b0d13
Show file tree

Hide file tree

Showing 23 changed files with 86 additions and 120 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -1,7 +1,6 @@
 language: java
 
 jdk:
-  - oraclejdk7
   - oraclejdk8
 
 script:

diff --git a/distribution/pom.xml b/distribution/pom.xml
@@ -79,6 +79,8 @@
                                 <argument>-c</argument>
                                 <argument>io.druid.extensions:druid-avro-extensions</argument>
                                 <argument>-c</argument>
+                                <argument>io.druid.extensions:druid-caffeine-cache</argument>
+                                <argument>-c</argument>
                                 <argument>io.druid.extensions:druid-datasketches</argument>
                                 <argument>-c</argument>
                                 <argument>io.druid.extensions:druid-hdfs-storage</argument>
@@ -166,53 +168,6 @@
         </plugins>
     </build>
     <profiles>
-        <profile>
-            <id>java8</id>
-            <activation>
-                <jdk>1.8</jdk>
-            </activation>
-            <build>
-                <plugins>
-                    <plugin>
-                        <groupId>org.codehaus.mojo</groupId>
-                        <artifactId>exec-maven-plugin</artifactId>
-                        <executions>
-                            <execution>
-                                <id>pull-deps-jdk8</id>
-                                <phase>package</phase>
-                                <goals>
-                                    <goal>exec</goal>
-                                </goals>
-                                <configuration>
-                                    <executable>java</executable>
-                                    <arguments>
-                                        <argument>-classpath</argument>
-                                        <classpath/>
-                                        <argument>-Ddruid.extensions.loadList=[]</argument>
-                                        <argument>-Ddruid.extensions.directory=${project.build.directory}/extensions
-                                        </argument>
-                                        <argument>
-                                            -Ddruid.extensions.hadoopDependenciesDir=${project.build.directory}/hadoop-dependencies
-                                        </argument>
-                                        <argument>io.druid.cli.Main</argument>
-                                        <argument>tools</argument>
-                                        <argument>pull-deps</argument>
-                                        <argument>--defaultVersion</argument>
-                                        <argument>${project.parent.version}</argument>
-                                        <argument>-l</argument>
-                                        <argument>${settings.localRepository}</argument>
-                                        <!-- Only need stuff here that is NOT included in the general release -->
-                                        <argument>--no-default-hadoop</argument>
-                                        <argument>-c</argument>
-                                        <argument>io.druid.extensions:druid-caffeine-cache</argument>
-                                    </arguments>
-                                </configuration>
-                            </execution>
-                        </executions>
-                    </plugin>
-                </plugins>
-            </build>
-        </profile>
         <profile>
             <id>bundle-contrib-exts</id>
             <build>

diff --git a/docs/content/tutorials/cluster.md b/docs/content/tutorials/cluster.md
@@ -46,7 +46,7 @@ Very large clusters should consider selecting larger servers.
 
 We recommend running your favorite Linux distribution. You will also need:
 
-  * Java 7 or better
+  * Java 8 or better
 
 Your OS package manager should be able to help for both Java. If your Ubuntu-based OS
 does not have a recent enough version of Java, WebUpd8 offers [packages for those

diff --git a/docs/content/tutorials/quickstart.md b/docs/content/tutorials/quickstart.md
@@ -10,7 +10,7 @@ In this quickstart, we will download Druid, set up it up on a single machine, lo
 
 You will need:
 
-  * Java 7 or higher
+  * Java 8 or higher
   * Linux, Mac OS X, or other Unix-like OS (Windows is not supported)
   * 8G of RAM
   * 2 vCPUs

diff --git a/extensions-core/datasketches/pom.xml b/extensions-core/datasketches/pom.xml
@@ -38,7 +38,7 @@
     <dependency>
       <groupId>com.yahoo.datasketches</groupId>
       <artifactId>sketches-core</artifactId>
-      <version>0.4.1</version>
+      <version>0.8.4</version>
     </dependency>
     <dependency>
       <groupId>io.druid</groupId>

diff --git a/...s/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchBufferAggregator.java b/...s/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchBufferAggregator.java
@@ -19,10 +19,10 @@
 
 package io.druid.query.aggregation.datasketches.theta;
 
+import com.yahoo.memory.Memory;
+import com.yahoo.memory.MemoryRegion;
+import com.yahoo.memory.NativeMemory;
 import com.yahoo.sketches.Family;
-import com.yahoo.sketches.memory.Memory;
-import com.yahoo.sketches.memory.MemoryRegion;
-import com.yahoo.sketches.memory.NativeMemory;
 import com.yahoo.sketches.theta.SetOperation;
 import com.yahoo.sketches.theta.Union;
 import io.druid.query.aggregation.BufferAggregator;

diff --git a/...atasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchHolder.java b/...atasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchHolder.java
@@ -24,9 +24,9 @@
 import com.google.common.collect.Ordering;
 import com.google.common.primitives.Doubles;
 import com.google.common.primitives.Longs;
+import com.yahoo.memory.Memory;
+import com.yahoo.memory.NativeMemory;
 import com.yahoo.sketches.Family;
-import com.yahoo.sketches.memory.Memory;
-import com.yahoo.sketches.memory.NativeMemory;
 import com.yahoo.sketches.theta.AnotB;
 import com.yahoo.sketches.theta.Intersection;
 import com.yahoo.sketches.theta.SetOperation;

diff --git a/...hes/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchObjectStrategy.java b/...hes/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchObjectStrategy.java
@@ -19,8 +19,8 @@
 
 package io.druid.query.aggregation.datasketches.theta;
 
-import com.yahoo.sketches.memory.MemoryRegion;
-import com.yahoo.sketches.memory.NativeMemory;
+import com.yahoo.memory.MemoryRegion;
+import com.yahoo.memory.NativeMemory;
 import com.yahoo.sketches.theta.Sketch;
 import io.druid.java.util.common.IAE;
 import io.druid.segment.data.ObjectStrategy;

diff --git a/...etches/src/main/java/io/druid/query/aggregation/datasketches/theta/SynchronizedUnion.java b/...etches/src/main/java/io/druid/query/aggregation/datasketches/theta/SynchronizedUnion.java
@@ -19,7 +19,7 @@
 
 package io.druid.query.aggregation.datasketches.theta;
 
-import com.yahoo.sketches.memory.Memory;
+import com.yahoo.memory.Memory;
 import com.yahoo.sketches.theta.CompactSketch;
 import com.yahoo.sketches.theta.Sketch;
 import com.yahoo.sketches.theta.Union;
@@ -77,6 +77,12 @@ public synchronized void update(int[] data)
     delegate.update(data);
   }
 
+  @Override
+  public void update(char[] chars)
+  {
+    delegate.update(chars);
+  }
+
   @Override
   public synchronized void update(long[] data)
   {

diff --git a/pom.xml b/pom.xml
@@ -63,7 +63,7 @@
         <calcite.version>1.11.0</calcite.version>
         <guava.version>16.0.1</guava.version>
         <guice.version>4.1.0</guice.version>
-        <jetty.version>9.2.5.v20141112</jetty.version>
+        <jetty.version>9.3.16.v20170120</jetty.version>
         <jersey.version>1.19</jersey.version>
         <!-- Watch out for Hadoop compatibility when updating to >= 2.5; see https://github.com/druid-io/druid/pull/1669 -->
         <jackson.version>2.4.6</jackson.version>
@@ -95,6 +95,7 @@
         <module>hll</module>
         <!-- Core extensions -->
         <module>extensions-core/avro-extensions</module>
+        <module>extensions-core/caffeine-cache</module>
         <module>extensions-core/datasketches</module>
         <module>extensions-core/druid-kerberos</module>
         <module>extensions-core/hdfs-storage</module>
@@ -835,6 +836,9 @@
                         </goals>
                         <configuration>
                             <rules>
+                                <requireJavaVersion>
+                                    <version>1.8.0</version>
+                                </requireJavaVersion>
                                 <bannedDependencies>
                                     <excludes>
                                         <!--LGPL licenced library-->
@@ -1007,17 +1011,6 @@
             <activation>
                 <jdk>1.8</jdk>
             </activation>
-            <modules>
-                <module>extensions-core/caffeine-cache</module>
-                <!-- distribution packaging -->
-                <module>distribution</module>
-            </modules>
-        </profile>
-        <profile>
-            <id>java7</id>
-            <activation>
-                <jdk>1.7</jdk>
-            </activation>
             <modules>
                 <!-- distribution packaging -->
                 <module>distribution</module>

diff --git a/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java b/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java
@@ -141,7 +141,8 @@ public void init() throws ServletException
 
     try {
       broadcastClient.start();
-    } catch(Exception e) {
+    }
+    catch (Exception e) {
       throw new ServletException(e);
     }
   }
@@ -152,7 +153,8 @@ public void destroy()
     super.destroy();
     try {
       broadcastClient.stop();
-    } catch(Exception e) {
+    }
+    catch (Exception e) {
       log.warn(e, "Error stopping servlet");
     }
   }
@@ -242,21 +244,31 @@ public void onComplete(Result result)
   }
 
   @Override
-  protected void customizeProxyRequest(Request proxyRequest, HttpServletRequest request)
+  protected void sendProxyRequest(
+      HttpServletRequest clientRequest,
+      HttpServletResponse proxyResponse,
+      Request proxyRequest
+  )
   {
     proxyRequest.timeout(httpClientConfig.getReadTimeout().getMillis(), TimeUnit.MILLISECONDS);
     proxyRequest.idleTimeout(httpClientConfig.getReadTimeout().getMillis(), TimeUnit.MILLISECONDS);
 
-    final Query query = (Query) request.getAttribute(QUERY_ATTRIBUTE);
+    final Query query = (Query) clientRequest.getAttribute(QUERY_ATTRIBUTE);
     if (query != null) {
-      final ObjectMapper objectMapper = (ObjectMapper) request.getAttribute(OBJECTMAPPER_ATTRIBUTE);
+      final ObjectMapper objectMapper = (ObjectMapper) clientRequest.getAttribute(OBJECTMAPPER_ATTRIBUTE);
       try {
         proxyRequest.content(new BytesContentProvider(objectMapper.writeValueAsBytes(query)));
       }
       catch (JsonProcessingException e) {
         Throwables.propagate(e);
       }
     }
+
+    super.sendProxyRequest(
+        clientRequest,
+        proxyResponse,
+        proxyRequest
+    );
   }
 
   @Override
@@ -273,9 +285,9 @@ protected Response.Listener newProxyResponseListener(
   }
 
   @Override
-  protected URI rewriteURI(HttpServletRequest request)
+  protected String rewriteTarget(HttpServletRequest request)
   {
-    return rewriteURI(request, (String) request.getAttribute(HOST_ATTRIBUTE));
+    return rewriteURI(request, (String) request.getAttribute(HOST_ATTRIBUTE)).toString();
   }
 
   protected URI rewriteURI(HttpServletRequest request, String host)

diff --git a/server/src/main/java/io/druid/server/http/OverlordProxyServlet.java b/server/src/main/java/io/druid/server/http/OverlordProxyServlet.java
@@ -49,7 +49,7 @@ public class OverlordProxyServlet extends ProxyServlet
   }
 
   @Override
-  protected URI rewriteURI(HttpServletRequest request)
+  protected String rewriteTarget(HttpServletRequest request)
   {
     try {
       final Server indexer = selector.pick();
@@ -62,7 +62,7 @@ protected URI rewriteURI(HttpServletRequest request)
           request.getRequestURI(),
           request.getQueryString(),
           null
-      );
+      ).toString();
     }
     catch (URISyntaxException e) {
       throw Throwables.propagate(e);

diff --git a/.../src/main/java/io/druid/server/initialization/jetty/JettyMonitoringConnectionFactory.java b/.../src/main/java/io/druid/server/initialization/jetty/JettyMonitoringConnectionFactory.java
@@ -25,6 +25,7 @@
 import org.eclipse.jetty.server.Connector;
 import org.eclipse.jetty.util.component.ContainerLifeCycle;
 
+import java.util.List;
 import java.util.concurrent.atomic.AtomicInteger;
 
 public class JettyMonitoringConnectionFactory extends ContainerLifeCycle implements ConnectionFactory
@@ -39,6 +40,12 @@ public JettyMonitoringConnectionFactory(ConnectionFactory connectionFactory, Ato
     this.activeConns = activeConns;
   }
 
+  @Override
+  public List<String> getProtocols()
+  {
+    return connectionFactory.getProtocols();
+  }
+
   @Override
   public String getProtocol()
   {

diff --git a/server/src/main/java/io/druid/server/initialization/jetty/JettyRequestLog.java b/server/src/main/java/io/druid/server/initialization/jetty/JettyRequestLog.java
@@ -38,8 +38,8 @@ public void log(Request request, Response response)
       logger.debug(
           "%s %s %s",
           request.getMethod(),
-          request.getUri().toString(),
-          request.getProtocol().toString()
+          request.getHttpURI().toString(),
+          request.getProtocol()
       );
     }
   }

diff --git a/server/src/main/java/io/druid/server/initialization/jetty/JettyServerInitUtils.java b/server/src/main/java/io/druid/server/initialization/jetty/JettyServerInitUtils.java
@@ -19,7 +19,6 @@
 
 package io.druid.server.initialization.jetty;
 
-import com.google.common.base.Joiner;
 import com.google.inject.Injector;
 import com.google.inject.Key;
 import com.google.inject.TypeLiteral;
@@ -28,40 +27,27 @@
 
 import org.eclipse.jetty.server.Handler;
 import org.eclipse.jetty.server.handler.RequestLogHandler;
+import org.eclipse.jetty.server.handler.gzip.GzipHandler;
 import org.eclipse.jetty.servlet.FilterHolder;
 import org.eclipse.jetty.servlet.ServletContextHandler;
-import org.eclipse.jetty.servlets.AsyncGzipFilter;
-import org.eclipse.jetty.servlets.GzipFilter;
 
 import javax.ws.rs.HttpMethod;
 import java.util.Set;
 
 public class JettyServerInitUtils
 {
+  private static final String[] GZIP_METHODS = new String[]{HttpMethod.GET, HttpMethod.POST};
 
-  public static final String GZIP_METHODS = Joiner.on(",").join(HttpMethod.GET, HttpMethod.POST);
-
-  public static FilterHolder defaultGzipFilterHolder()
-  {
-    final FilterHolder gzipFilterHolder = new FilterHolder(GzipFilter.class);
-    setDefaultGzipFilterHolderParameters(gzipFilterHolder);
-    return gzipFilterHolder;
-  }
-
-  public static FilterHolder defaultAsyncGzipFilterHolder()
-  {
-    final FilterHolder gzipFilterHolder = new FilterHolder(AsyncGzipFilter.class);
-    setDefaultGzipFilterHolderParameters(gzipFilterHolder);
-    return gzipFilterHolder;
-  }
-
-  private static void setDefaultGzipFilterHolderParameters(final FilterHolder filterHolder)
+  public static GzipHandler wrapWithDefaultGzipHandler(final Handler handler)
   {
-    filterHolder.setInitParameter("minGzipSize", "0");
-    filterHolder.setInitParameter("methods", GZIP_METHODS);
+    GzipHandler gzipHandler = new GzipHandler();
+    gzipHandler.setMinGzipSize(0);
+    gzipHandler.setIncludedMethods(GZIP_METHODS);
 
     // We don't actually have any precomputed .gz resources, and checking for them inside jars is expensive.
-    filterHolder.setInitParameter("checkGzExists", String.valueOf(false));
+    gzipHandler.setCheckGzExists(false);
+    gzipHandler.setHandler(handler);
+    return gzipHandler;
   }
 
   public static void addExtensionFilters(ServletContextHandler handler, Injector injector)

diff --git a/server/src/test/java/io/druid/server/AsyncQueryForwardingServletTest.java b/server/src/test/java/io/druid/server/AsyncQueryForwardingServletTest.java
@@ -249,13 +249,12 @@ protected URI rewriteURI(HttpServletRequest request, String host)
       root.addServlet(holder, "/proxy/*");
       root.addServlet(holder, "/druid/v2/*");
       JettyServerInitUtils.addExtensionFilters(root, injector);
-      root.addFilter(JettyServerInitUtils.defaultAsyncGzipFilterHolder(), "/*", null);
       root.addFilter(GuiceFilter.class, "/slow/*", null);
       root.addFilter(GuiceFilter.class, "/default/*", null);
       root.addFilter(GuiceFilter.class, "/exception/*", null);
 
       final HandlerList handlerList = new HandlerList();
-      handlerList.setHandlers(new Handler[]{root});
+      handlerList.setHandlers(new Handler[]{JettyServerInitUtils.wrapWithDefaultGzipHandler(root)});
       server.setHandler(handlerList);
     }
   }