[FLINK-5369] [build] Rework jsr305 and logging dependencies.

Currently, every project in Flink has a hard (compile scope) dependency on the jsr305, slf4j, and log4j artifacts. That way they are pulled into every fat jar, including user fat jars as soon as they refer to a connector or library. This commit changes the behavior in two ways: 1. It removes the concrete logger dependencies from the root pom file. Instead, it adds them to the 'flink-core' project. That way, all modules that refer to 'flink-core' will have those dependencies as well, but the projects that have 'flink-core' as provided (connectors, libraries, user programs, etc) will have those dependencies transitively as provided as well. 2. The commit overrides the slf4j and jsr305 dependencies in the parents of 'flink-connectors', 'flink-libraries', and 'flink-metrics' and sets the to 'provided'. That way all core projects pull the logger classes, but all projects that are not part of flink-dist (and rather bundled in fat jars) will not bundle these dependencies again. The flink-dist puts the dependencies into the fat jar (slf4j, jsr305) or the lib folder (log4j).
xiaobabyLu · Dec 20, 2016 · cefb8db · cefb8db
1 parent 9bc1063
commit cefb8db
Show file tree

Hide file tree

Showing 17 changed files with 344 additions and 39 deletions.
diff --git a/flink-connectors/pom.xml b/flink-connectors/pom.xml
@@ -54,6 +54,23 @@ under the License.
 		<module>flink-connector-filesystem</module>
 	</modules>
 
+	<!-- override these root dependencies as 'provided', so they don't end up
+		in the jars-with-dependencies (uber jars) of connectors and
+		user programs that depend on the connectors -->
+
+	<dependencies>
+		<dependency>
+			<groupId>org.slf4j</groupId>
+			<artifactId>slf4j-api</artifactId>
+			<scope>provided</scope>
+		</dependency>
+		<dependency>
+			<groupId>com.google.code.findbugs</groupId>
+			<artifactId>jsr305</artifactId>
+			<scope>provided</scope>
+		</dependency>
+	</dependencies>
+
 	<!-- See main pom.xml for explanation of profiles -->
 	<profiles>
 		<!--

diff --git a/flink-core/pom.xml b/flink-core/pom.xml
@@ -47,6 +47,14 @@ under the License.
 			<version>${project.version}</version>
 		</dependency>
 
+		<!-- standard utilities -->
+		<dependency>
+			<groupId>org.apache.commons</groupId>
+			<artifactId>commons-lang3</artifactId>
+			<!-- managed version -->
+		</dependency>
+
+		<!-- for the fallback generic serializer -->
 		<dependency>
 			<groupId>com.esotericsoftware.kryo</groupId>
 			<artifactId>kryo</artifactId>
@@ -87,7 +95,28 @@ under the License.
 			<version>${asm.version}</version>
 		</dependency>
 
-		<!-- test dependencies -->
+		<!--
+			Because there are no logger implementation dependency in the root pom, we
+			add them here so that they are available during execution of code (core 
+			and example) in the IDE
+
+			NOTE: Once we are confident that users will use the newer quickstart templates,
+			we can drop these dependencies and only add them to 'flink-dist' and as test
+			dependencies
+		-->
+
+		<dependency>
+			<groupId>org.slf4j</groupId>
+			<artifactId>slf4j-log4j12</artifactId>
+		</dependency>
+
+		<dependency>
+			<groupId>log4j</groupId>
+			<artifactId>log4j</artifactId>
+		</dependency>
+
+		<!-- ================== test dependencies ================== -->
+
 		<dependency>
 			<groupId>org.apache.flink</groupId>
 			<artifactId>flink-test-utils-junit</artifactId>

diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml
@@ -35,7 +35,8 @@ under the License.
 
 	<dependencies>
 
-		<!-- BINARIES -->
+		<!-- Flink project binaries -->
+
 		<dependency>
 			<groupId>org.apache.flink</groupId>
 			<artifactId>flink-core</artifactId>
@@ -137,7 +138,22 @@ under the License.
 			<artifactId>flink-yarn_2.10</artifactId>
 			<version>${project.version}</version>
 		</dependency>
-
+
+		<!-- Concrete logging framework - we only add this here to not tie
+			the projects to one specific framework and make it easier for
+			users to swap logging frameworks -->
+
+		<dependency>
+			<groupId>org.slf4j</groupId>
+			<artifactId>slf4j-log4j12</artifactId>
+			<scope>compile</scope>
+		</dependency>
+
+		<dependency>
+			<groupId>log4j</groupId>
+			<artifactId>log4j</artifactId>
+			<scope>compile</scope>
+		</dependency>
 	</dependencies>
 
 	<profiles>

diff --git a/flink-examples/pom.xml b/flink-examples/pom.xml
@@ -33,6 +33,9 @@ under the License.
 	<packaging>pom</packaging>
 
 	<dependencies>
+
+		<!-- Flink dependencies -->
+
 		<dependency>
 			<groupId>org.apache.flink</groupId>
 			<artifactId>flink-core</artifactId>
@@ -44,6 +47,22 @@ under the License.
 			<artifactId>flink-clients_2.10</artifactId>
 			<version>${project.version}</version>
 		</dependency>
+
+		<!-- Add a logging Framework, to make the examples produce -->
+		<!--             logs when executing in the IDE            -->
+
+		<dependency>
+			<groupId>org.slf4j</groupId>
+			<artifactId>slf4j-log4j12</artifactId>
+			<scope>compile</scope>
+		</dependency>
+
+		<dependency>
+			<groupId>log4j</groupId>
+			<artifactId>log4j</artifactId>
+			<scope>compile</scope>
+		</dependency>
+
 	</dependencies>
 
 	<modules>

diff --git a/flink-java/pom.xml b/flink-java/pom.xml
@@ -53,12 +53,20 @@ under the License.
 			<version>${asm.version}</version>
 		</dependency>
 
+		<dependency>
+			<groupId>org.apache.commons</groupId>
+			<artifactId>commons-lang3</artifactId>
+			<!-- managed version -->
+		</dependency>
+
 		<dependency>
 			<groupId>org.apache.commons</groupId>
 			<artifactId>commons-math3</artifactId>
 			<!-- managed version -->
 		</dependency>
 
+		<!-- test dependencies -->
+
 		<dependency>
 			<groupId>com.google.guava</groupId>
 			<artifactId>guava</artifactId>

diff --git a/flink-libraries/flink-cep-scala/pom.xml b/flink-libraries/flink-cep-scala/pom.xml
@@ -52,12 +52,33 @@ under the License.
         </dependency>
 
         <!-- We need to add this explicitly due to shading -->
+
         <dependency>
             <groupId>org.ow2.asm</groupId>
             <artifactId>asm</artifactId>
             <version>${asm.version}</version>
         </dependency>
 
+        <!-- the dependencies below are already provided in Flink -->
+
+        <dependency>
+            <groupId>org.scala-lang</groupId>
+            <artifactId>scala-reflect</artifactId>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.scala-lang</groupId>
+            <artifactId>scala-library</artifactId>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.scala-lang</groupId>
+            <artifactId>scala-compiler</artifactId>
+            <scope>provided</scope>
+        </dependency>
+
         <!-- test dependencies -->
 
         <dependency>

diff --git a/flink-libraries/flink-gelly-examples/pom.xml b/flink-libraries/flink-gelly-examples/pom.xml
@@ -61,6 +61,20 @@
 			<version>${project.version}</version>
 		</dependency>
 
+		<!-- to be able to execute the examples properly in common IDEs, we need to
+			restate these dependencies in 'compile' scope -->
+
+		<dependency>
+			<groupId>org.slf4j</groupId>
+			<artifactId>slf4j-api</artifactId>
+			<scope>compile</scope>
+		</dependency>
+		<dependency>
+			<groupId>com.google.code.findbugs</groupId>
+			<artifactId>jsr305</artifactId>
+			<scope>compile</scope>
+		</dependency>
+
 		<!-- test dependencies -->
 
 		<dependency>

diff --git a/flink-libraries/flink-gelly-scala/pom.xml b/flink-libraries/flink-gelly-scala/pom.xml
@@ -34,29 +34,54 @@ under the License.
     <packaging>jar</packaging>
 
     <dependencies>
+
+        <!-- core dependencies -->
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-scala_2.10</artifactId>
             <version>${project.version}</version>
-			<scope>provided</scope>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-clients_2.10</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-gelly_2.10</artifactId>
+            <version>${project.version}</version>
         </dependency>
+
         <!-- We need to add this explicitly because through shading the dependency on asm seems
         to go away. -->
         <dependency>
             <groupId>org.ow2.asm</groupId>
             <artifactId>asm</artifactId>
             <version>${asm.version}</version>
         </dependency>
+
+        <!-- the dependencies below are already provided in Flink -->
+
         <dependency>
-            <groupId>org.apache.flink</groupId>
-            <artifactId>flink-clients_2.10</artifactId>
-            <version>${project.version}</version>
-			<scope>provided</scope>
+            <groupId>org.scala-lang</groupId>
+            <artifactId>scala-reflect</artifactId>
+            <scope>provided</scope>
         </dependency>
+
         <dependency>
-            <groupId>org.apache.flink</groupId>
-            <artifactId>flink-gelly_2.10</artifactId>
-            <version>${project.version}</version>
+            <groupId>org.scala-lang</groupId>
+            <artifactId>scala-library</artifactId>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.scala-lang</groupId>
+            <artifactId>scala-compiler</artifactId>
+            <scope>provided</scope>
         </dependency>
 
         <!-- test dependencies -->

diff --git a/flink-libraries/flink-gelly/pom.xml b/flink-libraries/flink-gelly/pom.xml
@@ -52,6 +52,12 @@ under the License.
 			<scope>provided</scope>
 		</dependency>
 
+		<dependency>
+			<groupId>org.apache.commons</groupId>
+			<artifactId>commons-lang3</artifactId>
+			<scope>provided</scope>
+		</dependency>
+
 		<!-- test dependencies -->
 
 		<dependency>

diff --git a/flink-libraries/flink-ml/pom.xml b/flink-libraries/flink-ml/pom.xml
@@ -50,6 +50,32 @@
 			<version>0.12</version>
 		</dependency>
 
+		<!-- the dependencies below are already provided in Flink -->
+
+		<dependency>
+			<groupId>org.scala-lang</groupId>
+			<artifactId>scala-reflect</artifactId>
+			<scope>provided</scope>
+		</dependency>
+
+		<dependency>
+			<groupId>org.scala-lang</groupId>
+			<artifactId>scala-library</artifactId>
+			<scope>provided</scope>
+		</dependency>
+
+		<dependency>
+			<groupId>org.scala-lang</groupId>
+			<artifactId>scala-compiler</artifactId>
+			<scope>provided</scope>
+		</dependency>
+
+		<dependency>
+			<groupId>org.apache.commons</groupId>
+			<artifactId>commons-math3</artifactId>
+			<scope>provided</scope>
+		</dependency>
+
 		<!-- test dependencies -->
 
 		<dependency>

diff --git a/flink-libraries/pom.xml b/flink-libraries/pom.xml
@@ -43,4 +43,22 @@ under the License.
 		<module>flink-cep</module>
 		<module>flink-cep-scala</module>
 	</modules>
+
+	<!-- override these root dependencies as 'provided', so they don't end up
+		in the jars-with-dependencies (uber jars) of connectors and
+		user programs that depend on the connectors -->
+
+	<dependencies>
+		<dependency>
+			<groupId>org.slf4j</groupId>
+			<artifactId>slf4j-api</artifactId>
+			<scope>provided</scope>
+		</dependency>
+		<dependency>
+			<groupId>com.google.code.findbugs</groupId>
+			<artifactId>jsr305</artifactId>
+			<scope>provided</scope>
+		</dependency>
+	</dependencies>
+
 </project>
diff --git a/flink-metrics/pom.xml b/flink-metrics/pom.xml
@@ -41,4 +41,22 @@ under the License.
 		<module>flink-metrics-jmx</module>
 		<module>flink-metrics-statsd</module>
 	</modules>
+
+	<!-- override these root dependencies as 'provided', so they don't end up
+		in the jars-with-dependencies. They are already contained
+		in the flink-dist build -->
+
+	<dependencies>
+		<dependency>
+			<groupId>org.slf4j</groupId>
+			<artifactId>slf4j-api</artifactId>
+			<scope>provided</scope>
+		</dependency>
+		<dependency>
+			<groupId>com.google.code.findbugs</groupId>
+			<artifactId>jsr305</artifactId>
+			<scope>provided</scope>
+		</dependency>
+	</dependencies>
+
 </project>