modules

fishcus · May 25, 2017 · 8cd1351 · 8cd1351
1 parent af18b5f
commit 8cd1351
Show file tree

Hide file tree

Showing 163 changed files with 893 additions and 253 deletions.
diff --git a/beam/pom.xml b/beam/pom.xml
@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>com.github.zqhxuyuan</groupId>
+ <artifactId>kafka-book</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ </parent>
+
+ <groupId>com.github.zqhxuyuan</groupId>
+ <artifactId>beam</artifactId>
+ <version>1.0-SNAPSHOT</version>
+
+ <properties>
+ <beam.version>2.0.0</beam.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-core-java</artifactId>
+ <version>2.0.0</version>
+ </dependency>
+
+ </dependencies>
+</project>
diff --git a/bin/start-kafka08.sh b/bin/start-kafka08.sh
@@ -0,0 +1,33 @@
+rm -rf /tmp/zookeeper
+rm -rf /tmp/kafka-logs08
+
+# 进入kafka目录
+cd Soft/kafka_2.10-0.8.2.2
+
+# 启动ZooKeeper服务
+nohup bin/zookeeper-server-start.sh config/zookeeper.properties &
+
+# 启动Kafka服务
+bin/kafka-server-start.sh config/server.properties
+
+# topic列表
+bin/kafka-topics.sh --list --zookeeper localhost:2181
+
+# 创建一个topic
+bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 2 --topic test
+
+# 控制台生产者
+bin/kafka-console-producer.sh --broker-list localhost:9092 --topic test
+
+# 控制台消费者,每个控制台都是一个消费组
+bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic test --from-beginning
+
+# 进入node-zk目录
+cd ~/Github/_oss/node-zk-browser
+node app.js
+
+# 模拟同一个消费组,多个消费者
+bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic test --from-beginning --consumer.config config/consumer.properties
+bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic test --from-beginning --consumer.config config/consumer.properties
+
+
diff --git a/clients/pom.xml b/clients/pom.xml
@@ -0,0 +1,111 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>com.github.zqhxuyuan</groupId>
+ <artifactId>kafka-book</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ </parent>
+
+ <groupId>com.github.zqhxuyuan</groupId>
+ <artifactId>clients</artifactId>
+ <version>1.0-SNAPSHOT</version>
+
+ <repositories>
+ <repository>
+ <id>spring-repo</id>
+ <url>http://repo.spring.io/milestone</url>
+ </repository>
+ <repository>
+ <id>cake</id>
+ <name>https://bintray.com/cakesolutions/maven</name>
+ <url>http://dl.bintray.com/cakesolutions/maven</url>
+ </repository>
+ </repositories>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.lmax</groupId>
+ <artifactId>disruptor</artifactId>
+ <version>3.3.2</version>
+ </dependency>
+
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
+ <version>4.0.36.Final</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework.kafka</groupId>
+ <artifactId>spring-kafka</artifactId>
+ <version>1.2.1.RELEASE</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.typesafe.akka</groupId>
+ <artifactId>akka-stream-kafka_2.11</artifactId>
+ <version>0.16</version>
+ </dependency>
+
+ <dependency>
+ <groupId>io.projectreactor</groupId>
+ <artifactId>reactor-core</artifactId>
+ <version>3.1.0.M1</version>
+ </dependency>
+ <dependency>
+ <groupId>io.projectreactor.kafka</groupId>
+ <artifactId>reactor-kafka</artifactId>
+ <version>1.0.0.M2</version>
+ </dependency>
+
+ <!--
+ <dependency>
+ <groupId>net.cakesolutions</groupId>
+ <artifactId>scala-kafka-client_${scalaBin.version}</artifactId>
+ <version>0.10.2.1</version>
+ <type>pom</type>
+ </dependency>
+ <dependency>
+ <groupId>net.cakesolutions</groupId>
+ <artifactId>scala-kafka-client-akka_${scalaBin.version}</artifactId>
+ <version>0.10.2.1</version>
+ <type>pom</type>
+ </dependency>
+ -->
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ <version>3.2.0</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>compile</goal>
+ <goal>testCompile</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <mainSourceDir>src/main/scala</mainSourceDir>
+ <testSourceDir>src/test/scala</testSourceDir>
+ <scalaVersion>${scala.version}</scalaVersion>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.1</version>
+ <configuration>
+ <source>1.8</source>
+ <target>1.8</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/clients/src/main/resources/application.conf b/clients/src/main/resources/application.conf
diff --git a/src/main/resources/config.properties → clients/src/main/resources/config.properties b/src/main/resources/config.properties → clients/src/main/resources/config.properties
diff --git a/...ka/CommitConsumerToFlowProducerMain.scala → ...ms/CommitConsumerToFlowProducerMain.scala b/...ka/CommitConsumerToFlowProducerMain.scala → ...ms/CommitConsumerToFlowProducerMain.scala
@@ -1,4 +1,4 @@
-package clients.akka
+package clients.akka.streams
 
 import akka.actor.ActorSystem
 import akka.kafka.scaladsl.{Consumer, Producer}

diff --git a/...onsumerToCommitableSinkProducerMain.scala → ...onsumerToCommitableSinkProducerMain.scala b/...onsumerToCommitableSinkProducerMain.scala → ...onsumerToCommitableSinkProducerMain.scala
@@ -1,4 +1,4 @@
-package clients.akka
+package clients.akka.streams
 
 import akka.actor.ActorSystem
 import akka.kafka.ConsumerMessage.CommittableOffsetBatch

diff --git a/...scala/clients/akka/ProducerFlowDemo.scala → ...ients/akka/streams/ProducerFlowDemo.scala b/...scala/clients/akka/ProducerFlowDemo.scala → ...ients/akka/streams/ProducerFlowDemo.scala
@@ -1,4 +1,4 @@
-package clients.akka
+package clients.akka.streams
 
 import akka.actor.ActorSystem
 import akka.kafka.scaladsl.Producer

diff --git a/...ala/clients/disruptor/DisruptorEvent.java → ...ala/clients/disruptor/DisruptorEvent.java b/...ala/clients/disruptor/DisruptorEvent.java → ...ala/clients/disruptor/DisruptorEvent.java
diff --git a/...ents/disruptor/DisruptorEventFactory.java → ...ents/disruptor/DisruptorEventFactory.java b/...ents/disruptor/DisruptorEventFactory.java → ...ents/disruptor/DisruptorEventFactory.java
diff --git a/...ents/disruptor/DisruptorEventHandler.java → ...ents/disruptor/DisruptorEventHandler.java b/...ents/disruptor/DisruptorEventHandler.java → ...ents/disruptor/DisruptorEventHandler.java
diff --git a/...a/clients/disruptor/EventLogProducer.java → ...a/clients/disruptor/EventLogProducer.java b/...a/clients/disruptor/EventLogProducer.java → ...a/clients/disruptor/EventLogProducer.java
diff --git a/.../disruptor/EventTranslatorOneArgImpl.java → .../disruptor/EventTranslatorOneArgImpl.java b/.../disruptor/EventTranslatorOneArgImpl.java → .../disruptor/EventTranslatorOneArgImpl.java
diff --git a/...ala/clients/disruptor/KafkaDisruptor.java → ...ala/clients/disruptor/KafkaDisruptor.java b/...ala/clients/disruptor/KafkaDisruptor.java → ...ala/clients/disruptor/KafkaDisruptor.java
diff --git a/...nts/disruptor/KafkaDisruptorRunnable.java → ...nts/disruptor/KafkaDisruptorRunnable.java b/...nts/disruptor/KafkaDisruptorRunnable.java → ...nts/disruptor/KafkaDisruptorRunnable.java
diff --git a/...a/clients/disruptor/NewKafkaConsumer.java → ...a/clients/disruptor/NewKafkaConsumer.java b/...a/clients/disruptor/NewKafkaConsumer.java → ...a/clients/disruptor/NewKafkaConsumer.java
diff --git a/...ala/clients/disruptor/base/LongEvent.java → ...ala/clients/disruptor/base/LongEvent.java b/...ala/clients/disruptor/base/LongEvent.java → ...ala/clients/disruptor/base/LongEvent.java
diff --git a/...ents/disruptor/base/LongEventFactory.java → ...ents/disruptor/base/LongEventFactory.java b/...ents/disruptor/base/LongEventFactory.java → ...ents/disruptor/base/LongEventFactory.java
diff --git a/...ents/disruptor/base/LongEventHandler.java → ...ents/disruptor/base/LongEventHandler.java b/...ents/disruptor/base/LongEventHandler.java → ...ents/disruptor/base/LongEventHandler.java
diff --git a/...clients/disruptor/base/LongEventMain.java → ...clients/disruptor/base/LongEventMain.java b/...clients/disruptor/base/LongEventMain.java → ...clients/disruptor/base/LongEventMain.java
diff --git a/...lients/disruptor/base/LongEventMain8.java → ...lients/disruptor/base/LongEventMain8.java b/...lients/disruptor/base/LongEventMain8.java → ...lients/disruptor/base/LongEventMain8.java
diff --git a/...nts/disruptor/base/LongEventProducer.java → ...nts/disruptor/base/LongEventProducer.java b/...nts/disruptor/base/LongEventProducer.java → ...nts/disruptor/base/LongEventProducer.java
diff --git a/...base/LongEventProducerWithTranslator.java → ...base/LongEventProducerWithTranslator.java b/...base/LongEventProducerWithTranslator.java → ...base/LongEventProducerWithTranslator.java
diff --git a/...cala/clients/netty/DemoServerHandler.java → ...cala/clients/netty/DemoServerHandler.java b/...cala/clients/netty/DemoServerHandler.java → ...cala/clients/netty/DemoServerHandler.java
diff --git a/...cala/clients/netty/KafKaProducerDemo.java → ...cala/clients/netty/KafKaProducerDemo.java b/...cala/clients/netty/KafKaProducerDemo.java → ...cala/clients/netty/KafKaProducerDemo.java
diff --git a/.../scala/clients/netty/NettyServerDemo.java → .../scala/clients/netty/NettyServerDemo.java b/.../scala/clients/netty/NettyServerDemo.java → .../scala/clients/netty/NettyServerDemo.java
diff --git a/...scala/clients/reactor/SampleConsumer.java → ...scala/clients/reactor/SampleConsumer.java b/...scala/clients/reactor/SampleConsumer.java → ...scala/clients/reactor/SampleConsumer.java
diff --git a/...scala/clients/reactor/SampleProducer.java → ...scala/clients/reactor/SampleProducer.java b/...scala/clients/reactor/SampleProducer.java → ...scala/clients/reactor/SampleProducer.java
diff --git a/flink/pom.xml b/flink/pom.xml
@@ -0,0 +1,84 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>com.github.zqhxuyuan</groupId>
+ <artifactId>kafka-book</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ </parent>
+
+ <groupId>com.github.zqhxuyuan</groupId>
+ <artifactId>flink</artifactId>
+ <version>1.0-SNAPSHOT</version>
+
+ <properties>
+ <flink.version>1.2.1</flink.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-core</artifactId>
+ <version>${flink.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-scala_${scalaBin.version}</artifactId>
+ <version>${flink.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-clients_${scalaBin.version}</artifactId>
+ <version>${flink.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-scala_${scalaBin.version}</artifactId>
+ <version>${flink.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-connector-kafka-0.8_${scalaBin.version}</artifactId>
+ <version>${flink.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-connector-kafka-0.10_${scalaBin.version}</artifactId>
+ <version>${flink.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ <version>3.2.0</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>compile</goal>
+ <goal>testCompile</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <mainSourceDir>src/main/scala</mainSourceDir>
+ <testSourceDir>src/test/scala</testSourceDir>
+ <scalaVersion>${scala.version}</scalaVersion>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.1</version>
+ <configuration>
+ <source>1.8</source>
+ <target>1.8</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/...k/streaming/examples/ReadFromKafka08.java → ...k/streaming/examples/ReadFromKafka08.java b/...k/streaming/examples/ReadFromKafka08.java → ...k/streaming/examples/ReadFromKafka08.java
diff --git a/.../flink/streaming/examples/WordCount.scala → .../flink/streaming/examples/WordCount.scala b/.../flink/streaming/examples/WordCount.scala → .../flink/streaming/examples/WordCount.scala
diff --git a/.../streaming/examples/WriteIntoKafka08.java → .../streaming/examples/WriteIntoKafka08.java b/.../streaming/examples/WriteIntoKafka08.java → .../streaming/examples/WriteIntoKafka08.java
diff --git a/kafka/pom.xml b/kafka/pom.xml
@@ -0,0 +1,110 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>com.github.zqhxuyuan</groupId>
+ <artifactId>kafka-book</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ </parent>
+
+ <repositories>
+ <repository>
+ <id>confluent</id>
+ <url>http://packages.confluent.io/maven/</url>
+ </repository>
+ </repositories>
+
+ <groupId>com.github.zqhxuyuan</groupId>
+ <artifactId>kafka</artifactId>
+ <version>1.0-SNAPSHOT</version>
+
+ <dependencies>
+ <dependency>
+ <groupId>io.confluent</groupId>
+ <artifactId>kafka-avro-serializer</artifactId>
+ <version>${confluent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.confluent</groupId>
+ <artifactId>kafka-schema-registry-client</artifactId>
+ <version>${confluent.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ <version>${avro.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro-maven-plugin</artifactId>
+ <version>${avro.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro-compiler</artifactId>
+ <version>${avro.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro-ipc</artifactId>
+ <version>${avro.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro-maven-plugin</artifactId>
+ <version>1.7.7</version>
+ <executions>
+ <execution>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>schema</goal>
+ <goal>protocol</goal>
+ <goal>idl-protocol</goal>
+ </goals>
+ <!--
+ <configuration>
+ <sourceDirectory>${project.basedir}/src/main/avro/</sourceDirectory>
+ <outputDirectory>${project.basedir}/src/main/java/avro</outputDirectory>
+ </configuration>
+ -->
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ <version>3.2.0</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>compile</goal>
+ <goal>testCompile</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <mainSourceDir>src/main/scala</mainSourceDir>
+ <testSourceDir>src/test/scala</testSourceDir>
+ <scalaVersion>${scala.version}</scalaVersion>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.1</version>
+ <configuration>
+ <source>1.8</source>
+ <target>1.8</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/src/main/avro/LogLine.avsc → kafka/src/main/avro/LogLine.avsc b/src/main/avro/LogLine.avsc → kafka/src/main/avro/LogLine.avsc
diff --git a/src/main/gen-java/avro/LogLine.java → kafka/src/main/gen-java/avro/LogLine.java b/src/main/gen-java/avro/LogLine.java → kafka/src/main/gen-java/avro/LogLine.java
diff --git a/kafka/src/main/resources/config.properties b/kafka/src/main/resources/config.properties
@@ -0,0 +1,3 @@
+kafka.local=localhost:9092
+kafka.local.cluster=localhost:9092,localhost:9093,localhost:9094
+zookeeper.local=localhost:2181
diff --git a/...a/base/concurrent/CountDownLatchDemo.java → ...a/base/concurrent/CountDownLatchDemo.java b/...a/base/concurrent/CountDownLatchDemo.java → ...a/base/concurrent/CountDownLatchDemo.java
diff --git a/...scala/base/concurrent/DelayQueueDemo.java → ...scala/base/concurrent/DelayQueueDemo.java b/...scala/base/concurrent/DelayQueueDemo.java → ...scala/base/concurrent/DelayQueueDemo.java
diff --git a/...main/scala/base/concurrent/DequeTest.java → ...main/scala/base/concurrent/DequeTest.java b/...main/scala/base/concurrent/DequeTest.java → ...main/scala/base/concurrent/DequeTest.java
diff --git a/...n/scala/base/concurrent/ListIterator.java → ...n/scala/base/concurrent/ListIterator.java b/...n/scala/base/concurrent/ListIterator.java → ...n/scala/base/concurrent/ListIterator.java
diff --git a/...ain/scala/base/concurrent/Restaurant.java → ...ain/scala/base/concurrent/Restaurant.java b/...ain/scala/base/concurrent/Restaurant.java → ...ain/scala/base/concurrent/Restaurant.java
diff --git a/...scala/base/concurrent/ThreadJoinTest.java → ...scala/base/concurrent/ThreadJoinTest.java b/...scala/base/concurrent/ThreadJoinTest.java → ...scala/base/concurrent/ThreadJoinTest.java
diff --git a/...in/scala/base/concurrent/cache/Cache.java → ...in/scala/base/concurrent/cache/Cache.java b/...in/scala/base/concurrent/cache/Cache.java → ...in/scala/base/concurrent/cache/Cache.java
diff --git a/...cala/base/concurrent/cache/DelayItem.java → ...cala/base/concurrent/cache/DelayItem.java b/...cala/base/concurrent/cache/DelayItem.java → ...cala/base/concurrent/cache/DelayItem.java
diff --git a/...ain/scala/base/concurrent/cache/Pair.java → ...ain/scala/base/concurrent/cache/Pair.java b/...ain/scala/base/concurrent/cache/Pair.java → ...ain/scala/base/concurrent/cache/Pair.java
diff --git a/.../base/concurrent/future/IteratorTest.java → .../base/concurrent/future/IteratorTest.java b/.../base/concurrent/future/IteratorTest.java → .../base/concurrent/future/IteratorTest.java
diff --git a/...a/base/concurrent/future/JDBCExample.java → ...a/base/concurrent/future/JDBCExample.java b/...a/base/concurrent/future/JDBCExample.java → ...a/base/concurrent/future/JDBCExample.java
diff --git a/.../concurrent/future/JDBCFutureExample.java → .../concurrent/future/JDBCFutureExample.java b/.../concurrent/future/JDBCFutureExample.java → .../concurrent/future/JDBCFutureExample.java
diff --git a/.../base/concurrent/future/SimpleFuture.java → .../base/concurrent/future/SimpleFuture.java b/.../base/concurrent/future/SimpleFuture.java → .../base/concurrent/future/SimpleFuture.java
diff --git a/...urrent/queue/ProducerConsumerService.java → ...urrent/queue/ProducerConsumerService.java b/...urrent/queue/ProducerConsumerService.java → ...urrent/queue/ProducerConsumerService.java
diff --git a/...ent/queue/ProducerConsumerWaitNotify.java → ...ent/queue/ProducerConsumerWaitNotify.java b/...ent/queue/ProducerConsumerWaitNotify.java → ...ent/queue/ProducerConsumerWaitNotify.java