From 9c159818102621258da69f6f499fba095d7ded76 Mon Sep 17 00:00:00 2001 From: rmyers19 Date: Thu, 3 May 2018 11:04:45 -0400 Subject: [PATCH 001/109] initial commit --- .gitignore | 295 +++++++++++++++++++++++++++++++++++++++++++++++ README.md | 2 + requirements.txt | 2 + server.py | 20 ++++ 4 files changed, 319 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 requirements.txt create mode 100644 server.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2cf9fd4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,295 @@ +# General +.DS_Store +.AppleDouble +.LSOverride + +#linux +*.swp + +#pythonC +ENV/ + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# Typescript v1 declaration files +typings/ + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variables file +.env + + +# Mac sed bak files +*.bak + +##### JETBRAINS gitignore +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff: +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/dictionaries + +# Sensitive or high-churn files: +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.xml +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml + +# Gradle: +.idea/**/gradle.xml +.idea/**/libraries + +# CMake +cmake-build-debug/ +cmake-build-release/ + +# Mongo Explorer plugin: +.idea/**/mongoSettings.xml + +## File-based project format: +*.iws + +## Plugin-specific files: + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + + +# Mac sed bak files +*.bak + +.gradle +/build/ + +# Ignore Gradle GUI config +# gradle-app.setting +# +# # Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) +# !gradle-wrapper.jar +# +# # Cache of project +# .gradletasknamecache +# +# # # Work around https://youtrack.jetbrains.com/issue/IDEA-116898 +# # gradle/wrapper/gradle-wrapper.properties +# + +#---------------------------------------# +# Project Ignores # +#---------------------------------------# + +# Maven +target/ +pom.xml.tag +pom.xml.releaseBackup +pom.xml.versionsBackup +pom.xml.next +release.properties +dependency-reduced-pom.xml +buildNumber.properties +.mvn/timing.properties + +!/.mvn/wrapper/maven-wrapper.jar + +# Compiled class file +*.class + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files +*.jar +*.war +*.ear +*.zip +*.tar.gz +*.rar + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* + +# misc +*.log +log/ +logs/ + + +#---------------------------------------# +# IDEs & Editors Ignores # +#---------------------------------------# + +# Sublime Text +/*.sublime* +.sublime-gulp.cache + +# VS Code +/.vscode + +# JetBrains IDEs +.idea/ +*.iml + +# Eclipse +.project +.classpath +.settings + +# Netbeans +nbactions*.xml +nb-configuration*.xml + +# BlueJ +*.ctxt + + +#---------------------------------------# +# General Ignores # +#---------------------------------------# + +*~ +*.orig +.vagrant + + +#---------------------------------------# +# Linux Ignores # +#---------------------------------------# + +# KDE directory preferences +.directory + + +#---------------------------------------# +# OSX Ignores # +#---------------------------------------# + +.DS_Store +.AppleDouble +.LSOverride +.localized + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear on external disk +.Spotlight-V100 +.Trashes + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + + +#---------------------------------------# +# Windows Ignores # +#---------------------------------------# + +# Windows image file caches +Thumbs.db +ehthumbs.db + +# Folder config file +Desktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..66ebffe --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +Simple kafka/flask server. +Tutorial to come. diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0c06f78 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +flask +kafka-python diff --git a/server.py b/server.py new file mode 100644 index 0000000..da060b2 --- /dev/null +++ b/server.py @@ -0,0 +1,20 @@ +from flask import Flask, Response, request +from kafka import KafkaProducer +import json + +#producer = KafkaProducer(bootstrap_servers='host:port',value_serializer=lambda v: json.dumps(v).encode('utf-8')) + +app = Flask(__name__) + +@app.route("/", methods=["GET","POST"]) +def hello(): + print request.get_json() + + producer = KafkaProducer(bootstrap_servers='localhost:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) + producer.send(body) + return "Complete" + +if __name__ == "__main__": + app.run('0.0.0.0') + + From 1a47c13f7fb30792d77fa346f8d4b67cff4b2b7a Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Thu, 3 May 2018 11:04:45 -0400 Subject: [PATCH 002/109] initial commit --- .gitignore | 295 +++++++++++++++++++++++++++++++++++++++++++++++ README.md | 2 + requirements.txt | 2 + server.py | 20 ++++ 4 files changed, 319 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 requirements.txt create mode 100644 server.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2cf9fd4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,295 @@ +# General +.DS_Store +.AppleDouble +.LSOverride + +#linux +*.swp + +#pythonC +ENV/ + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# Typescript v1 declaration files +typings/ + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variables file +.env + + +# Mac sed bak files +*.bak + +##### JETBRAINS gitignore +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff: +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/dictionaries + +# Sensitive or high-churn files: +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.xml +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml + +# Gradle: +.idea/**/gradle.xml +.idea/**/libraries + +# CMake +cmake-build-debug/ +cmake-build-release/ + +# Mongo Explorer plugin: +.idea/**/mongoSettings.xml + +## File-based project format: +*.iws + +## Plugin-specific files: + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + + +# Mac sed bak files +*.bak + +.gradle +/build/ + +# Ignore Gradle GUI config +# gradle-app.setting +# +# # Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) +# !gradle-wrapper.jar +# +# # Cache of project +# .gradletasknamecache +# +# # # Work around https://youtrack.jetbrains.com/issue/IDEA-116898 +# # gradle/wrapper/gradle-wrapper.properties +# + +#---------------------------------------# +# Project Ignores # +#---------------------------------------# + +# Maven +target/ +pom.xml.tag +pom.xml.releaseBackup +pom.xml.versionsBackup +pom.xml.next +release.properties +dependency-reduced-pom.xml +buildNumber.properties +.mvn/timing.properties + +!/.mvn/wrapper/maven-wrapper.jar + +# Compiled class file +*.class + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files +*.jar +*.war +*.ear +*.zip +*.tar.gz +*.rar + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* + +# misc +*.log +log/ +logs/ + + +#---------------------------------------# +# IDEs & Editors Ignores # +#---------------------------------------# + +# Sublime Text +/*.sublime* +.sublime-gulp.cache + +# VS Code +/.vscode + +# JetBrains IDEs +.idea/ +*.iml + +# Eclipse +.project +.classpath +.settings + +# Netbeans +nbactions*.xml +nb-configuration*.xml + +# BlueJ +*.ctxt + + +#---------------------------------------# +# General Ignores # +#---------------------------------------# + +*~ +*.orig +.vagrant + + +#---------------------------------------# +# Linux Ignores # +#---------------------------------------# + +# KDE directory preferences +.directory + + +#---------------------------------------# +# OSX Ignores # +#---------------------------------------# + +.DS_Store +.AppleDouble +.LSOverride +.localized + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear on external disk +.Spotlight-V100 +.Trashes + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + + +#---------------------------------------# +# Windows Ignores # +#---------------------------------------# + +# Windows image file caches +Thumbs.db +ehthumbs.db + +# Folder config file +Desktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..66ebffe --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +Simple kafka/flask server. +Tutorial to come. diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0c06f78 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +flask +kafka-python diff --git a/server.py b/server.py new file mode 100644 index 0000000..da060b2 --- /dev/null +++ b/server.py @@ -0,0 +1,20 @@ +from flask import Flask, Response, request +from kafka import KafkaProducer +import json + +#producer = KafkaProducer(bootstrap_servers='host:port',value_serializer=lambda v: json.dumps(v).encode('utf-8')) + +app = Flask(__name__) + +@app.route("/", methods=["GET","POST"]) +def hello(): + print request.get_json() + + producer = KafkaProducer(bootstrap_servers='localhost:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) + producer.send(body) + return "Complete" + +if __name__ == "__main__": + app.run('0.0.0.0') + + From 7502d3966bbddb193fda1c53abc605a88a97717e Mon Sep 17 00:00:00 2001 From: rmyers19 Date: Thu, 3 May 2018 11:04:45 -0400 Subject: [PATCH 003/109] initial commit --- .gitignore | 295 +++++++++++++++++++++++++++++++++++++++++++++++ README.md | 2 + requirements.txt | 2 + server.py | 20 ++++ 4 files changed, 319 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 requirements.txt create mode 100644 server.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2cf9fd4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,295 @@ +# General +.DS_Store +.AppleDouble +.LSOverride + +#linux +*.swp + +#pythonC +ENV/ + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# Typescript v1 declaration files +typings/ + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variables file +.env + + +# Mac sed bak files +*.bak + +##### JETBRAINS gitignore +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff: +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/dictionaries + +# Sensitive or high-churn files: +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.xml +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml + +# Gradle: +.idea/**/gradle.xml +.idea/**/libraries + +# CMake +cmake-build-debug/ +cmake-build-release/ + +# Mongo Explorer plugin: +.idea/**/mongoSettings.xml + +## File-based project format: +*.iws + +## Plugin-specific files: + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + + +# Mac sed bak files +*.bak + +.gradle +/build/ + +# Ignore Gradle GUI config +# gradle-app.setting +# +# # Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) +# !gradle-wrapper.jar +# +# # Cache of project +# .gradletasknamecache +# +# # # Work around https://youtrack.jetbrains.com/issue/IDEA-116898 +# # gradle/wrapper/gradle-wrapper.properties +# + +#---------------------------------------# +# Project Ignores # +#---------------------------------------# + +# Maven +target/ +pom.xml.tag +pom.xml.releaseBackup +pom.xml.versionsBackup +pom.xml.next +release.properties +dependency-reduced-pom.xml +buildNumber.properties +.mvn/timing.properties + +!/.mvn/wrapper/maven-wrapper.jar + +# Compiled class file +*.class + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files +*.jar +*.war +*.ear +*.zip +*.tar.gz +*.rar + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* + +# misc +*.log +log/ +logs/ + + +#---------------------------------------# +# IDEs & Editors Ignores # +#---------------------------------------# + +# Sublime Text +/*.sublime* +.sublime-gulp.cache + +# VS Code +/.vscode + +# JetBrains IDEs +.idea/ +*.iml + +# Eclipse +.project +.classpath +.settings + +# Netbeans +nbactions*.xml +nb-configuration*.xml + +# BlueJ +*.ctxt + + +#---------------------------------------# +# General Ignores # +#---------------------------------------# + +*~ +*.orig +.vagrant + + +#---------------------------------------# +# Linux Ignores # +#---------------------------------------# + +# KDE directory preferences +.directory + + +#---------------------------------------# +# OSX Ignores # +#---------------------------------------# + +.DS_Store +.AppleDouble +.LSOverride +.localized + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear on external disk +.Spotlight-V100 +.Trashes + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + + +#---------------------------------------# +# Windows Ignores # +#---------------------------------------# + +# Windows image file caches +Thumbs.db +ehthumbs.db + +# Folder config file +Desktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..66ebffe --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +Simple kafka/flask server. +Tutorial to come. diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0c06f78 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +flask +kafka-python diff --git a/server.py b/server.py new file mode 100644 index 0000000..da060b2 --- /dev/null +++ b/server.py @@ -0,0 +1,20 @@ +from flask import Flask, Response, request +from kafka import KafkaProducer +import json + +#producer = KafkaProducer(bootstrap_servers='host:port',value_serializer=lambda v: json.dumps(v).encode('utf-8')) + +app = Flask(__name__) + +@app.route("/", methods=["GET","POST"]) +def hello(): + print request.get_json() + + producer = KafkaProducer(bootstrap_servers='localhost:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) + producer.send(body) + return "Complete" + +if __name__ == "__main__": + app.run('0.0.0.0') + + From 4f7b133b5f0ec595e12c115b0f281d7046ddf9ff Mon Sep 17 00:00:00 2001 From: myersr Date: Thu, 3 May 2018 11:04:45 -0400 Subject: [PATCH 004/109] initial commit --- .gitignore | 295 +++++++++++++++++++++++++++++++++++++++++++++++ README.md | 2 + requirements.txt | 2 + server.py | 20 ++++ 4 files changed, 319 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 requirements.txt create mode 100644 server.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2cf9fd4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,295 @@ +# General +.DS_Store +.AppleDouble +.LSOverride + +#linux +*.swp + +#pythonC +ENV/ + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# Typescript v1 declaration files +typings/ + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variables file +.env + + +# Mac sed bak files +*.bak + +##### JETBRAINS gitignore +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff: +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/dictionaries + +# Sensitive or high-churn files: +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.xml +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml + +# Gradle: +.idea/**/gradle.xml +.idea/**/libraries + +# CMake +cmake-build-debug/ +cmake-build-release/ + +# Mongo Explorer plugin: +.idea/**/mongoSettings.xml + +## File-based project format: +*.iws + +## Plugin-specific files: + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + + +# Mac sed bak files +*.bak + +.gradle +/build/ + +# Ignore Gradle GUI config +# gradle-app.setting +# +# # Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) +# !gradle-wrapper.jar +# +# # Cache of project +# .gradletasknamecache +# +# # # Work around https://youtrack.jetbrains.com/issue/IDEA-116898 +# # gradle/wrapper/gradle-wrapper.properties +# + +#---------------------------------------# +# Project Ignores # +#---------------------------------------# + +# Maven +target/ +pom.xml.tag +pom.xml.releaseBackup +pom.xml.versionsBackup +pom.xml.next +release.properties +dependency-reduced-pom.xml +buildNumber.properties +.mvn/timing.properties + +!/.mvn/wrapper/maven-wrapper.jar + +# Compiled class file +*.class + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files +*.jar +*.war +*.ear +*.zip +*.tar.gz +*.rar + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* + +# misc +*.log +log/ +logs/ + + +#---------------------------------------# +# IDEs & Editors Ignores # +#---------------------------------------# + +# Sublime Text +/*.sublime* +.sublime-gulp.cache + +# VS Code +/.vscode + +# JetBrains IDEs +.idea/ +*.iml + +# Eclipse +.project +.classpath +.settings + +# Netbeans +nbactions*.xml +nb-configuration*.xml + +# BlueJ +*.ctxt + + +#---------------------------------------# +# General Ignores # +#---------------------------------------# + +*~ +*.orig +.vagrant + + +#---------------------------------------# +# Linux Ignores # +#---------------------------------------# + +# KDE directory preferences +.directory + + +#---------------------------------------# +# OSX Ignores # +#---------------------------------------# + +.DS_Store +.AppleDouble +.LSOverride +.localized + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear on external disk +.Spotlight-V100 +.Trashes + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + + +#---------------------------------------# +# Windows Ignores # +#---------------------------------------# + +# Windows image file caches +Thumbs.db +ehthumbs.db + +# Folder config file +Desktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..66ebffe --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +Simple kafka/flask server. +Tutorial to come. diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0c06f78 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +flask +kafka-python diff --git a/server.py b/server.py new file mode 100644 index 0000000..da060b2 --- /dev/null +++ b/server.py @@ -0,0 +1,20 @@ +from flask import Flask, Response, request +from kafka import KafkaProducer +import json + +#producer = KafkaProducer(bootstrap_servers='host:port',value_serializer=lambda v: json.dumps(v).encode('utf-8')) + +app = Flask(__name__) + +@app.route("/", methods=["GET","POST"]) +def hello(): + print request.get_json() + + producer = KafkaProducer(bootstrap_servers='localhost:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) + producer.send(body) + return "Complete" + +if __name__ == "__main__": + app.run('0.0.0.0') + + From 2a586a45664323bcbf6c643d0cdb8ae52a1c2b8b Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Thu, 3 May 2018 11:04:45 -0400 Subject: [PATCH 005/109] initial commit --- .gitignore | 295 +++++++++++++++++++++++++++++++++++++++++++++++ README.md | 2 + requirements.txt | 2 + server.py | 20 ++++ 4 files changed, 319 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 requirements.txt create mode 100644 server.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2cf9fd4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,295 @@ +# General +.DS_Store +.AppleDouble +.LSOverride + +#linux +*.swp + +#pythonC +ENV/ + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# Typescript v1 declaration files +typings/ + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variables file +.env + + +# Mac sed bak files +*.bak + +##### JETBRAINS gitignore +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff: +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/dictionaries + +# Sensitive or high-churn files: +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.xml +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml + +# Gradle: +.idea/**/gradle.xml +.idea/**/libraries + +# CMake +cmake-build-debug/ +cmake-build-release/ + +# Mongo Explorer plugin: +.idea/**/mongoSettings.xml + +## File-based project format: +*.iws + +## Plugin-specific files: + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + + +# Mac sed bak files +*.bak + +.gradle +/build/ + +# Ignore Gradle GUI config +# gradle-app.setting +# +# # Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) +# !gradle-wrapper.jar +# +# # Cache of project +# .gradletasknamecache +# +# # # Work around https://youtrack.jetbrains.com/issue/IDEA-116898 +# # gradle/wrapper/gradle-wrapper.properties +# + +#---------------------------------------# +# Project Ignores # +#---------------------------------------# + +# Maven +target/ +pom.xml.tag +pom.xml.releaseBackup +pom.xml.versionsBackup +pom.xml.next +release.properties +dependency-reduced-pom.xml +buildNumber.properties +.mvn/timing.properties + +!/.mvn/wrapper/maven-wrapper.jar + +# Compiled class file +*.class + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files +*.jar +*.war +*.ear +*.zip +*.tar.gz +*.rar + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* + +# misc +*.log +log/ +logs/ + + +#---------------------------------------# +# IDEs & Editors Ignores # +#---------------------------------------# + +# Sublime Text +/*.sublime* +.sublime-gulp.cache + +# VS Code +/.vscode + +# JetBrains IDEs +.idea/ +*.iml + +# Eclipse +.project +.classpath +.settings + +# Netbeans +nbactions*.xml +nb-configuration*.xml + +# BlueJ +*.ctxt + + +#---------------------------------------# +# General Ignores # +#---------------------------------------# + +*~ +*.orig +.vagrant + + +#---------------------------------------# +# Linux Ignores # +#---------------------------------------# + +# KDE directory preferences +.directory + + +#---------------------------------------# +# OSX Ignores # +#---------------------------------------# + +.DS_Store +.AppleDouble +.LSOverride +.localized + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear on external disk +.Spotlight-V100 +.Trashes + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + + +#---------------------------------------# +# Windows Ignores # +#---------------------------------------# + +# Windows image file caches +Thumbs.db +ehthumbs.db + +# Folder config file +Desktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..66ebffe --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +Simple kafka/flask server. +Tutorial to come. diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0c06f78 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +flask +kafka-python diff --git a/server.py b/server.py new file mode 100644 index 0000000..da060b2 --- /dev/null +++ b/server.py @@ -0,0 +1,20 @@ +from flask import Flask, Response, request +from kafka import KafkaProducer +import json + +#producer = KafkaProducer(bootstrap_servers='host:port',value_serializer=lambda v: json.dumps(v).encode('utf-8')) + +app = Flask(__name__) + +@app.route("/", methods=["GET","POST"]) +def hello(): + print request.get_json() + + producer = KafkaProducer(bootstrap_servers='localhost:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) + producer.send(body) + return "Complete" + +if __name__ == "__main__": + app.run('0.0.0.0') + + From 749bc718d504272ebaa3b050f3659e1e592450b6 Mon Sep 17 00:00:00 2001 From: myersr Date: Thu, 3 May 2018 11:04:45 -0400 Subject: [PATCH 006/109] initial commit --- .gitignore | 295 +++++++++++++++++++++++++++++++++++++++++++++++ README.md | 2 + requirements.txt | 2 + server.py | 20 ++++ 4 files changed, 319 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 requirements.txt create mode 100644 server.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2cf9fd4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,295 @@ +# General +.DS_Store +.AppleDouble +.LSOverride + +#linux +*.swp + +#pythonC +ENV/ + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# Typescript v1 declaration files +typings/ + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variables file +.env + + +# Mac sed bak files +*.bak + +##### JETBRAINS gitignore +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff: +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/dictionaries + +# Sensitive or high-churn files: +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.xml +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml + +# Gradle: +.idea/**/gradle.xml +.idea/**/libraries + +# CMake +cmake-build-debug/ +cmake-build-release/ + +# Mongo Explorer plugin: +.idea/**/mongoSettings.xml + +## File-based project format: +*.iws + +## Plugin-specific files: + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + + +# Mac sed bak files +*.bak + +.gradle +/build/ + +# Ignore Gradle GUI config +# gradle-app.setting +# +# # Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) +# !gradle-wrapper.jar +# +# # Cache of project +# .gradletasknamecache +# +# # # Work around https://youtrack.jetbrains.com/issue/IDEA-116898 +# # gradle/wrapper/gradle-wrapper.properties +# + +#---------------------------------------# +# Project Ignores # +#---------------------------------------# + +# Maven +target/ +pom.xml.tag +pom.xml.releaseBackup +pom.xml.versionsBackup +pom.xml.next +release.properties +dependency-reduced-pom.xml +buildNumber.properties +.mvn/timing.properties + +!/.mvn/wrapper/maven-wrapper.jar + +# Compiled class file +*.class + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files +*.jar +*.war +*.ear +*.zip +*.tar.gz +*.rar + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* + +# misc +*.log +log/ +logs/ + + +#---------------------------------------# +# IDEs & Editors Ignores # +#---------------------------------------# + +# Sublime Text +/*.sublime* +.sublime-gulp.cache + +# VS Code +/.vscode + +# JetBrains IDEs +.idea/ +*.iml + +# Eclipse +.project +.classpath +.settings + +# Netbeans +nbactions*.xml +nb-configuration*.xml + +# BlueJ +*.ctxt + + +#---------------------------------------# +# General Ignores # +#---------------------------------------# + +*~ +*.orig +.vagrant + + +#---------------------------------------# +# Linux Ignores # +#---------------------------------------# + +# KDE directory preferences +.directory + + +#---------------------------------------# +# OSX Ignores # +#---------------------------------------# + +.DS_Store +.AppleDouble +.LSOverride +.localized + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear on external disk +.Spotlight-V100 +.Trashes + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + + +#---------------------------------------# +# Windows Ignores # +#---------------------------------------# + +# Windows image file caches +Thumbs.db +ehthumbs.db + +# Folder config file +Desktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..66ebffe --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +Simple kafka/flask server. +Tutorial to come. diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0c06f78 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +flask +kafka-python diff --git a/server.py b/server.py new file mode 100644 index 0000000..da060b2 --- /dev/null +++ b/server.py @@ -0,0 +1,20 @@ +from flask import Flask, Response, request +from kafka import KafkaProducer +import json + +#producer = KafkaProducer(bootstrap_servers='host:port',value_serializer=lambda v: json.dumps(v).encode('utf-8')) + +app = Flask(__name__) + +@app.route("/", methods=["GET","POST"]) +def hello(): + print request.get_json() + + producer = KafkaProducer(bootstrap_servers='localhost:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) + producer.send(body) + return "Complete" + +if __name__ == "__main__": + app.run('0.0.0.0') + + From 82258c3f83204413f07c155777e2d54442d30a7b Mon Sep 17 00:00:00 2001 From: rmyers19 Date: Thu, 3 May 2018 11:08:22 -0400 Subject: [PATCH 007/109] added links in docs --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 66ebffe..e88faa1 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,10 @@ Simple kafka/flask server. Tutorial to come. + + +---- +## Links +[kafka-python](http://kafka-python.readthedocs.io/en/master/) +[Flask](http://flask.pocoo.org/) +[Example kafka-python program/server](https://github.com/dpkp/kafka-python/blob/master/example.py) +[Spotify image of Kafka](https://hub.docker.com/r/spotify/kafka/) From 10e5620a5155c47e397d6f484fa52d753e23c4a0 Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Thu, 3 May 2018 11:08:22 -0400 Subject: [PATCH 008/109] added links in docs --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 66ebffe..e88faa1 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,10 @@ Simple kafka/flask server. Tutorial to come. + + +---- +## Links +[kafka-python](http://kafka-python.readthedocs.io/en/master/) +[Flask](http://flask.pocoo.org/) +[Example kafka-python program/server](https://github.com/dpkp/kafka-python/blob/master/example.py) +[Spotify image of Kafka](https://hub.docker.com/r/spotify/kafka/) From 2d5d4ec84c061230ee04d5e802b32f0d0608ef7b Mon Sep 17 00:00:00 2001 From: rmyers19 Date: Thu, 3 May 2018 11:08:22 -0400 Subject: [PATCH 009/109] added links in docs --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 66ebffe..e88faa1 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,10 @@ Simple kafka/flask server. Tutorial to come. + + +---- +## Links +[kafka-python](http://kafka-python.readthedocs.io/en/master/) +[Flask](http://flask.pocoo.org/) +[Example kafka-python program/server](https://github.com/dpkp/kafka-python/blob/master/example.py) +[Spotify image of Kafka](https://hub.docker.com/r/spotify/kafka/) From aa7ca077e9fb18768d3180398c1680fc54425066 Mon Sep 17 00:00:00 2001 From: myersr Date: Thu, 3 May 2018 11:08:22 -0400 Subject: [PATCH 010/109] added links in docs --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 66ebffe..e88faa1 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,10 @@ Simple kafka/flask server. Tutorial to come. + + +---- +## Links +[kafka-python](http://kafka-python.readthedocs.io/en/master/) +[Flask](http://flask.pocoo.org/) +[Example kafka-python program/server](https://github.com/dpkp/kafka-python/blob/master/example.py) +[Spotify image of Kafka](https://hub.docker.com/r/spotify/kafka/) From 4f5674030628fdbc9c88d725aff082648ba3c3bc Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Thu, 3 May 2018 11:08:22 -0400 Subject: [PATCH 011/109] added links in docs --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 66ebffe..e88faa1 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,10 @@ Simple kafka/flask server. Tutorial to come. + + +---- +## Links +[kafka-python](http://kafka-python.readthedocs.io/en/master/) +[Flask](http://flask.pocoo.org/) +[Example kafka-python program/server](https://github.com/dpkp/kafka-python/blob/master/example.py) +[Spotify image of Kafka](https://hub.docker.com/r/spotify/kafka/) From ffd39cf9768819a2aea34072b7853ff9fa5a8455 Mon Sep 17 00:00:00 2001 From: myersr Date: Thu, 3 May 2018 11:08:22 -0400 Subject: [PATCH 012/109] added links in docs --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 66ebffe..e88faa1 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,10 @@ Simple kafka/flask server. Tutorial to come. + + +---- +## Links +[kafka-python](http://kafka-python.readthedocs.io/en/master/) +[Flask](http://flask.pocoo.org/) +[Example kafka-python program/server](https://github.com/dpkp/kafka-python/blob/master/example.py) +[Spotify image of Kafka](https://hub.docker.com/r/spotify/kafka/) From 9fadca7a03cfeb3a64830d557ab68ad48008024e Mon Sep 17 00:00:00 2001 From: rmyers19 Date: Thu, 3 May 2018 11:10:00 -0400 Subject: [PATCH 013/109] added md syntax --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index e88faa1..4ec777e 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Tutorial to come. ---- ## Links -[kafka-python](http://kafka-python.readthedocs.io/en/master/) -[Flask](http://flask.pocoo.org/) -[Example kafka-python program/server](https://github.com/dpkp/kafka-python/blob/master/example.py) -[Spotify image of Kafka](https://hub.docker.com/r/spotify/kafka/) +[kafka-python](http://kafka-python.readthedocs.io/en/master/) +[Flask](http://flask.pocoo.org/) +[Example kafka-python program/server](https://github.com/dpkp/kafka-python/blob/master/example.py) +[Spotify image of Kafka](https://hub.docker.com/r/spotify/kafka/) From 67c1ea321ff07d1a71023d64171d60a46850c76a Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Thu, 3 May 2018 11:10:00 -0400 Subject: [PATCH 014/109] added md syntax --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index e88faa1..4ec777e 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Tutorial to come. ---- ## Links -[kafka-python](http://kafka-python.readthedocs.io/en/master/) -[Flask](http://flask.pocoo.org/) -[Example kafka-python program/server](https://github.com/dpkp/kafka-python/blob/master/example.py) -[Spotify image of Kafka](https://hub.docker.com/r/spotify/kafka/) +[kafka-python](http://kafka-python.readthedocs.io/en/master/) +[Flask](http://flask.pocoo.org/) +[Example kafka-python program/server](https://github.com/dpkp/kafka-python/blob/master/example.py) +[Spotify image of Kafka](https://hub.docker.com/r/spotify/kafka/) From a4b26ab41849c54a2672f6d4ea30b0050356f56a Mon Sep 17 00:00:00 2001 From: rmyers19 Date: Thu, 3 May 2018 11:10:00 -0400 Subject: [PATCH 015/109] added md syntax --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index e88faa1..4ec777e 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Tutorial to come. ---- ## Links -[kafka-python](http://kafka-python.readthedocs.io/en/master/) -[Flask](http://flask.pocoo.org/) -[Example kafka-python program/server](https://github.com/dpkp/kafka-python/blob/master/example.py) -[Spotify image of Kafka](https://hub.docker.com/r/spotify/kafka/) +[kafka-python](http://kafka-python.readthedocs.io/en/master/) +[Flask](http://flask.pocoo.org/) +[Example kafka-python program/server](https://github.com/dpkp/kafka-python/blob/master/example.py) +[Spotify image of Kafka](https://hub.docker.com/r/spotify/kafka/) From 29ae343de4c1f0fdf2752624763a95bc7af2a5a1 Mon Sep 17 00:00:00 2001 From: myersr Date: Thu, 3 May 2018 11:10:00 -0400 Subject: [PATCH 016/109] added md syntax --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index e88faa1..4ec777e 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Tutorial to come. ---- ## Links -[kafka-python](http://kafka-python.readthedocs.io/en/master/) -[Flask](http://flask.pocoo.org/) -[Example kafka-python program/server](https://github.com/dpkp/kafka-python/blob/master/example.py) -[Spotify image of Kafka](https://hub.docker.com/r/spotify/kafka/) +[kafka-python](http://kafka-python.readthedocs.io/en/master/) +[Flask](http://flask.pocoo.org/) +[Example kafka-python program/server](https://github.com/dpkp/kafka-python/blob/master/example.py) +[Spotify image of Kafka](https://hub.docker.com/r/spotify/kafka/) From 6914d06da80c137ce92dc1ac6d2d283f4471da08 Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Thu, 3 May 2018 11:10:00 -0400 Subject: [PATCH 017/109] added md syntax --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index e88faa1..4ec777e 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Tutorial to come. ---- ## Links -[kafka-python](http://kafka-python.readthedocs.io/en/master/) -[Flask](http://flask.pocoo.org/) -[Example kafka-python program/server](https://github.com/dpkp/kafka-python/blob/master/example.py) -[Spotify image of Kafka](https://hub.docker.com/r/spotify/kafka/) +[kafka-python](http://kafka-python.readthedocs.io/en/master/) +[Flask](http://flask.pocoo.org/) +[Example kafka-python program/server](https://github.com/dpkp/kafka-python/blob/master/example.py) +[Spotify image of Kafka](https://hub.docker.com/r/spotify/kafka/) From 3295c781d87b9c2360178a0f149c82a06434ca18 Mon Sep 17 00:00:00 2001 From: myersr Date: Thu, 3 May 2018 11:10:00 -0400 Subject: [PATCH 018/109] added md syntax --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index e88faa1..4ec777e 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Tutorial to come. ---- ## Links -[kafka-python](http://kafka-python.readthedocs.io/en/master/) -[Flask](http://flask.pocoo.org/) -[Example kafka-python program/server](https://github.com/dpkp/kafka-python/blob/master/example.py) -[Spotify image of Kafka](https://hub.docker.com/r/spotify/kafka/) +[kafka-python](http://kafka-python.readthedocs.io/en/master/) +[Flask](http://flask.pocoo.org/) +[Example kafka-python program/server](https://github.com/dpkp/kafka-python/blob/master/example.py) +[Spotify image of Kafka](https://hub.docker.com/r/spotify/kafka/) From 2d75b5727c6ebfb43bb19df974db3c3da53a7abe Mon Sep 17 00:00:00 2001 From: rmyers19 Date: Thu, 3 May 2018 14:44:37 -0400 Subject: [PATCH 019/109] Not sure where I was. Past tutorial level so saving herer --- README.md | 6 ++++++ kafkaProducerService.py | 48 +++++++++++++++++++++++++++++++++++++++++ server.py | 5 +++-- 3 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 kafkaProducerService.py diff --git a/README.md b/README.md index 4ec777e..f0e2eb0 100644 --- a/README.md +++ b/README.md @@ -8,3 +8,9 @@ Tutorial to come. [Flask](http://flask.pocoo.org/) [Example kafka-python program/server](https://github.com/dpkp/kafka-python/blob/master/example.py) [Spotify image of Kafka](https://hub.docker.com/r/spotify/kafka/) +[How to better use Kafka](https://gist.github.com/abacaphiliac/f0553548f9c577214d16290c2e751071) + + + +Test if Zookeeper has registered a broker: `echo dump | nc localhost 2181 | grep brokers` and look for an ID +To run Spotify container `docker run -p 2181:2181 -p 9092:9092 --env ADVERTISED_HOST=kafka-spotify --env ADVERTISED_PORT=9092 --name kafka-spotify spotify/kafka` instead of the supplied command as I don't have docker-machine. diff --git a/kafkaProducerService.py b/kafkaProducerService.py new file mode 100644 index 0000000..95b7a3e --- /dev/null +++ b/kafkaProducerService.py @@ -0,0 +1,48 @@ +import threading, logging, time +import multiprocessing + +from kafka import KafkaProducer + + +def sendStreamM(topicN, tContent): + producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092') + producer.send(topicN, tContent) + producer.close() + +class Producer(threading.Thread): + def __init__(self): + threading.Thread.__init__(self) + self.stop_event = threading.Event() + + def stop(self): + self.stop_event.set() + + def run(self): + producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092') + + #while not self.stop_event.is_set(): + # producer.send('test', b"test") + # producer.send('test', b"\xc2Hola, mundo!") + # time.sleep(3) + + producer.close() + + +def main(): + kafkProducer = Producer() + + kafkProducer.start() + + time.sleep(17) + + kafkProducer.stop() + + kafkProducer.join() + + +if __name__ == "__main__": + logging.basicConfig( + format='%(asctime)s.%(msecs)s:%(name)s:%(thread)d:%(levelname)s:%(process)d:%(message)s', + level=logging.INFO + ) + main() diff --git a/server.py b/server.py index da060b2..82427ac 100644 --- a/server.py +++ b/server.py @@ -10,11 +10,12 @@ def hello(): print request.get_json() - producer = KafkaProducer(bootstrap_servers='localhost:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) - producer.send(body) + #producer = KafkaProducer(bootstrap_servers='localhost:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) + #producer.send(body) return "Complete" if __name__ == "__main__": app.run('0.0.0.0') + print "After app.run" From 38c56a0a536592e738150240f47e15df82c6c69c Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Thu, 3 May 2018 14:44:37 -0400 Subject: [PATCH 020/109] Not sure where I was. Past tutorial level so saving herer --- README.md | 6 ++++++ kafkaProducerService.py | 48 +++++++++++++++++++++++++++++++++++++++++ server.py | 5 +++-- 3 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 kafkaProducerService.py diff --git a/README.md b/README.md index 4ec777e..f0e2eb0 100644 --- a/README.md +++ b/README.md @@ -8,3 +8,9 @@ Tutorial to come. [Flask](http://flask.pocoo.org/) [Example kafka-python program/server](https://github.com/dpkp/kafka-python/blob/master/example.py) [Spotify image of Kafka](https://hub.docker.com/r/spotify/kafka/) +[How to better use Kafka](https://gist.github.com/abacaphiliac/f0553548f9c577214d16290c2e751071) + + + +Test if Zookeeper has registered a broker: `echo dump | nc localhost 2181 | grep brokers` and look for an ID +To run Spotify container `docker run -p 2181:2181 -p 9092:9092 --env ADVERTISED_HOST=kafka-spotify --env ADVERTISED_PORT=9092 --name kafka-spotify spotify/kafka` instead of the supplied command as I don't have docker-machine. diff --git a/kafkaProducerService.py b/kafkaProducerService.py new file mode 100644 index 0000000..95b7a3e --- /dev/null +++ b/kafkaProducerService.py @@ -0,0 +1,48 @@ +import threading, logging, time +import multiprocessing + +from kafka import KafkaProducer + + +def sendStreamM(topicN, tContent): + producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092') + producer.send(topicN, tContent) + producer.close() + +class Producer(threading.Thread): + def __init__(self): + threading.Thread.__init__(self) + self.stop_event = threading.Event() + + def stop(self): + self.stop_event.set() + + def run(self): + producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092') + + #while not self.stop_event.is_set(): + # producer.send('test', b"test") + # producer.send('test', b"\xc2Hola, mundo!") + # time.sleep(3) + + producer.close() + + +def main(): + kafkProducer = Producer() + + kafkProducer.start() + + time.sleep(17) + + kafkProducer.stop() + + kafkProducer.join() + + +if __name__ == "__main__": + logging.basicConfig( + format='%(asctime)s.%(msecs)s:%(name)s:%(thread)d:%(levelname)s:%(process)d:%(message)s', + level=logging.INFO + ) + main() diff --git a/server.py b/server.py index da060b2..82427ac 100644 --- a/server.py +++ b/server.py @@ -10,11 +10,12 @@ def hello(): print request.get_json() - producer = KafkaProducer(bootstrap_servers='localhost:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) - producer.send(body) + #producer = KafkaProducer(bootstrap_servers='localhost:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) + #producer.send(body) return "Complete" if __name__ == "__main__": app.run('0.0.0.0') + print "After app.run" From 8a9bd2caf98abd5aa9b36e36ce6e09b0721a3c29 Mon Sep 17 00:00:00 2001 From: rmyers19 Date: Thu, 3 May 2018 14:44:37 -0400 Subject: [PATCH 021/109] Not sure where I was. Past tutorial level so saving herer --- README.md | 6 ++++++ kafkaProducerService.py | 48 +++++++++++++++++++++++++++++++++++++++++ server.py | 5 +++-- 3 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 kafkaProducerService.py diff --git a/README.md b/README.md index 4ec777e..f0e2eb0 100644 --- a/README.md +++ b/README.md @@ -8,3 +8,9 @@ Tutorial to come. [Flask](http://flask.pocoo.org/) [Example kafka-python program/server](https://github.com/dpkp/kafka-python/blob/master/example.py) [Spotify image of Kafka](https://hub.docker.com/r/spotify/kafka/) +[How to better use Kafka](https://gist.github.com/abacaphiliac/f0553548f9c577214d16290c2e751071) + + + +Test if Zookeeper has registered a broker: `echo dump | nc localhost 2181 | grep brokers` and look for an ID +To run Spotify container `docker run -p 2181:2181 -p 9092:9092 --env ADVERTISED_HOST=kafka-spotify --env ADVERTISED_PORT=9092 --name kafka-spotify spotify/kafka` instead of the supplied command as I don't have docker-machine. diff --git a/kafkaProducerService.py b/kafkaProducerService.py new file mode 100644 index 0000000..95b7a3e --- /dev/null +++ b/kafkaProducerService.py @@ -0,0 +1,48 @@ +import threading, logging, time +import multiprocessing + +from kafka import KafkaProducer + + +def sendStreamM(topicN, tContent): + producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092') + producer.send(topicN, tContent) + producer.close() + +class Producer(threading.Thread): + def __init__(self): + threading.Thread.__init__(self) + self.stop_event = threading.Event() + + def stop(self): + self.stop_event.set() + + def run(self): + producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092') + + #while not self.stop_event.is_set(): + # producer.send('test', b"test") + # producer.send('test', b"\xc2Hola, mundo!") + # time.sleep(3) + + producer.close() + + +def main(): + kafkProducer = Producer() + + kafkProducer.start() + + time.sleep(17) + + kafkProducer.stop() + + kafkProducer.join() + + +if __name__ == "__main__": + logging.basicConfig( + format='%(asctime)s.%(msecs)s:%(name)s:%(thread)d:%(levelname)s:%(process)d:%(message)s', + level=logging.INFO + ) + main() diff --git a/server.py b/server.py index da060b2..82427ac 100644 --- a/server.py +++ b/server.py @@ -10,11 +10,12 @@ def hello(): print request.get_json() - producer = KafkaProducer(bootstrap_servers='localhost:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) - producer.send(body) + #producer = KafkaProducer(bootstrap_servers='localhost:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) + #producer.send(body) return "Complete" if __name__ == "__main__": app.run('0.0.0.0') + print "After app.run" From 9c225095fadc99921f1f52f629d129261cc1bd17 Mon Sep 17 00:00:00 2001 From: myersr Date: Thu, 3 May 2018 14:44:37 -0400 Subject: [PATCH 022/109] Not sure where I was. Past tutorial level so saving herer --- README.md | 6 ++++++ kafkaProducerService.py | 48 +++++++++++++++++++++++++++++++++++++++++ server.py | 5 +++-- 3 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 kafkaProducerService.py diff --git a/README.md b/README.md index 4ec777e..f0e2eb0 100644 --- a/README.md +++ b/README.md @@ -8,3 +8,9 @@ Tutorial to come. [Flask](http://flask.pocoo.org/) [Example kafka-python program/server](https://github.com/dpkp/kafka-python/blob/master/example.py) [Spotify image of Kafka](https://hub.docker.com/r/spotify/kafka/) +[How to better use Kafka](https://gist.github.com/abacaphiliac/f0553548f9c577214d16290c2e751071) + + + +Test if Zookeeper has registered a broker: `echo dump | nc localhost 2181 | grep brokers` and look for an ID +To run Spotify container `docker run -p 2181:2181 -p 9092:9092 --env ADVERTISED_HOST=kafka-spotify --env ADVERTISED_PORT=9092 --name kafka-spotify spotify/kafka` instead of the supplied command as I don't have docker-machine. diff --git a/kafkaProducerService.py b/kafkaProducerService.py new file mode 100644 index 0000000..95b7a3e --- /dev/null +++ b/kafkaProducerService.py @@ -0,0 +1,48 @@ +import threading, logging, time +import multiprocessing + +from kafka import KafkaProducer + + +def sendStreamM(topicN, tContent): + producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092') + producer.send(topicN, tContent) + producer.close() + +class Producer(threading.Thread): + def __init__(self): + threading.Thread.__init__(self) + self.stop_event = threading.Event() + + def stop(self): + self.stop_event.set() + + def run(self): + producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092') + + #while not self.stop_event.is_set(): + # producer.send('test', b"test") + # producer.send('test', b"\xc2Hola, mundo!") + # time.sleep(3) + + producer.close() + + +def main(): + kafkProducer = Producer() + + kafkProducer.start() + + time.sleep(17) + + kafkProducer.stop() + + kafkProducer.join() + + +if __name__ == "__main__": + logging.basicConfig( + format='%(asctime)s.%(msecs)s:%(name)s:%(thread)d:%(levelname)s:%(process)d:%(message)s', + level=logging.INFO + ) + main() diff --git a/server.py b/server.py index da060b2..82427ac 100644 --- a/server.py +++ b/server.py @@ -10,11 +10,12 @@ def hello(): print request.get_json() - producer = KafkaProducer(bootstrap_servers='localhost:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) - producer.send(body) + #producer = KafkaProducer(bootstrap_servers='localhost:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) + #producer.send(body) return "Complete" if __name__ == "__main__": app.run('0.0.0.0') + print "After app.run" From b058646afc8d5fa77c0a9a32af3cf2ce869fe2a4 Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Thu, 3 May 2018 14:44:37 -0400 Subject: [PATCH 023/109] Not sure where I was. Past tutorial level so saving herer --- README.md | 6 ++++++ kafkaProducerService.py | 48 +++++++++++++++++++++++++++++++++++++++++ server.py | 5 +++-- 3 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 kafkaProducerService.py diff --git a/README.md b/README.md index 4ec777e..f0e2eb0 100644 --- a/README.md +++ b/README.md @@ -8,3 +8,9 @@ Tutorial to come. [Flask](http://flask.pocoo.org/) [Example kafka-python program/server](https://github.com/dpkp/kafka-python/blob/master/example.py) [Spotify image of Kafka](https://hub.docker.com/r/spotify/kafka/) +[How to better use Kafka](https://gist.github.com/abacaphiliac/f0553548f9c577214d16290c2e751071) + + + +Test if Zookeeper has registered a broker: `echo dump | nc localhost 2181 | grep brokers` and look for an ID +To run Spotify container `docker run -p 2181:2181 -p 9092:9092 --env ADVERTISED_HOST=kafka-spotify --env ADVERTISED_PORT=9092 --name kafka-spotify spotify/kafka` instead of the supplied command as I don't have docker-machine. diff --git a/kafkaProducerService.py b/kafkaProducerService.py new file mode 100644 index 0000000..95b7a3e --- /dev/null +++ b/kafkaProducerService.py @@ -0,0 +1,48 @@ +import threading, logging, time +import multiprocessing + +from kafka import KafkaProducer + + +def sendStreamM(topicN, tContent): + producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092') + producer.send(topicN, tContent) + producer.close() + +class Producer(threading.Thread): + def __init__(self): + threading.Thread.__init__(self) + self.stop_event = threading.Event() + + def stop(self): + self.stop_event.set() + + def run(self): + producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092') + + #while not self.stop_event.is_set(): + # producer.send('test', b"test") + # producer.send('test', b"\xc2Hola, mundo!") + # time.sleep(3) + + producer.close() + + +def main(): + kafkProducer = Producer() + + kafkProducer.start() + + time.sleep(17) + + kafkProducer.stop() + + kafkProducer.join() + + +if __name__ == "__main__": + logging.basicConfig( + format='%(asctime)s.%(msecs)s:%(name)s:%(thread)d:%(levelname)s:%(process)d:%(message)s', + level=logging.INFO + ) + main() diff --git a/server.py b/server.py index da060b2..82427ac 100644 --- a/server.py +++ b/server.py @@ -10,11 +10,12 @@ def hello(): print request.get_json() - producer = KafkaProducer(bootstrap_servers='localhost:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) - producer.send(body) + #producer = KafkaProducer(bootstrap_servers='localhost:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) + #producer.send(body) return "Complete" if __name__ == "__main__": app.run('0.0.0.0') + print "After app.run" From e1750d13ef75f491c0f11cdb2e55098d1be6af61 Mon Sep 17 00:00:00 2001 From: myersr Date: Thu, 3 May 2018 14:44:37 -0400 Subject: [PATCH 024/109] Not sure where I was. Past tutorial level so saving herer --- README.md | 6 ++++++ kafkaProducerService.py | 48 +++++++++++++++++++++++++++++++++++++++++ server.py | 5 +++-- 3 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 kafkaProducerService.py diff --git a/README.md b/README.md index 4ec777e..f0e2eb0 100644 --- a/README.md +++ b/README.md @@ -8,3 +8,9 @@ Tutorial to come. [Flask](http://flask.pocoo.org/) [Example kafka-python program/server](https://github.com/dpkp/kafka-python/blob/master/example.py) [Spotify image of Kafka](https://hub.docker.com/r/spotify/kafka/) +[How to better use Kafka](https://gist.github.com/abacaphiliac/f0553548f9c577214d16290c2e751071) + + + +Test if Zookeeper has registered a broker: `echo dump | nc localhost 2181 | grep brokers` and look for an ID +To run Spotify container `docker run -p 2181:2181 -p 9092:9092 --env ADVERTISED_HOST=kafka-spotify --env ADVERTISED_PORT=9092 --name kafka-spotify spotify/kafka` instead of the supplied command as I don't have docker-machine. diff --git a/kafkaProducerService.py b/kafkaProducerService.py new file mode 100644 index 0000000..95b7a3e --- /dev/null +++ b/kafkaProducerService.py @@ -0,0 +1,48 @@ +import threading, logging, time +import multiprocessing + +from kafka import KafkaProducer + + +def sendStreamM(topicN, tContent): + producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092') + producer.send(topicN, tContent) + producer.close() + +class Producer(threading.Thread): + def __init__(self): + threading.Thread.__init__(self) + self.stop_event = threading.Event() + + def stop(self): + self.stop_event.set() + + def run(self): + producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092') + + #while not self.stop_event.is_set(): + # producer.send('test', b"test") + # producer.send('test', b"\xc2Hola, mundo!") + # time.sleep(3) + + producer.close() + + +def main(): + kafkProducer = Producer() + + kafkProducer.start() + + time.sleep(17) + + kafkProducer.stop() + + kafkProducer.join() + + +if __name__ == "__main__": + logging.basicConfig( + format='%(asctime)s.%(msecs)s:%(name)s:%(thread)d:%(levelname)s:%(process)d:%(message)s', + level=logging.INFO + ) + main() diff --git a/server.py b/server.py index da060b2..82427ac 100644 --- a/server.py +++ b/server.py @@ -10,11 +10,12 @@ def hello(): print request.get_json() - producer = KafkaProducer(bootstrap_servers='localhost:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) - producer.send(body) + #producer = KafkaProducer(bootstrap_servers='localhost:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) + #producer.send(body) return "Complete" if __name__ == "__main__": app.run('0.0.0.0') + print "After app.run" From b589fed54648e99d49f81abc841538c37f0dbf74 Mon Sep 17 00:00:00 2001 From: rmyers19 Date: Thu, 3 May 2018 16:08:49 -0400 Subject: [PATCH 025/109] CORS patch added. Just opens and closes in server code. --- requirements.txt | 1 + server.py | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0c06f78..598f1ae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ flask kafka-python +flask-cors diff --git a/server.py b/server.py index 82427ac..0d2c0e6 100644 --- a/server.py +++ b/server.py @@ -1,17 +1,21 @@ from flask import Flask, Response, request +from flask_cors import CORS from kafka import KafkaProducer import json #producer = KafkaProducer(bootstrap_servers='host:port',value_serializer=lambda v: json.dumps(v).encode('utf-8')) app = Flask(__name__) +CORS(app) @app.route("/", methods=["GET","POST"]) def hello(): print request.get_json() + body = request.get_json() - #producer = KafkaProducer(bootstrap_servers='localhost:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) - #producer.send(body) + producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) + producer.send('test', body) + producer.close() return "Complete" if __name__ == "__main__": From 3ecc1aebee651f72e252a955673fab11e6c6982b Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Thu, 3 May 2018 16:08:49 -0400 Subject: [PATCH 026/109] CORS patch added. Just opens and closes in server code. --- requirements.txt | 1 + server.py | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0c06f78..598f1ae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ flask kafka-python +flask-cors diff --git a/server.py b/server.py index 82427ac..0d2c0e6 100644 --- a/server.py +++ b/server.py @@ -1,17 +1,21 @@ from flask import Flask, Response, request +from flask_cors import CORS from kafka import KafkaProducer import json #producer = KafkaProducer(bootstrap_servers='host:port',value_serializer=lambda v: json.dumps(v).encode('utf-8')) app = Flask(__name__) +CORS(app) @app.route("/", methods=["GET","POST"]) def hello(): print request.get_json() + body = request.get_json() - #producer = KafkaProducer(bootstrap_servers='localhost:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) - #producer.send(body) + producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) + producer.send('test', body) + producer.close() return "Complete" if __name__ == "__main__": From ade73207e3699589bcb20d771becc741d4e34319 Mon Sep 17 00:00:00 2001 From: rmyers19 Date: Thu, 3 May 2018 16:08:49 -0400 Subject: [PATCH 027/109] CORS patch added. Just opens and closes in server code. --- requirements.txt | 1 + server.py | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0c06f78..598f1ae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ flask kafka-python +flask-cors diff --git a/server.py b/server.py index 82427ac..0d2c0e6 100644 --- a/server.py +++ b/server.py @@ -1,17 +1,21 @@ from flask import Flask, Response, request +from flask_cors import CORS from kafka import KafkaProducer import json #producer = KafkaProducer(bootstrap_servers='host:port',value_serializer=lambda v: json.dumps(v).encode('utf-8')) app = Flask(__name__) +CORS(app) @app.route("/", methods=["GET","POST"]) def hello(): print request.get_json() + body = request.get_json() - #producer = KafkaProducer(bootstrap_servers='localhost:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) - #producer.send(body) + producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) + producer.send('test', body) + producer.close() return "Complete" if __name__ == "__main__": From 30c47302e050c34c3a4e9a20d8ec20eba573aea2 Mon Sep 17 00:00:00 2001 From: myersr Date: Thu, 3 May 2018 16:08:49 -0400 Subject: [PATCH 028/109] CORS patch added. Just opens and closes in server code. --- requirements.txt | 1 + server.py | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0c06f78..598f1ae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ flask kafka-python +flask-cors diff --git a/server.py b/server.py index 82427ac..0d2c0e6 100644 --- a/server.py +++ b/server.py @@ -1,17 +1,21 @@ from flask import Flask, Response, request +from flask_cors import CORS from kafka import KafkaProducer import json #producer = KafkaProducer(bootstrap_servers='host:port',value_serializer=lambda v: json.dumps(v).encode('utf-8')) app = Flask(__name__) +CORS(app) @app.route("/", methods=["GET","POST"]) def hello(): print request.get_json() + body = request.get_json() - #producer = KafkaProducer(bootstrap_servers='localhost:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) - #producer.send(body) + producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) + producer.send('test', body) + producer.close() return "Complete" if __name__ == "__main__": From 88663b45e3eae68778af3ea8d8e53574e21978c8 Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Thu, 3 May 2018 16:08:49 -0400 Subject: [PATCH 029/109] CORS patch added. Just opens and closes in server code. --- requirements.txt | 1 + server.py | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0c06f78..598f1ae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ flask kafka-python +flask-cors diff --git a/server.py b/server.py index 82427ac..0d2c0e6 100644 --- a/server.py +++ b/server.py @@ -1,17 +1,21 @@ from flask import Flask, Response, request +from flask_cors import CORS from kafka import KafkaProducer import json #producer = KafkaProducer(bootstrap_servers='host:port',value_serializer=lambda v: json.dumps(v).encode('utf-8')) app = Flask(__name__) +CORS(app) @app.route("/", methods=["GET","POST"]) def hello(): print request.get_json() + body = request.get_json() - #producer = KafkaProducer(bootstrap_servers='localhost:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) - #producer.send(body) + producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) + producer.send('test', body) + producer.close() return "Complete" if __name__ == "__main__": From 83da98986941a975b853ae297d9d9e7bc47e861b Mon Sep 17 00:00:00 2001 From: myersr Date: Thu, 3 May 2018 16:08:49 -0400 Subject: [PATCH 030/109] CORS patch added. Just opens and closes in server code. --- requirements.txt | 1 + server.py | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0c06f78..598f1ae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ flask kafka-python +flask-cors diff --git a/server.py b/server.py index 82427ac..0d2c0e6 100644 --- a/server.py +++ b/server.py @@ -1,17 +1,21 @@ from flask import Flask, Response, request +from flask_cors import CORS from kafka import KafkaProducer import json #producer = KafkaProducer(bootstrap_servers='host:port',value_serializer=lambda v: json.dumps(v).encode('utf-8')) app = Flask(__name__) +CORS(app) @app.route("/", methods=["GET","POST"]) def hello(): print request.get_json() + body = request.get_json() - #producer = KafkaProducer(bootstrap_servers='localhost:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) - #producer.send(body) + producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) + producer.send('test', body) + producer.close() return "Complete" if __name__ == "__main__": From e2c5f7590aa022e7785cb680c923bfb193c33531 Mon Sep 17 00:00:00 2001 From: rmyers19 Date: Thu, 3 May 2018 16:14:37 -0400 Subject: [PATCH 031/109] reverting back to a loop/sleep and text message --- kafkaProducerService.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/kafkaProducerService.py b/kafkaProducerService.py index 95b7a3e..1a15b84 100644 --- a/kafkaProducerService.py +++ b/kafkaProducerService.py @@ -4,11 +4,6 @@ from kafka import KafkaProducer -def sendStreamM(topicN, tContent): - producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092') - producer.send(topicN, tContent) - producer.close() - class Producer(threading.Thread): def __init__(self): threading.Thread.__init__(self) @@ -20,10 +15,10 @@ def stop(self): def run(self): producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092') - #while not self.stop_event.is_set(): - # producer.send('test', b"test") - # producer.send('test', b"\xc2Hola, mundo!") - # time.sleep(3) + while not self.stop_event.is_set(): + producer.send('test', b"test") + producer.send('test', b"\xc2Hola, mundo!") + time.sleep(3) producer.close() From 9e1025ac7771118cd2a496150ce0191d90c3c256 Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Thu, 3 May 2018 16:14:37 -0400 Subject: [PATCH 032/109] reverting back to a loop/sleep and text message --- kafkaProducerService.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/kafkaProducerService.py b/kafkaProducerService.py index 95b7a3e..1a15b84 100644 --- a/kafkaProducerService.py +++ b/kafkaProducerService.py @@ -4,11 +4,6 @@ from kafka import KafkaProducer -def sendStreamM(topicN, tContent): - producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092') - producer.send(topicN, tContent) - producer.close() - class Producer(threading.Thread): def __init__(self): threading.Thread.__init__(self) @@ -20,10 +15,10 @@ def stop(self): def run(self): producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092') - #while not self.stop_event.is_set(): - # producer.send('test', b"test") - # producer.send('test', b"\xc2Hola, mundo!") - # time.sleep(3) + while not self.stop_event.is_set(): + producer.send('test', b"test") + producer.send('test', b"\xc2Hola, mundo!") + time.sleep(3) producer.close() From 66074bf594e54759bbf4c6f03590fe1d4e0d529e Mon Sep 17 00:00:00 2001 From: rmyers19 Date: Thu, 3 May 2018 16:14:37 -0400 Subject: [PATCH 033/109] reverting back to a loop/sleep and text message --- kafkaProducerService.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/kafkaProducerService.py b/kafkaProducerService.py index 95b7a3e..1a15b84 100644 --- a/kafkaProducerService.py +++ b/kafkaProducerService.py @@ -4,11 +4,6 @@ from kafka import KafkaProducer -def sendStreamM(topicN, tContent): - producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092') - producer.send(topicN, tContent) - producer.close() - class Producer(threading.Thread): def __init__(self): threading.Thread.__init__(self) @@ -20,10 +15,10 @@ def stop(self): def run(self): producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092') - #while not self.stop_event.is_set(): - # producer.send('test', b"test") - # producer.send('test', b"\xc2Hola, mundo!") - # time.sleep(3) + while not self.stop_event.is_set(): + producer.send('test', b"test") + producer.send('test', b"\xc2Hola, mundo!") + time.sleep(3) producer.close() From 31fdc6e35be6e93fc71461c9abeab77f29fb1d40 Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Thu, 3 May 2018 16:14:37 -0400 Subject: [PATCH 034/109] reverting back to a loop/sleep and text message --- kafkaProducerService.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/kafkaProducerService.py b/kafkaProducerService.py index 95b7a3e..1a15b84 100644 --- a/kafkaProducerService.py +++ b/kafkaProducerService.py @@ -4,11 +4,6 @@ from kafka import KafkaProducer -def sendStreamM(topicN, tContent): - producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092') - producer.send(topicN, tContent) - producer.close() - class Producer(threading.Thread): def __init__(self): threading.Thread.__init__(self) @@ -20,10 +15,10 @@ def stop(self): def run(self): producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092') - #while not self.stop_event.is_set(): - # producer.send('test', b"test") - # producer.send('test', b"\xc2Hola, mundo!") - # time.sleep(3) + while not self.stop_event.is_set(): + producer.send('test', b"test") + producer.send('test', b"\xc2Hola, mundo!") + time.sleep(3) producer.close() From 12ef9ffb60aa442baf756d8f236cd864d3a68a99 Mon Sep 17 00:00:00 2001 From: myersr Date: Thu, 3 May 2018 16:14:37 -0400 Subject: [PATCH 035/109] reverting back to a loop/sleep and text message --- kafkaProducerService.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/kafkaProducerService.py b/kafkaProducerService.py index 95b7a3e..1a15b84 100644 --- a/kafkaProducerService.py +++ b/kafkaProducerService.py @@ -4,11 +4,6 @@ from kafka import KafkaProducer -def sendStreamM(topicN, tContent): - producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092') - producer.send(topicN, tContent) - producer.close() - class Producer(threading.Thread): def __init__(self): threading.Thread.__init__(self) @@ -20,10 +15,10 @@ def stop(self): def run(self): producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092') - #while not self.stop_event.is_set(): - # producer.send('test', b"test") - # producer.send('test', b"\xc2Hola, mundo!") - # time.sleep(3) + while not self.stop_event.is_set(): + producer.send('test', b"test") + producer.send('test', b"\xc2Hola, mundo!") + time.sleep(3) producer.close() From c5433de33d296a6820833be051ee29f2f55383b0 Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Thu, 3 May 2018 16:14:37 -0400 Subject: [PATCH 036/109] reverting back to a loop/sleep and text message --- kafkaProducerService.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/kafkaProducerService.py b/kafkaProducerService.py index 95b7a3e..1a15b84 100644 --- a/kafkaProducerService.py +++ b/kafkaProducerService.py @@ -4,11 +4,6 @@ from kafka import KafkaProducer -def sendStreamM(topicN, tContent): - producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092') - producer.send(topicN, tContent) - producer.close() - class Producer(threading.Thread): def __init__(self): threading.Thread.__init__(self) @@ -20,10 +15,10 @@ def stop(self): def run(self): producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092') - #while not self.stop_event.is_set(): - # producer.send('test', b"test") - # producer.send('test', b"\xc2Hola, mundo!") - # time.sleep(3) + while not self.stop_event.is_set(): + producer.send('test', b"test") + producer.send('test', b"\xc2Hola, mundo!") + time.sleep(3) producer.close() From 08c03a8051b8fc4fe7fe9162a633c74b4a77e037 Mon Sep 17 00:00:00 2001 From: myersr Date: Thu, 3 May 2018 16:14:37 -0400 Subject: [PATCH 037/109] reverting back to a loop/sleep and text message --- kafkaProducerService.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/kafkaProducerService.py b/kafkaProducerService.py index 95b7a3e..1a15b84 100644 --- a/kafkaProducerService.py +++ b/kafkaProducerService.py @@ -4,11 +4,6 @@ from kafka import KafkaProducer -def sendStreamM(topicN, tContent): - producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092') - producer.send(topicN, tContent) - producer.close() - class Producer(threading.Thread): def __init__(self): threading.Thread.__init__(self) @@ -20,10 +15,10 @@ def stop(self): def run(self): producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092') - #while not self.stop_event.is_set(): - # producer.send('test', b"test") - # producer.send('test', b"\xc2Hola, mundo!") - # time.sleep(3) + while not self.stop_event.is_set(): + producer.send('test', b"test") + producer.send('test', b"\xc2Hola, mundo!") + time.sleep(3) producer.close() From 404d35f94493925108a42e997e5f597aaa27edc2 Mon Sep 17 00:00:00 2001 From: rmyers19 Date: Thu, 3 May 2018 16:18:49 -0400 Subject: [PATCH 038/109] added script to listen and print messages to a topic --- scripts/listenToKafkaTopic.sh | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 scripts/listenToKafkaTopic.sh diff --git a/scripts/listenToKafkaTopic.sh b/scripts/listenToKafkaTopic.sh new file mode 100644 index 0000000..428a9a8 --- /dev/null +++ b/scripts/listenToKafkaTopic.sh @@ -0,0 +1,5 @@ +# Starts a docker container on the kafka-net docker network +# Exits on close Ctrl^C +# From spotify/kafka docker image +# Listens on topic test +docker run -it --rm --network kafka-net spotify/kafka /opt/kafka_2.11-0.10.1.0/bin/kafka-console-consumer.sh --bootstrap-server kafka-spotify:9092 --topic test --from-beginning From c881c498c7eeb3c5f2cc71f7b65e7fc8eb58fac3 Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Thu, 3 May 2018 16:18:49 -0400 Subject: [PATCH 039/109] added script to listen and print messages to a topic --- scripts/listenToKafkaTopic.sh | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 scripts/listenToKafkaTopic.sh diff --git a/scripts/listenToKafkaTopic.sh b/scripts/listenToKafkaTopic.sh new file mode 100644 index 0000000..428a9a8 --- /dev/null +++ b/scripts/listenToKafkaTopic.sh @@ -0,0 +1,5 @@ +# Starts a docker container on the kafka-net docker network +# Exits on close Ctrl^C +# From spotify/kafka docker image +# Listens on topic test +docker run -it --rm --network kafka-net spotify/kafka /opt/kafka_2.11-0.10.1.0/bin/kafka-console-consumer.sh --bootstrap-server kafka-spotify:9092 --topic test --from-beginning From 540a748405974a68abcf655fd1c7b7c535c09300 Mon Sep 17 00:00:00 2001 From: rmyers19 Date: Thu, 3 May 2018 16:18:49 -0400 Subject: [PATCH 040/109] added script to listen and print messages to a topic --- scripts/listenToKafkaTopic.sh | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 scripts/listenToKafkaTopic.sh diff --git a/scripts/listenToKafkaTopic.sh b/scripts/listenToKafkaTopic.sh new file mode 100644 index 0000000..428a9a8 --- /dev/null +++ b/scripts/listenToKafkaTopic.sh @@ -0,0 +1,5 @@ +# Starts a docker container on the kafka-net docker network +# Exits on close Ctrl^C +# From spotify/kafka docker image +# Listens on topic test +docker run -it --rm --network kafka-net spotify/kafka /opt/kafka_2.11-0.10.1.0/bin/kafka-console-consumer.sh --bootstrap-server kafka-spotify:9092 --topic test --from-beginning From 98edac5c5a80032082ad757ec6d6cd98b6ab461b Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Thu, 3 May 2018 16:18:49 -0400 Subject: [PATCH 041/109] added script to listen and print messages to a topic --- scripts/listenToKafkaTopic.sh | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 scripts/listenToKafkaTopic.sh diff --git a/scripts/listenToKafkaTopic.sh b/scripts/listenToKafkaTopic.sh new file mode 100644 index 0000000..428a9a8 --- /dev/null +++ b/scripts/listenToKafkaTopic.sh @@ -0,0 +1,5 @@ +# Starts a docker container on the kafka-net docker network +# Exits on close Ctrl^C +# From spotify/kafka docker image +# Listens on topic test +docker run -it --rm --network kafka-net spotify/kafka /opt/kafka_2.11-0.10.1.0/bin/kafka-console-consumer.sh --bootstrap-server kafka-spotify:9092 --topic test --from-beginning From 98de2a102c5b30f8791504408e6a28982bcd9577 Mon Sep 17 00:00:00 2001 From: myersr Date: Thu, 3 May 2018 16:18:49 -0400 Subject: [PATCH 042/109] added script to listen and print messages to a topic --- scripts/listenToKafkaTopic.sh | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 scripts/listenToKafkaTopic.sh diff --git a/scripts/listenToKafkaTopic.sh b/scripts/listenToKafkaTopic.sh new file mode 100644 index 0000000..428a9a8 --- /dev/null +++ b/scripts/listenToKafkaTopic.sh @@ -0,0 +1,5 @@ +# Starts a docker container on the kafka-net docker network +# Exits on close Ctrl^C +# From spotify/kafka docker image +# Listens on topic test +docker run -it --rm --network kafka-net spotify/kafka /opt/kafka_2.11-0.10.1.0/bin/kafka-console-consumer.sh --bootstrap-server kafka-spotify:9092 --topic test --from-beginning From 8d9a64959f3e4bfeb1520aac08607d9846410a5a Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Thu, 3 May 2018 16:18:49 -0400 Subject: [PATCH 043/109] added script to listen and print messages to a topic --- scripts/listenToKafkaTopic.sh | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 scripts/listenToKafkaTopic.sh diff --git a/scripts/listenToKafkaTopic.sh b/scripts/listenToKafkaTopic.sh new file mode 100644 index 0000000..428a9a8 --- /dev/null +++ b/scripts/listenToKafkaTopic.sh @@ -0,0 +1,5 @@ +# Starts a docker container on the kafka-net docker network +# Exits on close Ctrl^C +# From spotify/kafka docker image +# Listens on topic test +docker run -it --rm --network kafka-net spotify/kafka /opt/kafka_2.11-0.10.1.0/bin/kafka-console-consumer.sh --bootstrap-server kafka-spotify:9092 --topic test --from-beginning From 0605bdb42d8060a37ef5ff66230dd641da360a0b Mon Sep 17 00:00:00 2001 From: myersr Date: Thu, 3 May 2018 16:18:49 -0400 Subject: [PATCH 044/109] added script to listen and print messages to a topic --- scripts/listenToKafkaTopic.sh | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 scripts/listenToKafkaTopic.sh diff --git a/scripts/listenToKafkaTopic.sh b/scripts/listenToKafkaTopic.sh new file mode 100644 index 0000000..428a9a8 --- /dev/null +++ b/scripts/listenToKafkaTopic.sh @@ -0,0 +1,5 @@ +# Starts a docker container on the kafka-net docker network +# Exits on close Ctrl^C +# From spotify/kafka docker image +# Listens on topic test +docker run -it --rm --network kafka-net spotify/kafka /opt/kafka_2.11-0.10.1.0/bin/kafka-console-consumer.sh --bootstrap-server kafka-spotify:9092 --topic test --from-beginning From 40f45f56aface2ac2cfda82d153d7e7b1d4775ed Mon Sep 17 00:00:00 2001 From: rmyers19 Date: Thu, 3 May 2018 16:22:43 -0400 Subject: [PATCH 045/109] added two scripts to manage topics on kafka --- scripts/createTopicOnExistingKafka.sh | 3 +++ scripts/listTopicOnExistingKafka.sh | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 scripts/createTopicOnExistingKafka.sh create mode 100644 scripts/listTopicOnExistingKafka.sh diff --git a/scripts/createTopicOnExistingKafka.sh b/scripts/createTopicOnExistingKafka.sh new file mode 100644 index 0000000..2478339 --- /dev/null +++ b/scripts/createTopicOnExistingKafka.sh @@ -0,0 +1,3 @@ +# Runs command on the existing and running container kafka-spotify +# hits zookeeper and creates topic test +docker exec kafka-spotify /opt/kafka_2.11-0.10.1.0/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test diff --git a/scripts/listTopicOnExistingKafka.sh b/scripts/listTopicOnExistingKafka.sh new file mode 100644 index 0000000..ee53a6d --- /dev/null +++ b/scripts/listTopicOnExistingKafka.sh @@ -0,0 +1,3 @@ +# Runs command on existing running container kafka-spotify +# Tells zookeeper to list all topics and then it exits +docker exec kafka-spotify /opt/kafka_2.11-0.10.1.0/bin/kafka-topics.sh --list --zookeeper localhost:2181 From 0dd6ebf1af2dc0c27297c450a29d1beed553e786 Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Thu, 3 May 2018 16:22:43 -0400 Subject: [PATCH 046/109] added two scripts to manage topics on kafka --- scripts/createTopicOnExistingKafka.sh | 3 +++ scripts/listTopicOnExistingKafka.sh | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 scripts/createTopicOnExistingKafka.sh create mode 100644 scripts/listTopicOnExistingKafka.sh diff --git a/scripts/createTopicOnExistingKafka.sh b/scripts/createTopicOnExistingKafka.sh new file mode 100644 index 0000000..2478339 --- /dev/null +++ b/scripts/createTopicOnExistingKafka.sh @@ -0,0 +1,3 @@ +# Runs command on the existing and running container kafka-spotify +# hits zookeeper and creates topic test +docker exec kafka-spotify /opt/kafka_2.11-0.10.1.0/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test diff --git a/scripts/listTopicOnExistingKafka.sh b/scripts/listTopicOnExistingKafka.sh new file mode 100644 index 0000000..ee53a6d --- /dev/null +++ b/scripts/listTopicOnExistingKafka.sh @@ -0,0 +1,3 @@ +# Runs command on existing running container kafka-spotify +# Tells zookeeper to list all topics and then it exits +docker exec kafka-spotify /opt/kafka_2.11-0.10.1.0/bin/kafka-topics.sh --list --zookeeper localhost:2181 From 87d479cc3b6a91c3b03df3a6010fb0581a99d1c6 Mon Sep 17 00:00:00 2001 From: rmyers19 Date: Thu, 3 May 2018 16:22:43 -0400 Subject: [PATCH 047/109] added two scripts to manage topics on kafka --- scripts/createTopicOnExistingKafka.sh | 3 +++ scripts/listTopicOnExistingKafka.sh | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 scripts/createTopicOnExistingKafka.sh create mode 100644 scripts/listTopicOnExistingKafka.sh diff --git a/scripts/createTopicOnExistingKafka.sh b/scripts/createTopicOnExistingKafka.sh new file mode 100644 index 0000000..2478339 --- /dev/null +++ b/scripts/createTopicOnExistingKafka.sh @@ -0,0 +1,3 @@ +# Runs command on the existing and running container kafka-spotify +# hits zookeeper and creates topic test +docker exec kafka-spotify /opt/kafka_2.11-0.10.1.0/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test diff --git a/scripts/listTopicOnExistingKafka.sh b/scripts/listTopicOnExistingKafka.sh new file mode 100644 index 0000000..ee53a6d --- /dev/null +++ b/scripts/listTopicOnExistingKafka.sh @@ -0,0 +1,3 @@ +# Runs command on existing running container kafka-spotify +# Tells zookeeper to list all topics and then it exits +docker exec kafka-spotify /opt/kafka_2.11-0.10.1.0/bin/kafka-topics.sh --list --zookeeper localhost:2181 From 089883dee49a28a3f421f67c33adc8ee0bbdbd92 Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Thu, 3 May 2018 16:22:43 -0400 Subject: [PATCH 048/109] added two scripts to manage topics on kafka --- scripts/createTopicOnExistingKafka.sh | 3 +++ scripts/listTopicOnExistingKafka.sh | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 scripts/createTopicOnExistingKafka.sh create mode 100644 scripts/listTopicOnExistingKafka.sh diff --git a/scripts/createTopicOnExistingKafka.sh b/scripts/createTopicOnExistingKafka.sh new file mode 100644 index 0000000..2478339 --- /dev/null +++ b/scripts/createTopicOnExistingKafka.sh @@ -0,0 +1,3 @@ +# Runs command on the existing and running container kafka-spotify +# hits zookeeper and creates topic test +docker exec kafka-spotify /opt/kafka_2.11-0.10.1.0/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test diff --git a/scripts/listTopicOnExistingKafka.sh b/scripts/listTopicOnExistingKafka.sh new file mode 100644 index 0000000..ee53a6d --- /dev/null +++ b/scripts/listTopicOnExistingKafka.sh @@ -0,0 +1,3 @@ +# Runs command on existing running container kafka-spotify +# Tells zookeeper to list all topics and then it exits +docker exec kafka-spotify /opt/kafka_2.11-0.10.1.0/bin/kafka-topics.sh --list --zookeeper localhost:2181 From 362c28814ee087c10678e175a407745d01e6ee04 Mon Sep 17 00:00:00 2001 From: myersr Date: Thu, 3 May 2018 16:22:43 -0400 Subject: [PATCH 049/109] added two scripts to manage topics on kafka --- scripts/createTopicOnExistingKafka.sh | 3 +++ scripts/listTopicOnExistingKafka.sh | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 scripts/createTopicOnExistingKafka.sh create mode 100644 scripts/listTopicOnExistingKafka.sh diff --git a/scripts/createTopicOnExistingKafka.sh b/scripts/createTopicOnExistingKafka.sh new file mode 100644 index 0000000..2478339 --- /dev/null +++ b/scripts/createTopicOnExistingKafka.sh @@ -0,0 +1,3 @@ +# Runs command on the existing and running container kafka-spotify +# hits zookeeper and creates topic test +docker exec kafka-spotify /opt/kafka_2.11-0.10.1.0/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test diff --git a/scripts/listTopicOnExistingKafka.sh b/scripts/listTopicOnExistingKafka.sh new file mode 100644 index 0000000..ee53a6d --- /dev/null +++ b/scripts/listTopicOnExistingKafka.sh @@ -0,0 +1,3 @@ +# Runs command on existing running container kafka-spotify +# Tells zookeeper to list all topics and then it exits +docker exec kafka-spotify /opt/kafka_2.11-0.10.1.0/bin/kafka-topics.sh --list --zookeeper localhost:2181 From e62a7b1e999b0d4ab496199c24272e404f0c0a75 Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Thu, 3 May 2018 16:22:43 -0400 Subject: [PATCH 050/109] added two scripts to manage topics on kafka --- scripts/createTopicOnExistingKafka.sh | 3 +++ scripts/listTopicOnExistingKafka.sh | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 scripts/createTopicOnExistingKafka.sh create mode 100644 scripts/listTopicOnExistingKafka.sh diff --git a/scripts/createTopicOnExistingKafka.sh b/scripts/createTopicOnExistingKafka.sh new file mode 100644 index 0000000..2478339 --- /dev/null +++ b/scripts/createTopicOnExistingKafka.sh @@ -0,0 +1,3 @@ +# Runs command on the existing and running container kafka-spotify +# hits zookeeper and creates topic test +docker exec kafka-spotify /opt/kafka_2.11-0.10.1.0/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test diff --git a/scripts/listTopicOnExistingKafka.sh b/scripts/listTopicOnExistingKafka.sh new file mode 100644 index 0000000..ee53a6d --- /dev/null +++ b/scripts/listTopicOnExistingKafka.sh @@ -0,0 +1,3 @@ +# Runs command on existing running container kafka-spotify +# Tells zookeeper to list all topics and then it exits +docker exec kafka-spotify /opt/kafka_2.11-0.10.1.0/bin/kafka-topics.sh --list --zookeeper localhost:2181 From aeff6ee309e6c47392162fa07c82e4ac1bcfa7d0 Mon Sep 17 00:00:00 2001 From: myersr Date: Thu, 3 May 2018 16:22:43 -0400 Subject: [PATCH 051/109] added two scripts to manage topics on kafka --- scripts/createTopicOnExistingKafka.sh | 3 +++ scripts/listTopicOnExistingKafka.sh | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 scripts/createTopicOnExistingKafka.sh create mode 100644 scripts/listTopicOnExistingKafka.sh diff --git a/scripts/createTopicOnExistingKafka.sh b/scripts/createTopicOnExistingKafka.sh new file mode 100644 index 0000000..2478339 --- /dev/null +++ b/scripts/createTopicOnExistingKafka.sh @@ -0,0 +1,3 @@ +# Runs command on the existing and running container kafka-spotify +# hits zookeeper and creates topic test +docker exec kafka-spotify /opt/kafka_2.11-0.10.1.0/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test diff --git a/scripts/listTopicOnExistingKafka.sh b/scripts/listTopicOnExistingKafka.sh new file mode 100644 index 0000000..ee53a6d --- /dev/null +++ b/scripts/listTopicOnExistingKafka.sh @@ -0,0 +1,3 @@ +# Runs command on existing running container kafka-spotify +# Tells zookeeper to list all topics and then it exits +docker exec kafka-spotify /opt/kafka_2.11-0.10.1.0/bin/kafka-topics.sh --list --zookeeper localhost:2181 From 998a7126bc42ecf6040695444326795e233748ac Mon Sep 17 00:00:00 2001 From: rmyers19 Date: Thu, 3 May 2018 16:30:15 -0400 Subject: [PATCH 052/109] added script to create network and create docker container master --- scripts/startMasterZookeeperContainer.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 scripts/startMasterZookeeperContainer.sh diff --git a/scripts/startMasterZookeeperContainer.sh b/scripts/startMasterZookeeperContainer.sh new file mode 100644 index 0000000..cd0c59b --- /dev/null +++ b/scripts/startMasterZookeeperContainer.sh @@ -0,0 +1,10 @@ +# Starts container in the background from spotify/kafka +# Runs Zookeeper and kafka in same container +# ADVERTISED_HOST +# is set to the container name as it is the advertised domain name +# the container tries to connect to itself at this domain +# MUST add the record ` kafka-spotify` to /etc/hosts +# On kafka-net so domain record is shared by containers +# named kafka-spotify. This must be the same as the advertised_host var +docker network create -d bridge kafka-net +docker run -d -p 2181:2181 -p 9092:9092 --env ADVERTISED_HOST=kafka-spotify --env ADVERTISED_PORT=9092 --network kafka-net --name kafka-spotify spotify/kafka From e4cf5311d61dcc448cf45146724b252c9c1180b1 Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Thu, 3 May 2018 16:30:15 -0400 Subject: [PATCH 053/109] added script to create network and create docker container master --- scripts/startMasterZookeeperContainer.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 scripts/startMasterZookeeperContainer.sh diff --git a/scripts/startMasterZookeeperContainer.sh b/scripts/startMasterZookeeperContainer.sh new file mode 100644 index 0000000..cd0c59b --- /dev/null +++ b/scripts/startMasterZookeeperContainer.sh @@ -0,0 +1,10 @@ +# Starts container in the background from spotify/kafka +# Runs Zookeeper and kafka in same container +# ADVERTISED_HOST +# is set to the container name as it is the advertised domain name +# the container tries to connect to itself at this domain +# MUST add the record ` kafka-spotify` to /etc/hosts +# On kafka-net so domain record is shared by containers +# named kafka-spotify. This must be the same as the advertised_host var +docker network create -d bridge kafka-net +docker run -d -p 2181:2181 -p 9092:9092 --env ADVERTISED_HOST=kafka-spotify --env ADVERTISED_PORT=9092 --network kafka-net --name kafka-spotify spotify/kafka From c730f843c3246b854440cd79cfc3db7189b44639 Mon Sep 17 00:00:00 2001 From: rmyers19 Date: Thu, 3 May 2018 16:30:15 -0400 Subject: [PATCH 054/109] added script to create network and create docker container master --- scripts/startMasterZookeeperContainer.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 scripts/startMasterZookeeperContainer.sh diff --git a/scripts/startMasterZookeeperContainer.sh b/scripts/startMasterZookeeperContainer.sh new file mode 100644 index 0000000..cd0c59b --- /dev/null +++ b/scripts/startMasterZookeeperContainer.sh @@ -0,0 +1,10 @@ +# Starts container in the background from spotify/kafka +# Runs Zookeeper and kafka in same container +# ADVERTISED_HOST +# is set to the container name as it is the advertised domain name +# the container tries to connect to itself at this domain +# MUST add the record ` kafka-spotify` to /etc/hosts +# On kafka-net so domain record is shared by containers +# named kafka-spotify. This must be the same as the advertised_host var +docker network create -d bridge kafka-net +docker run -d -p 2181:2181 -p 9092:9092 --env ADVERTISED_HOST=kafka-spotify --env ADVERTISED_PORT=9092 --network kafka-net --name kafka-spotify spotify/kafka From 778aac5f2b8024f63ae560e0e7db99627c3fcc5b Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Thu, 3 May 2018 16:30:15 -0400 Subject: [PATCH 055/109] added script to create network and create docker container master --- scripts/startMasterZookeeperContainer.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 scripts/startMasterZookeeperContainer.sh diff --git a/scripts/startMasterZookeeperContainer.sh b/scripts/startMasterZookeeperContainer.sh new file mode 100644 index 0000000..cd0c59b --- /dev/null +++ b/scripts/startMasterZookeeperContainer.sh @@ -0,0 +1,10 @@ +# Starts container in the background from spotify/kafka +# Runs Zookeeper and kafka in same container +# ADVERTISED_HOST +# is set to the container name as it is the advertised domain name +# the container tries to connect to itself at this domain +# MUST add the record ` kafka-spotify` to /etc/hosts +# On kafka-net so domain record is shared by containers +# named kafka-spotify. This must be the same as the advertised_host var +docker network create -d bridge kafka-net +docker run -d -p 2181:2181 -p 9092:9092 --env ADVERTISED_HOST=kafka-spotify --env ADVERTISED_PORT=9092 --network kafka-net --name kafka-spotify spotify/kafka From d3e9619cf96a775f10f21b6e0f2f8aad885c8fbc Mon Sep 17 00:00:00 2001 From: myersr Date: Thu, 3 May 2018 16:30:15 -0400 Subject: [PATCH 056/109] added script to create network and create docker container master --- scripts/startMasterZookeeperContainer.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 scripts/startMasterZookeeperContainer.sh diff --git a/scripts/startMasterZookeeperContainer.sh b/scripts/startMasterZookeeperContainer.sh new file mode 100644 index 0000000..cd0c59b --- /dev/null +++ b/scripts/startMasterZookeeperContainer.sh @@ -0,0 +1,10 @@ +# Starts container in the background from spotify/kafka +# Runs Zookeeper and kafka in same container +# ADVERTISED_HOST +# is set to the container name as it is the advertised domain name +# the container tries to connect to itself at this domain +# MUST add the record ` kafka-spotify` to /etc/hosts +# On kafka-net so domain record is shared by containers +# named kafka-spotify. This must be the same as the advertised_host var +docker network create -d bridge kafka-net +docker run -d -p 2181:2181 -p 9092:9092 --env ADVERTISED_HOST=kafka-spotify --env ADVERTISED_PORT=9092 --network kafka-net --name kafka-spotify spotify/kafka From 2aca3756d225d99c89f34ed9c7d91bd73d380d71 Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Thu, 3 May 2018 16:30:15 -0400 Subject: [PATCH 057/109] added script to create network and create docker container master --- scripts/startMasterZookeeperContainer.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 scripts/startMasterZookeeperContainer.sh diff --git a/scripts/startMasterZookeeperContainer.sh b/scripts/startMasterZookeeperContainer.sh new file mode 100644 index 0000000..cd0c59b --- /dev/null +++ b/scripts/startMasterZookeeperContainer.sh @@ -0,0 +1,10 @@ +# Starts container in the background from spotify/kafka +# Runs Zookeeper and kafka in same container +# ADVERTISED_HOST +# is set to the container name as it is the advertised domain name +# the container tries to connect to itself at this domain +# MUST add the record ` kafka-spotify` to /etc/hosts +# On kafka-net so domain record is shared by containers +# named kafka-spotify. This must be the same as the advertised_host var +docker network create -d bridge kafka-net +docker run -d -p 2181:2181 -p 9092:9092 --env ADVERTISED_HOST=kafka-spotify --env ADVERTISED_PORT=9092 --network kafka-net --name kafka-spotify spotify/kafka From eadb96c3b9470ce6fcd3b1a2748007f8a1bad0b9 Mon Sep 17 00:00:00 2001 From: myersr Date: Thu, 3 May 2018 16:30:15 -0400 Subject: [PATCH 058/109] added script to create network and create docker container master --- scripts/startMasterZookeeperContainer.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 scripts/startMasterZookeeperContainer.sh diff --git a/scripts/startMasterZookeeperContainer.sh b/scripts/startMasterZookeeperContainer.sh new file mode 100644 index 0000000..cd0c59b --- /dev/null +++ b/scripts/startMasterZookeeperContainer.sh @@ -0,0 +1,10 @@ +# Starts container in the background from spotify/kafka +# Runs Zookeeper and kafka in same container +# ADVERTISED_HOST +# is set to the container name as it is the advertised domain name +# the container tries to connect to itself at this domain +# MUST add the record ` kafka-spotify` to /etc/hosts +# On kafka-net so domain record is shared by containers +# named kafka-spotify. This must be the same as the advertised_host var +docker network create -d bridge kafka-net +docker run -d -p 2181:2181 -p 9092:9092 --env ADVERTISED_HOST=kafka-spotify --env ADVERTISED_PORT=9092 --network kafka-net --name kafka-spotify spotify/kafka From 4d5c12c68f2cd85b933d4c69e345dddbd1741459 Mon Sep 17 00:00:00 2001 From: rmyers19 Date: Thu, 3 May 2018 17:27:17 -0400 Subject: [PATCH 059/109] added extensive readme and license --- LICENSE | 7 +++++ README.md | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 81 insertions(+), 3 deletions(-) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..5156403 --- /dev/null +++ b/LICENSE @@ -0,0 +1,7 @@ +Copyright 2017 Roy Myers + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md index f0e2eb0..569eb77 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,80 @@ -Simple kafka/flask server. -Tutorial to come. +# Simple kafka/flask server. +Tutorial to come and be linked. +# Project Title + +A couple examples of a python app that makes use of Kafka in a docker container, Kafka-python python library, and Flask as a webserver. The intent is to have several branches to cover multiple different scenarios utilizing Kafka. + * [kafkaProducerService.py](kafkaProducerService.py) + * Currently a multi-threaded python script that shoots out messages to a kafka consumer. + * Loops for 17 seconds. + * Thread sends 2 messages to the kafka topic 'test' every three seconds. + * only spawns one thread that is a producer + * //TODO is change it to our specific use case for the tutorial. + * [server.py](server.py) + * A flask server + * accepts a post + * submits the post body to a kafka topic 'test' as JSON. + +## Getting Started + +These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a live system `//TODO`. + +### Prerequisites + +What things you need to install the software and how to install them. + +These apps require a couple docker containers to run properly. The scripts should be all you need aside from Docker installed. +Only one of the containers is used for the functionality. The other running container is just used to repeat kafka messages and would be replaced in any kind of deployment. For those reasons I didn't create a docker-compose config. One is just a constant output feed. +``` +scripts/createTopicOnExistingKafka.sh - Runs command on the running Kafka container to create a topic. +scripts/listTopicOnExistingKafka.sh - Checks to make sure the topic was created. It runs a command on an existing Docker container +scripts/listenToKafkaTopic.sh - Spans a temporary docker container to consume from the Kafka topic and it prints the message on the screen +scripts/startMasterZookeeperContainer.sh - Creates a docker network. Starts a container in the background that runs zookeeper and Kafka in the same container. +``` + +Order to run: + * `startMasterZookeeperContainer.sh` > Output is the container ID + * `createTopicOnExistingKafka.sh` > Output is something along the lines of Topic created + * `listTopicOnExistingKafka.sh` > Output is something along the lines of current topics : + * `listenToKafkaTopic.sh` > Output is nothing at first then it is the Kafka messages as they get consumed. Container exits when you Ctrl^C + +You will end with a persistent container running Kafka and Zookeeper in the background, a container printing out to the terminal relaying messages to a Kafka topic. + +### Installing + +A step by step series of examples that tell you have to get a development env running. With python Always suggest a virtual environment. + + +Install the python packages + +``` +pip install -r requirements.txt +``` + + +Then you can run the apps. + +``` +python server.py +``` +or +``` +python kafkaProducerService.py +``` + + + +## Authors + +* **Roy Myers** - *Initial work* - [rmyers19](https://github.optum.com/rmyers19) + + +## License + +This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details ---- -## Links +## Links and Thanks! [kafka-python](http://kafka-python.readthedocs.io/en/master/) [Flask](http://flask.pocoo.org/) [Example kafka-python program/server](https://github.com/dpkp/kafka-python/blob/master/example.py) From 896948322b3ff8c771c2d8e08fb67b350cd1f073 Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Thu, 3 May 2018 17:27:17 -0400 Subject: [PATCH 060/109] added extensive readme and license --- LICENSE | 7 +++++ README.md | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 81 insertions(+), 3 deletions(-) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..5156403 --- /dev/null +++ b/LICENSE @@ -0,0 +1,7 @@ +Copyright 2017 Roy Myers + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md index f0e2eb0..569eb77 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,80 @@ -Simple kafka/flask server. -Tutorial to come. +# Simple kafka/flask server. +Tutorial to come and be linked. +# Project Title + +A couple examples of a python app that makes use of Kafka in a docker container, Kafka-python python library, and Flask as a webserver. The intent is to have several branches to cover multiple different scenarios utilizing Kafka. + * [kafkaProducerService.py](kafkaProducerService.py) + * Currently a multi-threaded python script that shoots out messages to a kafka consumer. + * Loops for 17 seconds. + * Thread sends 2 messages to the kafka topic 'test' every three seconds. + * only spawns one thread that is a producer + * //TODO is change it to our specific use case for the tutorial. + * [server.py](server.py) + * A flask server + * accepts a post + * submits the post body to a kafka topic 'test' as JSON. + +## Getting Started + +These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a live system `//TODO`. + +### Prerequisites + +What things you need to install the software and how to install them. + +These apps require a couple docker containers to run properly. The scripts should be all you need aside from Docker installed. +Only one of the containers is used for the functionality. The other running container is just used to repeat kafka messages and would be replaced in any kind of deployment. For those reasons I didn't create a docker-compose config. One is just a constant output feed. +``` +scripts/createTopicOnExistingKafka.sh - Runs command on the running Kafka container to create a topic. +scripts/listTopicOnExistingKafka.sh - Checks to make sure the topic was created. It runs a command on an existing Docker container +scripts/listenToKafkaTopic.sh - Spans a temporary docker container to consume from the Kafka topic and it prints the message on the screen +scripts/startMasterZookeeperContainer.sh - Creates a docker network. Starts a container in the background that runs zookeeper and Kafka in the same container. +``` + +Order to run: + * `startMasterZookeeperContainer.sh` > Output is the container ID + * `createTopicOnExistingKafka.sh` > Output is something along the lines of Topic created + * `listTopicOnExistingKafka.sh` > Output is something along the lines of current topics : + * `listenToKafkaTopic.sh` > Output is nothing at first then it is the Kafka messages as they get consumed. Container exits when you Ctrl^C + +You will end with a persistent container running Kafka and Zookeeper in the background, a container printing out to the terminal relaying messages to a Kafka topic. + +### Installing + +A step by step series of examples that tell you have to get a development env running. With python Always suggest a virtual environment. + + +Install the python packages + +``` +pip install -r requirements.txt +``` + + +Then you can run the apps. + +``` +python server.py +``` +or +``` +python kafkaProducerService.py +``` + + + +## Authors + +* **Roy Myers** - *Initial work* - [rmyers19](https://github.optum.com/rmyers19) + + +## License + +This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details ---- -## Links +## Links and Thanks! [kafka-python](http://kafka-python.readthedocs.io/en/master/) [Flask](http://flask.pocoo.org/) [Example kafka-python program/server](https://github.com/dpkp/kafka-python/blob/master/example.py) From 1853e5dd721739af4c99c4f0756cbca4207faac2 Mon Sep 17 00:00:00 2001 From: rmyers19 Date: Thu, 3 May 2018 17:27:17 -0400 Subject: [PATCH 061/109] added extensive readme and license --- LICENSE | 7 +++++ README.md | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 81 insertions(+), 3 deletions(-) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..5156403 --- /dev/null +++ b/LICENSE @@ -0,0 +1,7 @@ +Copyright 2017 Roy Myers + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md index f0e2eb0..569eb77 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,80 @@ -Simple kafka/flask server. -Tutorial to come. +# Simple kafka/flask server. +Tutorial to come and be linked. +# Project Title + +A couple examples of a python app that makes use of Kafka in a docker container, Kafka-python python library, and Flask as a webserver. The intent is to have several branches to cover multiple different scenarios utilizing Kafka. + * [kafkaProducerService.py](kafkaProducerService.py) + * Currently a multi-threaded python script that shoots out messages to a kafka consumer. + * Loops for 17 seconds. + * Thread sends 2 messages to the kafka topic 'test' every three seconds. + * only spawns one thread that is a producer + * //TODO is change it to our specific use case for the tutorial. + * [server.py](server.py) + * A flask server + * accepts a post + * submits the post body to a kafka topic 'test' as JSON. + +## Getting Started + +These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a live system `//TODO`. + +### Prerequisites + +What things you need to install the software and how to install them. + +These apps require a couple docker containers to run properly. The scripts should be all you need aside from Docker installed. +Only one of the containers is used for the functionality. The other running container is just used to repeat kafka messages and would be replaced in any kind of deployment. For those reasons I didn't create a docker-compose config. One is just a constant output feed. +``` +scripts/createTopicOnExistingKafka.sh - Runs command on the running Kafka container to create a topic. +scripts/listTopicOnExistingKafka.sh - Checks to make sure the topic was created. It runs a command on an existing Docker container +scripts/listenToKafkaTopic.sh - Spans a temporary docker container to consume from the Kafka topic and it prints the message on the screen +scripts/startMasterZookeeperContainer.sh - Creates a docker network. Starts a container in the background that runs zookeeper and Kafka in the same container. +``` + +Order to run: + * `startMasterZookeeperContainer.sh` > Output is the container ID + * `createTopicOnExistingKafka.sh` > Output is something along the lines of Topic created + * `listTopicOnExistingKafka.sh` > Output is something along the lines of current topics : + * `listenToKafkaTopic.sh` > Output is nothing at first then it is the Kafka messages as they get consumed. Container exits when you Ctrl^C + +You will end with a persistent container running Kafka and Zookeeper in the background, a container printing out to the terminal relaying messages to a Kafka topic. + +### Installing + +A step by step series of examples that tell you have to get a development env running. With python Always suggest a virtual environment. + + +Install the python packages + +``` +pip install -r requirements.txt +``` + + +Then you can run the apps. + +``` +python server.py +``` +or +``` +python kafkaProducerService.py +``` + + + +## Authors + +* **Roy Myers** - *Initial work* - [rmyers19](https://github.optum.com/rmyers19) + + +## License + +This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details ---- -## Links +## Links and Thanks! [kafka-python](http://kafka-python.readthedocs.io/en/master/) [Flask](http://flask.pocoo.org/) [Example kafka-python program/server](https://github.com/dpkp/kafka-python/blob/master/example.py) From d5d79bbb28303a8d97be1bb0ca71a11bb6ed5c64 Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Thu, 3 May 2018 17:27:17 -0400 Subject: [PATCH 062/109] added extensive readme and license --- LICENSE | 7 +++++ README.md | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 81 insertions(+), 3 deletions(-) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..5156403 --- /dev/null +++ b/LICENSE @@ -0,0 +1,7 @@ +Copyright 2017 Roy Myers + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md index f0e2eb0..569eb77 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,80 @@ -Simple kafka/flask server. -Tutorial to come. +# Simple kafka/flask server. +Tutorial to come and be linked. +# Project Title + +A couple examples of a python app that makes use of Kafka in a docker container, Kafka-python python library, and Flask as a webserver. The intent is to have several branches to cover multiple different scenarios utilizing Kafka. + * [kafkaProducerService.py](kafkaProducerService.py) + * Currently a multi-threaded python script that shoots out messages to a kafka consumer. + * Loops for 17 seconds. + * Thread sends 2 messages to the kafka topic 'test' every three seconds. + * only spawns one thread that is a producer + * //TODO is change it to our specific use case for the tutorial. + * [server.py](server.py) + * A flask server + * accepts a post + * submits the post body to a kafka topic 'test' as JSON. + +## Getting Started + +These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a live system `//TODO`. + +### Prerequisites + +What things you need to install the software and how to install them. + +These apps require a couple docker containers to run properly. The scripts should be all you need aside from Docker installed. +Only one of the containers is used for the functionality. The other running container is just used to repeat kafka messages and would be replaced in any kind of deployment. For those reasons I didn't create a docker-compose config. One is just a constant output feed. +``` +scripts/createTopicOnExistingKafka.sh - Runs command on the running Kafka container to create a topic. +scripts/listTopicOnExistingKafka.sh - Checks to make sure the topic was created. It runs a command on an existing Docker container +scripts/listenToKafkaTopic.sh - Spans a temporary docker container to consume from the Kafka topic and it prints the message on the screen +scripts/startMasterZookeeperContainer.sh - Creates a docker network. Starts a container in the background that runs zookeeper and Kafka in the same container. +``` + +Order to run: + * `startMasterZookeeperContainer.sh` > Output is the container ID + * `createTopicOnExistingKafka.sh` > Output is something along the lines of Topic created + * `listTopicOnExistingKafka.sh` > Output is something along the lines of current topics : + * `listenToKafkaTopic.sh` > Output is nothing at first then it is the Kafka messages as they get consumed. Container exits when you Ctrl^C + +You will end with a persistent container running Kafka and Zookeeper in the background, a container printing out to the terminal relaying messages to a Kafka topic. + +### Installing + +A step by step series of examples that tell you have to get a development env running. With python Always suggest a virtual environment. + + +Install the python packages + +``` +pip install -r requirements.txt +``` + + +Then you can run the apps. + +``` +python server.py +``` +or +``` +python kafkaProducerService.py +``` + + + +## Authors + +* **Roy Myers** - *Initial work* - [rmyers19](https://github.optum.com/rmyers19) + + +## License + +This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details ---- -## Links +## Links and Thanks! [kafka-python](http://kafka-python.readthedocs.io/en/master/) [Flask](http://flask.pocoo.org/) [Example kafka-python program/server](https://github.com/dpkp/kafka-python/blob/master/example.py) From ad1fa09dd978a23248905c8e823bd1c7dca969fe Mon Sep 17 00:00:00 2001 From: myersr Date: Thu, 3 May 2018 17:27:17 -0400 Subject: [PATCH 063/109] added extensive readme and license --- LICENSE | 7 +++++ README.md | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 81 insertions(+), 3 deletions(-) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..5156403 --- /dev/null +++ b/LICENSE @@ -0,0 +1,7 @@ +Copyright 2017 Roy Myers + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md index f0e2eb0..569eb77 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,80 @@ -Simple kafka/flask server. -Tutorial to come. +# Simple kafka/flask server. +Tutorial to come and be linked. +# Project Title + +A couple examples of a python app that makes use of Kafka in a docker container, Kafka-python python library, and Flask as a webserver. The intent is to have several branches to cover multiple different scenarios utilizing Kafka. + * [kafkaProducerService.py](kafkaProducerService.py) + * Currently a multi-threaded python script that shoots out messages to a kafka consumer. + * Loops for 17 seconds. + * Thread sends 2 messages to the kafka topic 'test' every three seconds. + * only spawns one thread that is a producer + * //TODO is change it to our specific use case for the tutorial. + * [server.py](server.py) + * A flask server + * accepts a post + * submits the post body to a kafka topic 'test' as JSON. + +## Getting Started + +These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a live system `//TODO`. + +### Prerequisites + +What things you need to install the software and how to install them. + +These apps require a couple docker containers to run properly. The scripts should be all you need aside from Docker installed. +Only one of the containers is used for the functionality. The other running container is just used to repeat kafka messages and would be replaced in any kind of deployment. For those reasons I didn't create a docker-compose config. One is just a constant output feed. +``` +scripts/createTopicOnExistingKafka.sh - Runs command on the running Kafka container to create a topic. +scripts/listTopicOnExistingKafka.sh - Checks to make sure the topic was created. It runs a command on an existing Docker container +scripts/listenToKafkaTopic.sh - Spans a temporary docker container to consume from the Kafka topic and it prints the message on the screen +scripts/startMasterZookeeperContainer.sh - Creates a docker network. Starts a container in the background that runs zookeeper and Kafka in the same container. +``` + +Order to run: + * `startMasterZookeeperContainer.sh` > Output is the container ID + * `createTopicOnExistingKafka.sh` > Output is something along the lines of Topic created + * `listTopicOnExistingKafka.sh` > Output is something along the lines of current topics : + * `listenToKafkaTopic.sh` > Output is nothing at first then it is the Kafka messages as they get consumed. Container exits when you Ctrl^C + +You will end with a persistent container running Kafka and Zookeeper in the background, a container printing out to the terminal relaying messages to a Kafka topic. + +### Installing + +A step by step series of examples that tell you have to get a development env running. With python Always suggest a virtual environment. + + +Install the python packages + +``` +pip install -r requirements.txt +``` + + +Then you can run the apps. + +``` +python server.py +``` +or +``` +python kafkaProducerService.py +``` + + + +## Authors + +* **Roy Myers** - *Initial work* - [rmyers19](https://github.optum.com/rmyers19) + + +## License + +This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details ---- -## Links +## Links and Thanks! [kafka-python](http://kafka-python.readthedocs.io/en/master/) [Flask](http://flask.pocoo.org/) [Example kafka-python program/server](https://github.com/dpkp/kafka-python/blob/master/example.py) From 2fafa642d30152e946e49740941e5f68f344ac0c Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Thu, 3 May 2018 17:27:17 -0400 Subject: [PATCH 064/109] added extensive readme and license --- LICENSE | 7 +++++ README.md | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 81 insertions(+), 3 deletions(-) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..5156403 --- /dev/null +++ b/LICENSE @@ -0,0 +1,7 @@ +Copyright 2017 Roy Myers + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md index f0e2eb0..569eb77 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,80 @@ -Simple kafka/flask server. -Tutorial to come. +# Simple kafka/flask server. +Tutorial to come and be linked. +# Project Title + +A couple examples of a python app that makes use of Kafka in a docker container, Kafka-python python library, and Flask as a webserver. The intent is to have several branches to cover multiple different scenarios utilizing Kafka. + * [kafkaProducerService.py](kafkaProducerService.py) + * Currently a multi-threaded python script that shoots out messages to a kafka consumer. + * Loops for 17 seconds. + * Thread sends 2 messages to the kafka topic 'test' every three seconds. + * only spawns one thread that is a producer + * //TODO is change it to our specific use case for the tutorial. + * [server.py](server.py) + * A flask server + * accepts a post + * submits the post body to a kafka topic 'test' as JSON. + +## Getting Started + +These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a live system `//TODO`. + +### Prerequisites + +What things you need to install the software and how to install them. + +These apps require a couple docker containers to run properly. The scripts should be all you need aside from Docker installed. +Only one of the containers is used for the functionality. The other running container is just used to repeat kafka messages and would be replaced in any kind of deployment. For those reasons I didn't create a docker-compose config. One is just a constant output feed. +``` +scripts/createTopicOnExistingKafka.sh - Runs command on the running Kafka container to create a topic. +scripts/listTopicOnExistingKafka.sh - Checks to make sure the topic was created. It runs a command on an existing Docker container +scripts/listenToKafkaTopic.sh - Spans a temporary docker container to consume from the Kafka topic and it prints the message on the screen +scripts/startMasterZookeeperContainer.sh - Creates a docker network. Starts a container in the background that runs zookeeper and Kafka in the same container. +``` + +Order to run: + * `startMasterZookeeperContainer.sh` > Output is the container ID + * `createTopicOnExistingKafka.sh` > Output is something along the lines of Topic created + * `listTopicOnExistingKafka.sh` > Output is something along the lines of current topics : + * `listenToKafkaTopic.sh` > Output is nothing at first then it is the Kafka messages as they get consumed. Container exits when you Ctrl^C + +You will end with a persistent container running Kafka and Zookeeper in the background, a container printing out to the terminal relaying messages to a Kafka topic. + +### Installing + +A step by step series of examples that tell you have to get a development env running. With python Always suggest a virtual environment. + + +Install the python packages + +``` +pip install -r requirements.txt +``` + + +Then you can run the apps. + +``` +python server.py +``` +or +``` +python kafkaProducerService.py +``` + + + +## Authors + +* **Roy Myers** - *Initial work* - [rmyers19](https://github.optum.com/rmyers19) + + +## License + +This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details ---- -## Links +## Links and Thanks! [kafka-python](http://kafka-python.readthedocs.io/en/master/) [Flask](http://flask.pocoo.org/) [Example kafka-python program/server](https://github.com/dpkp/kafka-python/blob/master/example.py) From e42b7fa95c2dce55a5e2c22652267cf7f833aba9 Mon Sep 17 00:00:00 2001 From: myersr Date: Thu, 3 May 2018 17:27:17 -0400 Subject: [PATCH 065/109] added extensive readme and license --- LICENSE | 7 +++++ README.md | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 81 insertions(+), 3 deletions(-) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..5156403 --- /dev/null +++ b/LICENSE @@ -0,0 +1,7 @@ +Copyright 2017 Roy Myers + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md index f0e2eb0..569eb77 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,80 @@ -Simple kafka/flask server. -Tutorial to come. +# Simple kafka/flask server. +Tutorial to come and be linked. +# Project Title + +A couple examples of a python app that makes use of Kafka in a docker container, Kafka-python python library, and Flask as a webserver. The intent is to have several branches to cover multiple different scenarios utilizing Kafka. + * [kafkaProducerService.py](kafkaProducerService.py) + * Currently a multi-threaded python script that shoots out messages to a kafka consumer. + * Loops for 17 seconds. + * Thread sends 2 messages to the kafka topic 'test' every three seconds. + * only spawns one thread that is a producer + * //TODO is change it to our specific use case for the tutorial. + * [server.py](server.py) + * A flask server + * accepts a post + * submits the post body to a kafka topic 'test' as JSON. + +## Getting Started + +These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a live system `//TODO`. + +### Prerequisites + +What things you need to install the software and how to install them. + +These apps require a couple docker containers to run properly. The scripts should be all you need aside from Docker installed. +Only one of the containers is used for the functionality. The other running container is just used to repeat kafka messages and would be replaced in any kind of deployment. For those reasons I didn't create a docker-compose config. One is just a constant output feed. +``` +scripts/createTopicOnExistingKafka.sh - Runs command on the running Kafka container to create a topic. +scripts/listTopicOnExistingKafka.sh - Checks to make sure the topic was created. It runs a command on an existing Docker container +scripts/listenToKafkaTopic.sh - Spans a temporary docker container to consume from the Kafka topic and it prints the message on the screen +scripts/startMasterZookeeperContainer.sh - Creates a docker network. Starts a container in the background that runs zookeeper and Kafka in the same container. +``` + +Order to run: + * `startMasterZookeeperContainer.sh` > Output is the container ID + * `createTopicOnExistingKafka.sh` > Output is something along the lines of Topic created + * `listTopicOnExistingKafka.sh` > Output is something along the lines of current topics : + * `listenToKafkaTopic.sh` > Output is nothing at first then it is the Kafka messages as they get consumed. Container exits when you Ctrl^C + +You will end with a persistent container running Kafka and Zookeeper in the background, a container printing out to the terminal relaying messages to a Kafka topic. + +### Installing + +A step by step series of examples that tell you have to get a development env running. With python Always suggest a virtual environment. + + +Install the python packages + +``` +pip install -r requirements.txt +``` + + +Then you can run the apps. + +``` +python server.py +``` +or +``` +python kafkaProducerService.py +``` + + + +## Authors + +* **Roy Myers** - *Initial work* - [rmyers19](https://github.optum.com/rmyers19) + + +## License + +This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details ---- -## Links +## Links and Thanks! [kafka-python](http://kafka-python.readthedocs.io/en/master/) [Flask](http://flask.pocoo.org/) [Example kafka-python program/server](https://github.com/dpkp/kafka-python/blob/master/example.py) From 49152924390649271f1f56cea89fd796e48d6664 Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Thu, 3 May 2018 17:27:17 -0400 Subject: [PATCH 066/109] added extensive readme and license --- LICENSE | 7 +++++ README.md | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 81 insertions(+), 3 deletions(-) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..5156403 --- /dev/null +++ b/LICENSE @@ -0,0 +1,7 @@ +Copyright 2017 Roy Myers + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md index f0e2eb0..569eb77 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,80 @@ -Simple kafka/flask server. -Tutorial to come. +# Simple kafka/flask server. +Tutorial to come and be linked. +# Project Title + +A couple examples of a python app that makes use of Kafka in a docker container, Kafka-python python library, and Flask as a webserver. The intent is to have several branches to cover multiple different scenarios utilizing Kafka. + * [kafkaProducerService.py](kafkaProducerService.py) + * Currently a multi-threaded python script that shoots out messages to a kafka consumer. + * Loops for 17 seconds. + * Thread sends 2 messages to the kafka topic 'test' every three seconds. + * only spawns one thread that is a producer + * //TODO is change it to our specific use case for the tutorial. + * [server.py](server.py) + * A flask server + * accepts a post + * submits the post body to a kafka topic 'test' as JSON. + +## Getting Started + +These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a live system `//TODO`. + +### Prerequisites + +What things you need to install the software and how to install them. + +These apps require a couple docker containers to run properly. The scripts should be all you need aside from Docker installed. +Only one of the containers is used for the functionality. The other running container is just used to repeat kafka messages and would be replaced in any kind of deployment. For those reasons I didn't create a docker-compose config. One is just a constant output feed. +``` +scripts/createTopicOnExistingKafka.sh - Runs command on the running Kafka container to create a topic. +scripts/listTopicOnExistingKafka.sh - Checks to make sure the topic was created. It runs a command on an existing Docker container +scripts/listenToKafkaTopic.sh - Spans a temporary docker container to consume from the Kafka topic and it prints the message on the screen +scripts/startMasterZookeeperContainer.sh - Creates a docker network. Starts a container in the background that runs zookeeper and Kafka in the same container. +``` + +Order to run: + * `startMasterZookeeperContainer.sh` > Output is the container ID + * `createTopicOnExistingKafka.sh` > Output is something along the lines of Topic created + * `listTopicOnExistingKafka.sh` > Output is something along the lines of current topics : + * `listenToKafkaTopic.sh` > Output is nothing at first then it is the Kafka messages as they get consumed. Container exits when you Ctrl^C + +You will end with a persistent container running Kafka and Zookeeper in the background, a container printing out to the terminal relaying messages to a Kafka topic. + +### Installing + +A step by step series of examples that tell you have to get a development env running. With python Always suggest a virtual environment. + + +Install the python packages + +``` +pip install -r requirements.txt +``` + + +Then you can run the apps. + +``` +python server.py +``` +or +``` +python kafkaProducerService.py +``` + + + +## Authors + +* **Roy Myers** - *Initial work* - [rmyers19](https://github.optum.com/rmyers19) + + +## License + +This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details ---- -## Links +## Links and Thanks! [kafka-python](http://kafka-python.readthedocs.io/en/master/) [Flask](http://flask.pocoo.org/) [Example kafka-python program/server](https://github.com/dpkp/kafka-python/blob/master/example.py) From 2a142ac07df325af79af026495be22d37ee54a51 Mon Sep 17 00:00:00 2001 From: rmyers19 Date: Thu, 3 May 2018 18:00:15 -0400 Subject: [PATCH 067/109] commented the main code --- kafkaProducerService.py | 38 +++++++++++++++++++++++++++++++++----- server.py | 21 ++++++++++++++++++--- 2 files changed, 51 insertions(+), 8 deletions(-) diff --git a/kafkaProducerService.py b/kafkaProducerService.py index 1a15b84..5d10fa7 100644 --- a/kafkaProducerService.py +++ b/kafkaProducerService.py @@ -4,40 +4,68 @@ from kafka import KafkaProducer +#Thread class that continuously sends messages to a Kafka message queue +# Input: +# threading.Thread = instance of multiproccessing thread class Producer(threading.Thread): + #default initializing function of the thread. + # Input: + # takes self to modify and initialize def __init__(self): + #initializes thread and passes self. Magic multithreading stuff threading.Thread.__init__(self) + #Gives the thread an envent called stop_event so it can be interrupted. self.stop_event = threading.Event() - + + #Function to stop the thread def stop(self): + #Calls even stop_event and sets it. + #This gives context to the thread from the outside and lets you stop it. self.stop_event.set() - + + #The main run function called when you call start. def run(self): - producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092') + #Bootstraps an instance of a Kafka producer. + #Initializes the producer and identifies the docker server. + #kafka-spotify is listed in /etc/hosts with the ip of the container + producer = KafkaProducer(bootstrap_servers='kafka-spotify:9092') + #loop until the thread is stopped by checking the stop event while not self.stop_event.is_set(): + #Send two messages of type binary to the 'test' Topic producer.send('test', b"test") producer.send('test', b"\xc2Hola, mundo!") + #Sleep for 3 seconds time.sleep(3) - + + #Close the TCP stream to Kafka producer.close() +#Main function called when the app is run def main(): + #initialize a producer object/thread kafkProducer = Producer() + #Start the thread working. kafkProducer.start() + #sleep for 17 second. If we weren't using threads this would halt the code time.sleep(17) + #Call stop to set the thread event so it knows to stop kafkProducer.stop() + #Wait until the thread terminates. Can see the docs for more + #https://docs.python.org/2/library/threading.html?highlight=thread#threading.Thread.join kafkProducer.join() - +#the logic of running as process if __name__ == "__main__": + #Set logging format and level logging.basicConfig( format='%(asctime)s.%(msecs)s:%(name)s:%(thread)d:%(levelname)s:%(process)d:%(message)s', level=logging.INFO ) + #Call the main function main() diff --git a/server.py b/server.py index 0d2c0e6..619cfb2 100644 --- a/server.py +++ b/server.py @@ -5,21 +5,36 @@ #producer = KafkaProducer(bootstrap_servers='host:port',value_serializer=lambda v: json.dumps(v).encode('utf-8')) +#Initialize the Flask app app = Flask(__name__) +#VERY UNSAFE for production. +#Allows flask to accept cross-origin requests for local development CORS(app) +#Register a function @ / that could be GET or POST +# defaults to http://localhost:5000/ @app.route("/", methods=["GET","POST"]) -def hello(): +def hello(): #hello() is registered to route / + #Gets the POST body as a JSON object print request.get_json() body = request.get_json() - producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) + #Bootstraps an instance of a Kafka producer. + #Initializes the producer and identifies the docker server. + #kafka-spotify is listed in /etc/hosts with the ip of the container + #Sets the producer serializer to JSON + producer = KafkaProducer(bootstrap_servers='kafa-python:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) + #Send a message to the kafka topic 'test' + #passes the POST JSON body as the message producer.send('test', body) + #Closes the TCP stream to Kafka producer.close() + #Returns a Complete string return "Complete" +#Main process function if __name__ == "__main__": + #Bind to broadcast so you can access the server app.run('0.0.0.0') - print "After app.run" From 42e83e84014abad57ff7a40e2f6704d0819ac233 Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Thu, 3 May 2018 18:00:15 -0400 Subject: [PATCH 068/109] commented the main code --- kafkaProducerService.py | 38 +++++++++++++++++++++++++++++++++----- server.py | 21 ++++++++++++++++++--- 2 files changed, 51 insertions(+), 8 deletions(-) diff --git a/kafkaProducerService.py b/kafkaProducerService.py index 1a15b84..5d10fa7 100644 --- a/kafkaProducerService.py +++ b/kafkaProducerService.py @@ -4,40 +4,68 @@ from kafka import KafkaProducer +#Thread class that continuously sends messages to a Kafka message queue +# Input: +# threading.Thread = instance of multiproccessing thread class Producer(threading.Thread): + #default initializing function of the thread. + # Input: + # takes self to modify and initialize def __init__(self): + #initializes thread and passes self. Magic multithreading stuff threading.Thread.__init__(self) + #Gives the thread an envent called stop_event so it can be interrupted. self.stop_event = threading.Event() - + + #Function to stop the thread def stop(self): + #Calls even stop_event and sets it. + #This gives context to the thread from the outside and lets you stop it. self.stop_event.set() - + + #The main run function called when you call start. def run(self): - producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092') + #Bootstraps an instance of a Kafka producer. + #Initializes the producer and identifies the docker server. + #kafka-spotify is listed in /etc/hosts with the ip of the container + producer = KafkaProducer(bootstrap_servers='kafka-spotify:9092') + #loop until the thread is stopped by checking the stop event while not self.stop_event.is_set(): + #Send two messages of type binary to the 'test' Topic producer.send('test', b"test") producer.send('test', b"\xc2Hola, mundo!") + #Sleep for 3 seconds time.sleep(3) - + + #Close the TCP stream to Kafka producer.close() +#Main function called when the app is run def main(): + #initialize a producer object/thread kafkProducer = Producer() + #Start the thread working. kafkProducer.start() + #sleep for 17 second. If we weren't using threads this would halt the code time.sleep(17) + #Call stop to set the thread event so it knows to stop kafkProducer.stop() + #Wait until the thread terminates. Can see the docs for more + #https://docs.python.org/2/library/threading.html?highlight=thread#threading.Thread.join kafkProducer.join() - +#the logic of running as process if __name__ == "__main__": + #Set logging format and level logging.basicConfig( format='%(asctime)s.%(msecs)s:%(name)s:%(thread)d:%(levelname)s:%(process)d:%(message)s', level=logging.INFO ) + #Call the main function main() diff --git a/server.py b/server.py index 0d2c0e6..619cfb2 100644 --- a/server.py +++ b/server.py @@ -5,21 +5,36 @@ #producer = KafkaProducer(bootstrap_servers='host:port',value_serializer=lambda v: json.dumps(v).encode('utf-8')) +#Initialize the Flask app app = Flask(__name__) +#VERY UNSAFE for production. +#Allows flask to accept cross-origin requests for local development CORS(app) +#Register a function @ / that could be GET or POST +# defaults to http://localhost:5000/ @app.route("/", methods=["GET","POST"]) -def hello(): +def hello(): #hello() is registered to route / + #Gets the POST body as a JSON object print request.get_json() body = request.get_json() - producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) + #Bootstraps an instance of a Kafka producer. + #Initializes the producer and identifies the docker server. + #kafka-spotify is listed in /etc/hosts with the ip of the container + #Sets the producer serializer to JSON + producer = KafkaProducer(bootstrap_servers='kafa-python:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) + #Send a message to the kafka topic 'test' + #passes the POST JSON body as the message producer.send('test', body) + #Closes the TCP stream to Kafka producer.close() + #Returns a Complete string return "Complete" +#Main process function if __name__ == "__main__": + #Bind to broadcast so you can access the server app.run('0.0.0.0') - print "After app.run" From 4e07a61515f21ecf6352d9274d6c940b1bb635a3 Mon Sep 17 00:00:00 2001 From: rmyers19 Date: Thu, 3 May 2018 18:00:15 -0400 Subject: [PATCH 069/109] commented the main code --- kafkaProducerService.py | 38 +++++++++++++++++++++++++++++++++----- server.py | 21 ++++++++++++++++++--- 2 files changed, 51 insertions(+), 8 deletions(-) diff --git a/kafkaProducerService.py b/kafkaProducerService.py index 1a15b84..5d10fa7 100644 --- a/kafkaProducerService.py +++ b/kafkaProducerService.py @@ -4,40 +4,68 @@ from kafka import KafkaProducer +#Thread class that continuously sends messages to a Kafka message queue +# Input: +# threading.Thread = instance of multiproccessing thread class Producer(threading.Thread): + #default initializing function of the thread. + # Input: + # takes self to modify and initialize def __init__(self): + #initializes thread and passes self. Magic multithreading stuff threading.Thread.__init__(self) + #Gives the thread an envent called stop_event so it can be interrupted. self.stop_event = threading.Event() - + + #Function to stop the thread def stop(self): + #Calls even stop_event and sets it. + #This gives context to the thread from the outside and lets you stop it. self.stop_event.set() - + + #The main run function called when you call start. def run(self): - producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092') + #Bootstraps an instance of a Kafka producer. + #Initializes the producer and identifies the docker server. + #kafka-spotify is listed in /etc/hosts with the ip of the container + producer = KafkaProducer(bootstrap_servers='kafka-spotify:9092') + #loop until the thread is stopped by checking the stop event while not self.stop_event.is_set(): + #Send two messages of type binary to the 'test' Topic producer.send('test', b"test") producer.send('test', b"\xc2Hola, mundo!") + #Sleep for 3 seconds time.sleep(3) - + + #Close the TCP stream to Kafka producer.close() +#Main function called when the app is run def main(): + #initialize a producer object/thread kafkProducer = Producer() + #Start the thread working. kafkProducer.start() + #sleep for 17 second. If we weren't using threads this would halt the code time.sleep(17) + #Call stop to set the thread event so it knows to stop kafkProducer.stop() + #Wait until the thread terminates. Can see the docs for more + #https://docs.python.org/2/library/threading.html?highlight=thread#threading.Thread.join kafkProducer.join() - +#the logic of running as process if __name__ == "__main__": + #Set logging format and level logging.basicConfig( format='%(asctime)s.%(msecs)s:%(name)s:%(thread)d:%(levelname)s:%(process)d:%(message)s', level=logging.INFO ) + #Call the main function main() diff --git a/server.py b/server.py index 0d2c0e6..619cfb2 100644 --- a/server.py +++ b/server.py @@ -5,21 +5,36 @@ #producer = KafkaProducer(bootstrap_servers='host:port',value_serializer=lambda v: json.dumps(v).encode('utf-8')) +#Initialize the Flask app app = Flask(__name__) +#VERY UNSAFE for production. +#Allows flask to accept cross-origin requests for local development CORS(app) +#Register a function @ / that could be GET or POST +# defaults to http://localhost:5000/ @app.route("/", methods=["GET","POST"]) -def hello(): +def hello(): #hello() is registered to route / + #Gets the POST body as a JSON object print request.get_json() body = request.get_json() - producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) + #Bootstraps an instance of a Kafka producer. + #Initializes the producer and identifies the docker server. + #kafka-spotify is listed in /etc/hosts with the ip of the container + #Sets the producer serializer to JSON + producer = KafkaProducer(bootstrap_servers='kafa-python:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) + #Send a message to the kafka topic 'test' + #passes the POST JSON body as the message producer.send('test', body) + #Closes the TCP stream to Kafka producer.close() + #Returns a Complete string return "Complete" +#Main process function if __name__ == "__main__": + #Bind to broadcast so you can access the server app.run('0.0.0.0') - print "After app.run" From a29fda5c4f7919a0e9e4f778e7d0f7ef5f53f8f6 Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Thu, 3 May 2018 18:00:15 -0400 Subject: [PATCH 070/109] commented the main code --- kafkaProducerService.py | 38 +++++++++++++++++++++++++++++++++----- server.py | 21 ++++++++++++++++++--- 2 files changed, 51 insertions(+), 8 deletions(-) diff --git a/kafkaProducerService.py b/kafkaProducerService.py index 1a15b84..5d10fa7 100644 --- a/kafkaProducerService.py +++ b/kafkaProducerService.py @@ -4,40 +4,68 @@ from kafka import KafkaProducer +#Thread class that continuously sends messages to a Kafka message queue +# Input: +# threading.Thread = instance of multiproccessing thread class Producer(threading.Thread): + #default initializing function of the thread. + # Input: + # takes self to modify and initialize def __init__(self): + #initializes thread and passes self. Magic multithreading stuff threading.Thread.__init__(self) + #Gives the thread an envent called stop_event so it can be interrupted. self.stop_event = threading.Event() - + + #Function to stop the thread def stop(self): + #Calls even stop_event and sets it. + #This gives context to the thread from the outside and lets you stop it. self.stop_event.set() - + + #The main run function called when you call start. def run(self): - producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092') + #Bootstraps an instance of a Kafka producer. + #Initializes the producer and identifies the docker server. + #kafka-spotify is listed in /etc/hosts with the ip of the container + producer = KafkaProducer(bootstrap_servers='kafka-spotify:9092') + #loop until the thread is stopped by checking the stop event while not self.stop_event.is_set(): + #Send two messages of type binary to the 'test' Topic producer.send('test', b"test") producer.send('test', b"\xc2Hola, mundo!") + #Sleep for 3 seconds time.sleep(3) - + + #Close the TCP stream to Kafka producer.close() +#Main function called when the app is run def main(): + #initialize a producer object/thread kafkProducer = Producer() + #Start the thread working. kafkProducer.start() + #sleep for 17 second. If we weren't using threads this would halt the code time.sleep(17) + #Call stop to set the thread event so it knows to stop kafkProducer.stop() + #Wait until the thread terminates. Can see the docs for more + #https://docs.python.org/2/library/threading.html?highlight=thread#threading.Thread.join kafkProducer.join() - +#the logic of running as process if __name__ == "__main__": + #Set logging format and level logging.basicConfig( format='%(asctime)s.%(msecs)s:%(name)s:%(thread)d:%(levelname)s:%(process)d:%(message)s', level=logging.INFO ) + #Call the main function main() diff --git a/server.py b/server.py index 0d2c0e6..619cfb2 100644 --- a/server.py +++ b/server.py @@ -5,21 +5,36 @@ #producer = KafkaProducer(bootstrap_servers='host:port',value_serializer=lambda v: json.dumps(v).encode('utf-8')) +#Initialize the Flask app app = Flask(__name__) +#VERY UNSAFE for production. +#Allows flask to accept cross-origin requests for local development CORS(app) +#Register a function @ / that could be GET or POST +# defaults to http://localhost:5000/ @app.route("/", methods=["GET","POST"]) -def hello(): +def hello(): #hello() is registered to route / + #Gets the POST body as a JSON object print request.get_json() body = request.get_json() - producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) + #Bootstraps an instance of a Kafka producer. + #Initializes the producer and identifies the docker server. + #kafka-spotify is listed in /etc/hosts with the ip of the container + #Sets the producer serializer to JSON + producer = KafkaProducer(bootstrap_servers='kafa-python:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) + #Send a message to the kafka topic 'test' + #passes the POST JSON body as the message producer.send('test', body) + #Closes the TCP stream to Kafka producer.close() + #Returns a Complete string return "Complete" +#Main process function if __name__ == "__main__": + #Bind to broadcast so you can access the server app.run('0.0.0.0') - print "After app.run" From 0940ac959f1075b033749131d8760a262019f15d Mon Sep 17 00:00:00 2001 From: myersr Date: Thu, 3 May 2018 18:00:15 -0400 Subject: [PATCH 071/109] commented the main code --- kafkaProducerService.py | 38 +++++++++++++++++++++++++++++++++----- server.py | 21 ++++++++++++++++++--- 2 files changed, 51 insertions(+), 8 deletions(-) diff --git a/kafkaProducerService.py b/kafkaProducerService.py index 1a15b84..5d10fa7 100644 --- a/kafkaProducerService.py +++ b/kafkaProducerService.py @@ -4,40 +4,68 @@ from kafka import KafkaProducer +#Thread class that continuously sends messages to a Kafka message queue +# Input: +# threading.Thread = instance of multiproccessing thread class Producer(threading.Thread): + #default initializing function of the thread. + # Input: + # takes self to modify and initialize def __init__(self): + #initializes thread and passes self. Magic multithreading stuff threading.Thread.__init__(self) + #Gives the thread an envent called stop_event so it can be interrupted. self.stop_event = threading.Event() - + + #Function to stop the thread def stop(self): + #Calls even stop_event and sets it. + #This gives context to the thread from the outside and lets you stop it. self.stop_event.set() - + + #The main run function called when you call start. def run(self): - producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092') + #Bootstraps an instance of a Kafka producer. + #Initializes the producer and identifies the docker server. + #kafka-spotify is listed in /etc/hosts with the ip of the container + producer = KafkaProducer(bootstrap_servers='kafka-spotify:9092') + #loop until the thread is stopped by checking the stop event while not self.stop_event.is_set(): + #Send two messages of type binary to the 'test' Topic producer.send('test', b"test") producer.send('test', b"\xc2Hola, mundo!") + #Sleep for 3 seconds time.sleep(3) - + + #Close the TCP stream to Kafka producer.close() +#Main function called when the app is run def main(): + #initialize a producer object/thread kafkProducer = Producer() + #Start the thread working. kafkProducer.start() + #sleep for 17 second. If we weren't using threads this would halt the code time.sleep(17) + #Call stop to set the thread event so it knows to stop kafkProducer.stop() + #Wait until the thread terminates. Can see the docs for more + #https://docs.python.org/2/library/threading.html?highlight=thread#threading.Thread.join kafkProducer.join() - +#the logic of running as process if __name__ == "__main__": + #Set logging format and level logging.basicConfig( format='%(asctime)s.%(msecs)s:%(name)s:%(thread)d:%(levelname)s:%(process)d:%(message)s', level=logging.INFO ) + #Call the main function main() diff --git a/server.py b/server.py index 0d2c0e6..619cfb2 100644 --- a/server.py +++ b/server.py @@ -5,21 +5,36 @@ #producer = KafkaProducer(bootstrap_servers='host:port',value_serializer=lambda v: json.dumps(v).encode('utf-8')) +#Initialize the Flask app app = Flask(__name__) +#VERY UNSAFE for production. +#Allows flask to accept cross-origin requests for local development CORS(app) +#Register a function @ / that could be GET or POST +# defaults to http://localhost:5000/ @app.route("/", methods=["GET","POST"]) -def hello(): +def hello(): #hello() is registered to route / + #Gets the POST body as a JSON object print request.get_json() body = request.get_json() - producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) + #Bootstraps an instance of a Kafka producer. + #Initializes the producer and identifies the docker server. + #kafka-spotify is listed in /etc/hosts with the ip of the container + #Sets the producer serializer to JSON + producer = KafkaProducer(bootstrap_servers='kafa-python:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) + #Send a message to the kafka topic 'test' + #passes the POST JSON body as the message producer.send('test', body) + #Closes the TCP stream to Kafka producer.close() + #Returns a Complete string return "Complete" +#Main process function if __name__ == "__main__": + #Bind to broadcast so you can access the server app.run('0.0.0.0') - print "After app.run" From cc6171420b6d0413e126b2174297638dce3015a7 Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Thu, 3 May 2018 18:00:15 -0400 Subject: [PATCH 072/109] commented the main code --- kafkaProducerService.py | 38 +++++++++++++++++++++++++++++++++----- server.py | 21 ++++++++++++++++++--- 2 files changed, 51 insertions(+), 8 deletions(-) diff --git a/kafkaProducerService.py b/kafkaProducerService.py index 1a15b84..5d10fa7 100644 --- a/kafkaProducerService.py +++ b/kafkaProducerService.py @@ -4,40 +4,68 @@ from kafka import KafkaProducer +#Thread class that continuously sends messages to a Kafka message queue +# Input: +# threading.Thread = instance of multiproccessing thread class Producer(threading.Thread): + #default initializing function of the thread. + # Input: + # takes self to modify and initialize def __init__(self): + #initializes thread and passes self. Magic multithreading stuff threading.Thread.__init__(self) + #Gives the thread an envent called stop_event so it can be interrupted. self.stop_event = threading.Event() - + + #Function to stop the thread def stop(self): + #Calls even stop_event and sets it. + #This gives context to the thread from the outside and lets you stop it. self.stop_event.set() - + + #The main run function called when you call start. def run(self): - producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092') + #Bootstraps an instance of a Kafka producer. + #Initializes the producer and identifies the docker server. + #kafka-spotify is listed in /etc/hosts with the ip of the container + producer = KafkaProducer(bootstrap_servers='kafka-spotify:9092') + #loop until the thread is stopped by checking the stop event while not self.stop_event.is_set(): + #Send two messages of type binary to the 'test' Topic producer.send('test', b"test") producer.send('test', b"\xc2Hola, mundo!") + #Sleep for 3 seconds time.sleep(3) - + + #Close the TCP stream to Kafka producer.close() +#Main function called when the app is run def main(): + #initialize a producer object/thread kafkProducer = Producer() + #Start the thread working. kafkProducer.start() + #sleep for 17 second. If we weren't using threads this would halt the code time.sleep(17) + #Call stop to set the thread event so it knows to stop kafkProducer.stop() + #Wait until the thread terminates. Can see the docs for more + #https://docs.python.org/2/library/threading.html?highlight=thread#threading.Thread.join kafkProducer.join() - +#the logic of running as process if __name__ == "__main__": + #Set logging format and level logging.basicConfig( format='%(asctime)s.%(msecs)s:%(name)s:%(thread)d:%(levelname)s:%(process)d:%(message)s', level=logging.INFO ) + #Call the main function main() diff --git a/server.py b/server.py index 0d2c0e6..619cfb2 100644 --- a/server.py +++ b/server.py @@ -5,21 +5,36 @@ #producer = KafkaProducer(bootstrap_servers='host:port',value_serializer=lambda v: json.dumps(v).encode('utf-8')) +#Initialize the Flask app app = Flask(__name__) +#VERY UNSAFE for production. +#Allows flask to accept cross-origin requests for local development CORS(app) +#Register a function @ / that could be GET or POST +# defaults to http://localhost:5000/ @app.route("/", methods=["GET","POST"]) -def hello(): +def hello(): #hello() is registered to route / + #Gets the POST body as a JSON object print request.get_json() body = request.get_json() - producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) + #Bootstraps an instance of a Kafka producer. + #Initializes the producer and identifies the docker server. + #kafka-spotify is listed in /etc/hosts with the ip of the container + #Sets the producer serializer to JSON + producer = KafkaProducer(bootstrap_servers='kafa-python:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) + #Send a message to the kafka topic 'test' + #passes the POST JSON body as the message producer.send('test', body) + #Closes the TCP stream to Kafka producer.close() + #Returns a Complete string return "Complete" +#Main process function if __name__ == "__main__": + #Bind to broadcast so you can access the server app.run('0.0.0.0') - print "After app.run" From bd7479c4b48b4eb795a90a796965cf3917473b11 Mon Sep 17 00:00:00 2001 From: myersr Date: Thu, 3 May 2018 18:00:15 -0400 Subject: [PATCH 073/109] commented the main code --- kafkaProducerService.py | 38 +++++++++++++++++++++++++++++++++----- server.py | 21 ++++++++++++++++++--- 2 files changed, 51 insertions(+), 8 deletions(-) diff --git a/kafkaProducerService.py b/kafkaProducerService.py index 1a15b84..5d10fa7 100644 --- a/kafkaProducerService.py +++ b/kafkaProducerService.py @@ -4,40 +4,68 @@ from kafka import KafkaProducer +#Thread class that continuously sends messages to a Kafka message queue +# Input: +# threading.Thread = instance of multiproccessing thread class Producer(threading.Thread): + #default initializing function of the thread. + # Input: + # takes self to modify and initialize def __init__(self): + #initializes thread and passes self. Magic multithreading stuff threading.Thread.__init__(self) + #Gives the thread an envent called stop_event so it can be interrupted. self.stop_event = threading.Event() - + + #Function to stop the thread def stop(self): + #Calls even stop_event and sets it. + #This gives context to the thread from the outside and lets you stop it. self.stop_event.set() - + + #The main run function called when you call start. def run(self): - producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092') + #Bootstraps an instance of a Kafka producer. + #Initializes the producer and identifies the docker server. + #kafka-spotify is listed in /etc/hosts with the ip of the container + producer = KafkaProducer(bootstrap_servers='kafka-spotify:9092') + #loop until the thread is stopped by checking the stop event while not self.stop_event.is_set(): + #Send two messages of type binary to the 'test' Topic producer.send('test', b"test") producer.send('test', b"\xc2Hola, mundo!") + #Sleep for 3 seconds time.sleep(3) - + + #Close the TCP stream to Kafka producer.close() +#Main function called when the app is run def main(): + #initialize a producer object/thread kafkProducer = Producer() + #Start the thread working. kafkProducer.start() + #sleep for 17 second. If we weren't using threads this would halt the code time.sleep(17) + #Call stop to set the thread event so it knows to stop kafkProducer.stop() + #Wait until the thread terminates. Can see the docs for more + #https://docs.python.org/2/library/threading.html?highlight=thread#threading.Thread.join kafkProducer.join() - +#the logic of running as process if __name__ == "__main__": + #Set logging format and level logging.basicConfig( format='%(asctime)s.%(msecs)s:%(name)s:%(thread)d:%(levelname)s:%(process)d:%(message)s', level=logging.INFO ) + #Call the main function main() diff --git a/server.py b/server.py index 0d2c0e6..619cfb2 100644 --- a/server.py +++ b/server.py @@ -5,21 +5,36 @@ #producer = KafkaProducer(bootstrap_servers='host:port',value_serializer=lambda v: json.dumps(v).encode('utf-8')) +#Initialize the Flask app app = Flask(__name__) +#VERY UNSAFE for production. +#Allows flask to accept cross-origin requests for local development CORS(app) +#Register a function @ / that could be GET or POST +# defaults to http://localhost:5000/ @app.route("/", methods=["GET","POST"]) -def hello(): +def hello(): #hello() is registered to route / + #Gets the POST body as a JSON object print request.get_json() body = request.get_json() - producer = KafkaProducer(bootstrap_servers='172.20.0.2:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) + #Bootstraps an instance of a Kafka producer. + #Initializes the producer and identifies the docker server. + #kafka-spotify is listed in /etc/hosts with the ip of the container + #Sets the producer serializer to JSON + producer = KafkaProducer(bootstrap_servers='kafa-python:9092',value_serializer=lambda v: json.dumps(v).encode('utf-8')) + #Send a message to the kafka topic 'test' + #passes the POST JSON body as the message producer.send('test', body) + #Closes the TCP stream to Kafka producer.close() + #Returns a Complete string return "Complete" +#Main process function if __name__ == "__main__": + #Bind to broadcast so you can access the server app.run('0.0.0.0') - print "After app.run" From 291f528bfc5f389e54433aada8e4edcf9b9bd667 Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Thu, 3 May 2018 18:33:50 -0400 Subject: [PATCH 074/109] fixing about me --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 569eb77..c009d52 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ python kafkaProducerService.py ## Authors -* **Roy Myers** - *Initial work* - [rmyers19](https://github.optum.com/rmyers19) +* **Roy Myers** - *Initial work* - [myersr](https://github.com/myersr) ## License From 586b1a456971a26230ec820cb10876ae2258d07f Mon Sep 17 00:00:00 2001 From: rmyers19 Date: Thu, 3 May 2018 18:33:50 -0400 Subject: [PATCH 075/109] fixing about me --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 569eb77..c009d52 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ python kafkaProducerService.py ## Authors -* **Roy Myers** - *Initial work* - [rmyers19](https://github.optum.com/rmyers19) +* **Roy Myers** - *Initial work* - [myersr](https://github.com/myersr) ## License From 18cf1bd795d99ef9b2e0f8162c63f636ccfb7c08 Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Thu, 3 May 2018 18:33:50 -0400 Subject: [PATCH 076/109] fixing about me --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 569eb77..c009d52 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ python kafkaProducerService.py ## Authors -* **Roy Myers** - *Initial work* - [rmyers19](https://github.optum.com/rmyers19) +* **Roy Myers** - *Initial work* - [myersr](https://github.com/myersr) ## License From 5dcf2ae3309ade10287c2b60dd007ddb95aaf466 Mon Sep 17 00:00:00 2001 From: rmyers19 Date: Thu, 3 May 2018 18:33:50 -0400 Subject: [PATCH 077/109] fixing about me --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 569eb77..c009d52 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ python kafkaProducerService.py ## Authors -* **Roy Myers** - *Initial work* - [rmyers19](https://github.optum.com/rmyers19) +* **Roy Myers** - *Initial work* - [myersr](https://github.com/myersr) ## License From 70606f7584f5ef20c3ab241963ed9dcf13f5aebe Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Thu, 3 May 2018 18:33:50 -0400 Subject: [PATCH 078/109] fixing about me --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 569eb77..c009d52 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ python kafkaProducerService.py ## Authors -* **Roy Myers** - *Initial work* - [rmyers19](https://github.optum.com/rmyers19) +* **Roy Myers** - *Initial work* - [myersr](https://github.com/myersr) ## License From 18e317b3e9b1dd3823b56f9c1fa5ea2df9cb782d Mon Sep 17 00:00:00 2001 From: myersr Date: Thu, 3 May 2018 18:33:50 -0400 Subject: [PATCH 079/109] fixing about me --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 569eb77..c009d52 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ python kafkaProducerService.py ## Authors -* **Roy Myers** - *Initial work* - [rmyers19](https://github.optum.com/rmyers19) +* **Roy Myers** - *Initial work* - [myersr](https://github.com/myersr) ## License From bd749a9ccf5a720d5e8f312ccb510f474bfd0494 Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Thu, 3 May 2018 18:33:50 -0400 Subject: [PATCH 080/109] fixing about me --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 569eb77..c009d52 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ python kafkaProducerService.py ## Authors -* **Roy Myers** - *Initial work* - [rmyers19](https://github.optum.com/rmyers19) +* **Roy Myers** - *Initial work* - [myersr](https://github.com/myersr) ## License From 601f0a48dca529353ae62bc1b5e3dc3d6c1151b1 Mon Sep 17 00:00:00 2001 From: myersr Date: Thu, 3 May 2018 18:33:50 -0400 Subject: [PATCH 081/109] fixing about me --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 569eb77..c009d52 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ python kafkaProducerService.py ## Authors -* **Roy Myers** - *Initial work* - [rmyers19](https://github.optum.com/rmyers19) +* **Roy Myers** - *Initial work* - [myersr](https://github.com/myersr) ## License From b9cf7270cda0d999a33dcd546a24a79ec1e03e3d Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Fri, 4 May 2018 16:34:48 -0400 Subject: [PATCH 082/109] added consumer script --- kafkaConsumerService.py | 96 +++++++++++++++++++++++++++++++++++++++++ kafkaProducerService.py | 2 +- 2 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 kafkaConsumerService.py diff --git a/kafkaConsumerService.py b/kafkaConsumerService.py new file mode 100644 index 0000000..a0580eb --- /dev/null +++ b/kafkaConsumerService.py @@ -0,0 +1,96 @@ +import threading, logging, time +import multiprocessing +import os + +from kafka import KafkaConsumer + + +#Thread class that continuously consumes messages from a Kafka message queue/topic +# Input: +# threading.Thread = instance of multiproccessing thread +class Consumer(multiprocessing.Process): + #default initializing function of the thread. + # Input: + # takes self to modify and initialize + def __init__(self): + #initializes thread and passes self. Magic multithreading stuff + multiprocessing.Process.__init__(self) + #Gives the thread an envent called stop_event so it can be interrupted. + self.stop_event = multiprocessing.Event() + + #Function to stop the process + def stop(self): + #Calls even stop_event and sets it. + #This gives context to the thread from the outside and lets you stop it. + self.stop_event.set() + + #The main run function called when you call start. + def run(self): + if hasattr(os, 'getppid'): # only available on Unix + print 'parent process:', os.getppid() + procID = os.getppid() + #Bootstraps an instance of a Kafka producer. + #Initializes the producer and identifies the docker server. + #kafka-spotify is listed in /etc/hosts with the ip of the container + #Input: + # topic to subscribe to: 'test' + # Id to identify the consumer should be unique to the connection + # Servers kafka is advertising as + # Which message rule to subscribe to. 'earliest' will grab the earliest unprocessed message + # Timeout limit + consumer = KafkaConsumer('test', + client_id='python-consumer-%s' % (procID), + bootstrap_servers=['kafka-spotify:9092'], + auto_offset_reset='latest', + consumer_timeout_ms=1000) + + #Alternative way to subscribe to a topic + #consumer.subscribe(['test']) + + #loop until the thread is stopped by checking the stop event + while not self.stop_event.is_set(): + #Loop through ConsumerRecord objects in the consumer object + for message in consumer: + #print the messages to the screen with a note of the thread/client ID + #print("python-consumer-%s processed message: %s" % (procID, message)) + #print the messages to the screen with a note of the thread/client ID, Current Topic, message number, # The value of the message decoded as it is sent as bytecode + print ("python-consumer-%s processed message: %s:%d: value=%s" % (procID, message.topic, + message.offset, message.value.decode('utf-8'))) + #break out of the for loop if the thread was notified of closure + if self.stop_event.is_set(): + break + + #Close the TCP connection to kafka + consumer.close() + + +#Main function called when the app is run +def main(): + #initialize a Consumer object/thread + kafkConsumer = Consumer() + + #Start the thread working. + kafkConsumer.start() + + #sleep for 17 second. If we weren't using threads this would halt the code + time.sleep(20) + + #Call stop to set the thread event so it knows to stop + print("Stopping kafkConsumer") + kafkConsumer.stop() + + #Wait until the thread terminates. Can see the docs for more + #https://docs.python.org/2/library/threading.html?highlight=thread#threading.Thread.join + print("Waiting for execution to halt") + kafkConsumer.join() + +#the logic to run as process +if __name__ == "__main__": + #Set logging format and level + #logging.basicConfig( + # format='%(asctime)s.%(msecs)s:%(name)s:%(thread)d:%(levelname)s:%(process)d:%(message)s', + # level=logging.INFO) + + #Call the main function + main() + diff --git a/kafkaProducerService.py b/kafkaProducerService.py index 5d10fa7..4c4b7c6 100644 --- a/kafkaProducerService.py +++ b/kafkaProducerService.py @@ -34,7 +34,7 @@ def run(self): while not self.stop_event.is_set(): #Send two messages of type binary to the 'test' Topic producer.send('test', b"test") - producer.send('test', b"\xc2Hola, mundo!") + producer.send('test', b"Hola, mundo!") #Sleep for 3 seconds time.sleep(3) From 557a3669bd93e2a8928dd69dcc2aae7416855f00 Mon Sep 17 00:00:00 2001 From: rmyers19 Date: Fri, 4 May 2018 16:34:48 -0400 Subject: [PATCH 083/109] added consumer script --- kafkaConsumerService.py | 96 +++++++++++++++++++++++++++++++++++++++++ kafkaProducerService.py | 2 +- 2 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 kafkaConsumerService.py diff --git a/kafkaConsumerService.py b/kafkaConsumerService.py new file mode 100644 index 0000000..a0580eb --- /dev/null +++ b/kafkaConsumerService.py @@ -0,0 +1,96 @@ +import threading, logging, time +import multiprocessing +import os + +from kafka import KafkaConsumer + + +#Thread class that continuously consumes messages from a Kafka message queue/topic +# Input: +# threading.Thread = instance of multiproccessing thread +class Consumer(multiprocessing.Process): + #default initializing function of the thread. + # Input: + # takes self to modify and initialize + def __init__(self): + #initializes thread and passes self. Magic multithreading stuff + multiprocessing.Process.__init__(self) + #Gives the thread an envent called stop_event so it can be interrupted. + self.stop_event = multiprocessing.Event() + + #Function to stop the process + def stop(self): + #Calls even stop_event and sets it. + #This gives context to the thread from the outside and lets you stop it. + self.stop_event.set() + + #The main run function called when you call start. + def run(self): + if hasattr(os, 'getppid'): # only available on Unix + print 'parent process:', os.getppid() + procID = os.getppid() + #Bootstraps an instance of a Kafka producer. + #Initializes the producer and identifies the docker server. + #kafka-spotify is listed in /etc/hosts with the ip of the container + #Input: + # topic to subscribe to: 'test' + # Id to identify the consumer should be unique to the connection + # Servers kafka is advertising as + # Which message rule to subscribe to. 'earliest' will grab the earliest unprocessed message + # Timeout limit + consumer = KafkaConsumer('test', + client_id='python-consumer-%s' % (procID), + bootstrap_servers=['kafka-spotify:9092'], + auto_offset_reset='latest', + consumer_timeout_ms=1000) + + #Alternative way to subscribe to a topic + #consumer.subscribe(['test']) + + #loop until the thread is stopped by checking the stop event + while not self.stop_event.is_set(): + #Loop through ConsumerRecord objects in the consumer object + for message in consumer: + #print the messages to the screen with a note of the thread/client ID + #print("python-consumer-%s processed message: %s" % (procID, message)) + #print the messages to the screen with a note of the thread/client ID, Current Topic, message number, # The value of the message decoded as it is sent as bytecode + print ("python-consumer-%s processed message: %s:%d: value=%s" % (procID, message.topic, + message.offset, message.value.decode('utf-8'))) + #break out of the for loop if the thread was notified of closure + if self.stop_event.is_set(): + break + + #Close the TCP connection to kafka + consumer.close() + + +#Main function called when the app is run +def main(): + #initialize a Consumer object/thread + kafkConsumer = Consumer() + + #Start the thread working. + kafkConsumer.start() + + #sleep for 17 second. If we weren't using threads this would halt the code + time.sleep(20) + + #Call stop to set the thread event so it knows to stop + print("Stopping kafkConsumer") + kafkConsumer.stop() + + #Wait until the thread terminates. Can see the docs for more + #https://docs.python.org/2/library/threading.html?highlight=thread#threading.Thread.join + print("Waiting for execution to halt") + kafkConsumer.join() + +#the logic to run as process +if __name__ == "__main__": + #Set logging format and level + #logging.basicConfig( + # format='%(asctime)s.%(msecs)s:%(name)s:%(thread)d:%(levelname)s:%(process)d:%(message)s', + # level=logging.INFO) + + #Call the main function + main() + diff --git a/kafkaProducerService.py b/kafkaProducerService.py index 5d10fa7..4c4b7c6 100644 --- a/kafkaProducerService.py +++ b/kafkaProducerService.py @@ -34,7 +34,7 @@ def run(self): while not self.stop_event.is_set(): #Send two messages of type binary to the 'test' Topic producer.send('test', b"test") - producer.send('test', b"\xc2Hola, mundo!") + producer.send('test', b"Hola, mundo!") #Sleep for 3 seconds time.sleep(3) From d3a28aa46d543d93ca8cbec319d211b5ba29a155 Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Fri, 4 May 2018 16:34:48 -0400 Subject: [PATCH 084/109] added consumer script --- kafkaConsumerService.py | 96 +++++++++++++++++++++++++++++++++++++++++ kafkaProducerService.py | 2 +- 2 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 kafkaConsumerService.py diff --git a/kafkaConsumerService.py b/kafkaConsumerService.py new file mode 100644 index 0000000..a0580eb --- /dev/null +++ b/kafkaConsumerService.py @@ -0,0 +1,96 @@ +import threading, logging, time +import multiprocessing +import os + +from kafka import KafkaConsumer + + +#Thread class that continuously consumes messages from a Kafka message queue/topic +# Input: +# threading.Thread = instance of multiproccessing thread +class Consumer(multiprocessing.Process): + #default initializing function of the thread. + # Input: + # takes self to modify and initialize + def __init__(self): + #initializes thread and passes self. Magic multithreading stuff + multiprocessing.Process.__init__(self) + #Gives the thread an envent called stop_event so it can be interrupted. + self.stop_event = multiprocessing.Event() + + #Function to stop the process + def stop(self): + #Calls even stop_event and sets it. + #This gives context to the thread from the outside and lets you stop it. + self.stop_event.set() + + #The main run function called when you call start. + def run(self): + if hasattr(os, 'getppid'): # only available on Unix + print 'parent process:', os.getppid() + procID = os.getppid() + #Bootstraps an instance of a Kafka producer. + #Initializes the producer and identifies the docker server. + #kafka-spotify is listed in /etc/hosts with the ip of the container + #Input: + # topic to subscribe to: 'test' + # Id to identify the consumer should be unique to the connection + # Servers kafka is advertising as + # Which message rule to subscribe to. 'earliest' will grab the earliest unprocessed message + # Timeout limit + consumer = KafkaConsumer('test', + client_id='python-consumer-%s' % (procID), + bootstrap_servers=['kafka-spotify:9092'], + auto_offset_reset='latest', + consumer_timeout_ms=1000) + + #Alternative way to subscribe to a topic + #consumer.subscribe(['test']) + + #loop until the thread is stopped by checking the stop event + while not self.stop_event.is_set(): + #Loop through ConsumerRecord objects in the consumer object + for message in consumer: + #print the messages to the screen with a note of the thread/client ID + #print("python-consumer-%s processed message: %s" % (procID, message)) + #print the messages to the screen with a note of the thread/client ID, Current Topic, message number, # The value of the message decoded as it is sent as bytecode + print ("python-consumer-%s processed message: %s:%d: value=%s" % (procID, message.topic, + message.offset, message.value.decode('utf-8'))) + #break out of the for loop if the thread was notified of closure + if self.stop_event.is_set(): + break + + #Close the TCP connection to kafka + consumer.close() + + +#Main function called when the app is run +def main(): + #initialize a Consumer object/thread + kafkConsumer = Consumer() + + #Start the thread working. + kafkConsumer.start() + + #sleep for 17 second. If we weren't using threads this would halt the code + time.sleep(20) + + #Call stop to set the thread event so it knows to stop + print("Stopping kafkConsumer") + kafkConsumer.stop() + + #Wait until the thread terminates. Can see the docs for more + #https://docs.python.org/2/library/threading.html?highlight=thread#threading.Thread.join + print("Waiting for execution to halt") + kafkConsumer.join() + +#the logic to run as process +if __name__ == "__main__": + #Set logging format and level + #logging.basicConfig( + # format='%(asctime)s.%(msecs)s:%(name)s:%(thread)d:%(levelname)s:%(process)d:%(message)s', + # level=logging.INFO) + + #Call the main function + main() + diff --git a/kafkaProducerService.py b/kafkaProducerService.py index 5d10fa7..4c4b7c6 100644 --- a/kafkaProducerService.py +++ b/kafkaProducerService.py @@ -34,7 +34,7 @@ def run(self): while not self.stop_event.is_set(): #Send two messages of type binary to the 'test' Topic producer.send('test', b"test") - producer.send('test', b"\xc2Hola, mundo!") + producer.send('test', b"Hola, mundo!") #Sleep for 3 seconds time.sleep(3) From 12e86c968f92920556dfb925a4ae33bcb4b10b65 Mon Sep 17 00:00:00 2001 From: rmyers19 Date: Fri, 4 May 2018 16:34:48 -0400 Subject: [PATCH 085/109] added consumer script --- kafkaConsumerService.py | 96 +++++++++++++++++++++++++++++++++++++++++ kafkaProducerService.py | 2 +- 2 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 kafkaConsumerService.py diff --git a/kafkaConsumerService.py b/kafkaConsumerService.py new file mode 100644 index 0000000..a0580eb --- /dev/null +++ b/kafkaConsumerService.py @@ -0,0 +1,96 @@ +import threading, logging, time +import multiprocessing +import os + +from kafka import KafkaConsumer + + +#Thread class that continuously consumes messages from a Kafka message queue/topic +# Input: +# threading.Thread = instance of multiproccessing thread +class Consumer(multiprocessing.Process): + #default initializing function of the thread. + # Input: + # takes self to modify and initialize + def __init__(self): + #initializes thread and passes self. Magic multithreading stuff + multiprocessing.Process.__init__(self) + #Gives the thread an envent called stop_event so it can be interrupted. + self.stop_event = multiprocessing.Event() + + #Function to stop the process + def stop(self): + #Calls even stop_event and sets it. + #This gives context to the thread from the outside and lets you stop it. + self.stop_event.set() + + #The main run function called when you call start. + def run(self): + if hasattr(os, 'getppid'): # only available on Unix + print 'parent process:', os.getppid() + procID = os.getppid() + #Bootstraps an instance of a Kafka producer. + #Initializes the producer and identifies the docker server. + #kafka-spotify is listed in /etc/hosts with the ip of the container + #Input: + # topic to subscribe to: 'test' + # Id to identify the consumer should be unique to the connection + # Servers kafka is advertising as + # Which message rule to subscribe to. 'earliest' will grab the earliest unprocessed message + # Timeout limit + consumer = KafkaConsumer('test', + client_id='python-consumer-%s' % (procID), + bootstrap_servers=['kafka-spotify:9092'], + auto_offset_reset='latest', + consumer_timeout_ms=1000) + + #Alternative way to subscribe to a topic + #consumer.subscribe(['test']) + + #loop until the thread is stopped by checking the stop event + while not self.stop_event.is_set(): + #Loop through ConsumerRecord objects in the consumer object + for message in consumer: + #print the messages to the screen with a note of the thread/client ID + #print("python-consumer-%s processed message: %s" % (procID, message)) + #print the messages to the screen with a note of the thread/client ID, Current Topic, message number, # The value of the message decoded as it is sent as bytecode + print ("python-consumer-%s processed message: %s:%d: value=%s" % (procID, message.topic, + message.offset, message.value.decode('utf-8'))) + #break out of the for loop if the thread was notified of closure + if self.stop_event.is_set(): + break + + #Close the TCP connection to kafka + consumer.close() + + +#Main function called when the app is run +def main(): + #initialize a Consumer object/thread + kafkConsumer = Consumer() + + #Start the thread working. + kafkConsumer.start() + + #sleep for 17 second. If we weren't using threads this would halt the code + time.sleep(20) + + #Call stop to set the thread event so it knows to stop + print("Stopping kafkConsumer") + kafkConsumer.stop() + + #Wait until the thread terminates. Can see the docs for more + #https://docs.python.org/2/library/threading.html?highlight=thread#threading.Thread.join + print("Waiting for execution to halt") + kafkConsumer.join() + +#the logic to run as process +if __name__ == "__main__": + #Set logging format and level + #logging.basicConfig( + # format='%(asctime)s.%(msecs)s:%(name)s:%(thread)d:%(levelname)s:%(process)d:%(message)s', + # level=logging.INFO) + + #Call the main function + main() + diff --git a/kafkaProducerService.py b/kafkaProducerService.py index 5d10fa7..4c4b7c6 100644 --- a/kafkaProducerService.py +++ b/kafkaProducerService.py @@ -34,7 +34,7 @@ def run(self): while not self.stop_event.is_set(): #Send two messages of type binary to the 'test' Topic producer.send('test', b"test") - producer.send('test', b"\xc2Hola, mundo!") + producer.send('test', b"Hola, mundo!") #Sleep for 3 seconds time.sleep(3) From ff5e441abbc31bd747f20a759d3a8163abb6c8f1 Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Fri, 4 May 2018 16:34:48 -0400 Subject: [PATCH 086/109] added consumer script --- kafkaConsumerService.py | 96 +++++++++++++++++++++++++++++++++++++++++ kafkaProducerService.py | 2 +- 2 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 kafkaConsumerService.py diff --git a/kafkaConsumerService.py b/kafkaConsumerService.py new file mode 100644 index 0000000..a0580eb --- /dev/null +++ b/kafkaConsumerService.py @@ -0,0 +1,96 @@ +import threading, logging, time +import multiprocessing +import os + +from kafka import KafkaConsumer + + +#Thread class that continuously consumes messages from a Kafka message queue/topic +# Input: +# threading.Thread = instance of multiproccessing thread +class Consumer(multiprocessing.Process): + #default initializing function of the thread. + # Input: + # takes self to modify and initialize + def __init__(self): + #initializes thread and passes self. Magic multithreading stuff + multiprocessing.Process.__init__(self) + #Gives the thread an envent called stop_event so it can be interrupted. + self.stop_event = multiprocessing.Event() + + #Function to stop the process + def stop(self): + #Calls even stop_event and sets it. + #This gives context to the thread from the outside and lets you stop it. + self.stop_event.set() + + #The main run function called when you call start. + def run(self): + if hasattr(os, 'getppid'): # only available on Unix + print 'parent process:', os.getppid() + procID = os.getppid() + #Bootstraps an instance of a Kafka producer. + #Initializes the producer and identifies the docker server. + #kafka-spotify is listed in /etc/hosts with the ip of the container + #Input: + # topic to subscribe to: 'test' + # Id to identify the consumer should be unique to the connection + # Servers kafka is advertising as + # Which message rule to subscribe to. 'earliest' will grab the earliest unprocessed message + # Timeout limit + consumer = KafkaConsumer('test', + client_id='python-consumer-%s' % (procID), + bootstrap_servers=['kafka-spotify:9092'], + auto_offset_reset='latest', + consumer_timeout_ms=1000) + + #Alternative way to subscribe to a topic + #consumer.subscribe(['test']) + + #loop until the thread is stopped by checking the stop event + while not self.stop_event.is_set(): + #Loop through ConsumerRecord objects in the consumer object + for message in consumer: + #print the messages to the screen with a note of the thread/client ID + #print("python-consumer-%s processed message: %s" % (procID, message)) + #print the messages to the screen with a note of the thread/client ID, Current Topic, message number, # The value of the message decoded as it is sent as bytecode + print ("python-consumer-%s processed message: %s:%d: value=%s" % (procID, message.topic, + message.offset, message.value.decode('utf-8'))) + #break out of the for loop if the thread was notified of closure + if self.stop_event.is_set(): + break + + #Close the TCP connection to kafka + consumer.close() + + +#Main function called when the app is run +def main(): + #initialize a Consumer object/thread + kafkConsumer = Consumer() + + #Start the thread working. + kafkConsumer.start() + + #sleep for 17 second. If we weren't using threads this would halt the code + time.sleep(20) + + #Call stop to set the thread event so it knows to stop + print("Stopping kafkConsumer") + kafkConsumer.stop() + + #Wait until the thread terminates. Can see the docs for more + #https://docs.python.org/2/library/threading.html?highlight=thread#threading.Thread.join + print("Waiting for execution to halt") + kafkConsumer.join() + +#the logic to run as process +if __name__ == "__main__": + #Set logging format and level + #logging.basicConfig( + # format='%(asctime)s.%(msecs)s:%(name)s:%(thread)d:%(levelname)s:%(process)d:%(message)s', + # level=logging.INFO) + + #Call the main function + main() + diff --git a/kafkaProducerService.py b/kafkaProducerService.py index 5d10fa7..4c4b7c6 100644 --- a/kafkaProducerService.py +++ b/kafkaProducerService.py @@ -34,7 +34,7 @@ def run(self): while not self.stop_event.is_set(): #Send two messages of type binary to the 'test' Topic producer.send('test', b"test") - producer.send('test', b"\xc2Hola, mundo!") + producer.send('test', b"Hola, mundo!") #Sleep for 3 seconds time.sleep(3) From 05b23b1c4850a5ec90e1ab10e1cbe6c17104d9ed Mon Sep 17 00:00:00 2001 From: myersr Date: Fri, 4 May 2018 16:34:48 -0400 Subject: [PATCH 087/109] added consumer script --- kafkaConsumerService.py | 96 +++++++++++++++++++++++++++++++++++++++++ kafkaProducerService.py | 2 +- 2 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 kafkaConsumerService.py diff --git a/kafkaConsumerService.py b/kafkaConsumerService.py new file mode 100644 index 0000000..a0580eb --- /dev/null +++ b/kafkaConsumerService.py @@ -0,0 +1,96 @@ +import threading, logging, time +import multiprocessing +import os + +from kafka import KafkaConsumer + + +#Thread class that continuously consumes messages from a Kafka message queue/topic +# Input: +# threading.Thread = instance of multiproccessing thread +class Consumer(multiprocessing.Process): + #default initializing function of the thread. + # Input: + # takes self to modify and initialize + def __init__(self): + #initializes thread and passes self. Magic multithreading stuff + multiprocessing.Process.__init__(self) + #Gives the thread an envent called stop_event so it can be interrupted. + self.stop_event = multiprocessing.Event() + + #Function to stop the process + def stop(self): + #Calls even stop_event and sets it. + #This gives context to the thread from the outside and lets you stop it. + self.stop_event.set() + + #The main run function called when you call start. + def run(self): + if hasattr(os, 'getppid'): # only available on Unix + print 'parent process:', os.getppid() + procID = os.getppid() + #Bootstraps an instance of a Kafka producer. + #Initializes the producer and identifies the docker server. + #kafka-spotify is listed in /etc/hosts with the ip of the container + #Input: + # topic to subscribe to: 'test' + # Id to identify the consumer should be unique to the connection + # Servers kafka is advertising as + # Which message rule to subscribe to. 'earliest' will grab the earliest unprocessed message + # Timeout limit + consumer = KafkaConsumer('test', + client_id='python-consumer-%s' % (procID), + bootstrap_servers=['kafka-spotify:9092'], + auto_offset_reset='latest', + consumer_timeout_ms=1000) + + #Alternative way to subscribe to a topic + #consumer.subscribe(['test']) + + #loop until the thread is stopped by checking the stop event + while not self.stop_event.is_set(): + #Loop through ConsumerRecord objects in the consumer object + for message in consumer: + #print the messages to the screen with a note of the thread/client ID + #print("python-consumer-%s processed message: %s" % (procID, message)) + #print the messages to the screen with a note of the thread/client ID, Current Topic, message number, # The value of the message decoded as it is sent as bytecode + print ("python-consumer-%s processed message: %s:%d: value=%s" % (procID, message.topic, + message.offset, message.value.decode('utf-8'))) + #break out of the for loop if the thread was notified of closure + if self.stop_event.is_set(): + break + + #Close the TCP connection to kafka + consumer.close() + + +#Main function called when the app is run +def main(): + #initialize a Consumer object/thread + kafkConsumer = Consumer() + + #Start the thread working. + kafkConsumer.start() + + #sleep for 17 second. If we weren't using threads this would halt the code + time.sleep(20) + + #Call stop to set the thread event so it knows to stop + print("Stopping kafkConsumer") + kafkConsumer.stop() + + #Wait until the thread terminates. Can see the docs for more + #https://docs.python.org/2/library/threading.html?highlight=thread#threading.Thread.join + print("Waiting for execution to halt") + kafkConsumer.join() + +#the logic to run as process +if __name__ == "__main__": + #Set logging format and level + #logging.basicConfig( + # format='%(asctime)s.%(msecs)s:%(name)s:%(thread)d:%(levelname)s:%(process)d:%(message)s', + # level=logging.INFO) + + #Call the main function + main() + diff --git a/kafkaProducerService.py b/kafkaProducerService.py index 5d10fa7..4c4b7c6 100644 --- a/kafkaProducerService.py +++ b/kafkaProducerService.py @@ -34,7 +34,7 @@ def run(self): while not self.stop_event.is_set(): #Send two messages of type binary to the 'test' Topic producer.send('test', b"test") - producer.send('test', b"\xc2Hola, mundo!") + producer.send('test', b"Hola, mundo!") #Sleep for 3 seconds time.sleep(3) From 469e4c4d99d902b40211d87067c753c50a91b80e Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Fri, 4 May 2018 16:34:48 -0400 Subject: [PATCH 088/109] added consumer script --- kafkaConsumerService.py | 96 +++++++++++++++++++++++++++++++++++++++++ kafkaProducerService.py | 2 +- 2 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 kafkaConsumerService.py diff --git a/kafkaConsumerService.py b/kafkaConsumerService.py new file mode 100644 index 0000000..a0580eb --- /dev/null +++ b/kafkaConsumerService.py @@ -0,0 +1,96 @@ +import threading, logging, time +import multiprocessing +import os + +from kafka import KafkaConsumer + + +#Thread class that continuously consumes messages from a Kafka message queue/topic +# Input: +# threading.Thread = instance of multiproccessing thread +class Consumer(multiprocessing.Process): + #default initializing function of the thread. + # Input: + # takes self to modify and initialize + def __init__(self): + #initializes thread and passes self. Magic multithreading stuff + multiprocessing.Process.__init__(self) + #Gives the thread an envent called stop_event so it can be interrupted. + self.stop_event = multiprocessing.Event() + + #Function to stop the process + def stop(self): + #Calls even stop_event and sets it. + #This gives context to the thread from the outside and lets you stop it. + self.stop_event.set() + + #The main run function called when you call start. + def run(self): + if hasattr(os, 'getppid'): # only available on Unix + print 'parent process:', os.getppid() + procID = os.getppid() + #Bootstraps an instance of a Kafka producer. + #Initializes the producer and identifies the docker server. + #kafka-spotify is listed in /etc/hosts with the ip of the container + #Input: + # topic to subscribe to: 'test' + # Id to identify the consumer should be unique to the connection + # Servers kafka is advertising as + # Which message rule to subscribe to. 'earliest' will grab the earliest unprocessed message + # Timeout limit + consumer = KafkaConsumer('test', + client_id='python-consumer-%s' % (procID), + bootstrap_servers=['kafka-spotify:9092'], + auto_offset_reset='latest', + consumer_timeout_ms=1000) + + #Alternative way to subscribe to a topic + #consumer.subscribe(['test']) + + #loop until the thread is stopped by checking the stop event + while not self.stop_event.is_set(): + #Loop through ConsumerRecord objects in the consumer object + for message in consumer: + #print the messages to the screen with a note of the thread/client ID + #print("python-consumer-%s processed message: %s" % (procID, message)) + #print the messages to the screen with a note of the thread/client ID, Current Topic, message number, # The value of the message decoded as it is sent as bytecode + print ("python-consumer-%s processed message: %s:%d: value=%s" % (procID, message.topic, + message.offset, message.value.decode('utf-8'))) + #break out of the for loop if the thread was notified of closure + if self.stop_event.is_set(): + break + + #Close the TCP connection to kafka + consumer.close() + + +#Main function called when the app is run +def main(): + #initialize a Consumer object/thread + kafkConsumer = Consumer() + + #Start the thread working. + kafkConsumer.start() + + #sleep for 17 second. If we weren't using threads this would halt the code + time.sleep(20) + + #Call stop to set the thread event so it knows to stop + print("Stopping kafkConsumer") + kafkConsumer.stop() + + #Wait until the thread terminates. Can see the docs for more + #https://docs.python.org/2/library/threading.html?highlight=thread#threading.Thread.join + print("Waiting for execution to halt") + kafkConsumer.join() + +#the logic to run as process +if __name__ == "__main__": + #Set logging format and level + #logging.basicConfig( + # format='%(asctime)s.%(msecs)s:%(name)s:%(thread)d:%(levelname)s:%(process)d:%(message)s', + # level=logging.INFO) + + #Call the main function + main() + diff --git a/kafkaProducerService.py b/kafkaProducerService.py index 5d10fa7..4c4b7c6 100644 --- a/kafkaProducerService.py +++ b/kafkaProducerService.py @@ -34,7 +34,7 @@ def run(self): while not self.stop_event.is_set(): #Send two messages of type binary to the 'test' Topic producer.send('test', b"test") - producer.send('test', b"\xc2Hola, mundo!") + producer.send('test', b"Hola, mundo!") #Sleep for 3 seconds time.sleep(3) From 8396e164b3a774e7089966f742d21cd04b690d66 Mon Sep 17 00:00:00 2001 From: myersr Date: Fri, 4 May 2018 16:34:48 -0400 Subject: [PATCH 089/109] added consumer script --- kafkaConsumerService.py | 96 +++++++++++++++++++++++++++++++++++++++++ kafkaProducerService.py | 2 +- 2 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 kafkaConsumerService.py diff --git a/kafkaConsumerService.py b/kafkaConsumerService.py new file mode 100644 index 0000000..a0580eb --- /dev/null +++ b/kafkaConsumerService.py @@ -0,0 +1,96 @@ +import threading, logging, time +import multiprocessing +import os + +from kafka import KafkaConsumer + + +#Thread class that continuously consumes messages from a Kafka message queue/topic +# Input: +# threading.Thread = instance of multiproccessing thread +class Consumer(multiprocessing.Process): + #default initializing function of the thread. + # Input: + # takes self to modify and initialize + def __init__(self): + #initializes thread and passes self. Magic multithreading stuff + multiprocessing.Process.__init__(self) + #Gives the thread an envent called stop_event so it can be interrupted. + self.stop_event = multiprocessing.Event() + + #Function to stop the process + def stop(self): + #Calls even stop_event and sets it. + #This gives context to the thread from the outside and lets you stop it. + self.stop_event.set() + + #The main run function called when you call start. + def run(self): + if hasattr(os, 'getppid'): # only available on Unix + print 'parent process:', os.getppid() + procID = os.getppid() + #Bootstraps an instance of a Kafka producer. + #Initializes the producer and identifies the docker server. + #kafka-spotify is listed in /etc/hosts with the ip of the container + #Input: + # topic to subscribe to: 'test' + # Id to identify the consumer should be unique to the connection + # Servers kafka is advertising as + # Which message rule to subscribe to. 'earliest' will grab the earliest unprocessed message + # Timeout limit + consumer = KafkaConsumer('test', + client_id='python-consumer-%s' % (procID), + bootstrap_servers=['kafka-spotify:9092'], + auto_offset_reset='latest', + consumer_timeout_ms=1000) + + #Alternative way to subscribe to a topic + #consumer.subscribe(['test']) + + #loop until the thread is stopped by checking the stop event + while not self.stop_event.is_set(): + #Loop through ConsumerRecord objects in the consumer object + for message in consumer: + #print the messages to the screen with a note of the thread/client ID + #print("python-consumer-%s processed message: %s" % (procID, message)) + #print the messages to the screen with a note of the thread/client ID, Current Topic, message number, # The value of the message decoded as it is sent as bytecode + print ("python-consumer-%s processed message: %s:%d: value=%s" % (procID, message.topic, + message.offset, message.value.decode('utf-8'))) + #break out of the for loop if the thread was notified of closure + if self.stop_event.is_set(): + break + + #Close the TCP connection to kafka + consumer.close() + + +#Main function called when the app is run +def main(): + #initialize a Consumer object/thread + kafkConsumer = Consumer() + + #Start the thread working. + kafkConsumer.start() + + #sleep for 17 second. If we weren't using threads this would halt the code + time.sleep(20) + + #Call stop to set the thread event so it knows to stop + print("Stopping kafkConsumer") + kafkConsumer.stop() + + #Wait until the thread terminates. Can see the docs for more + #https://docs.python.org/2/library/threading.html?highlight=thread#threading.Thread.join + print("Waiting for execution to halt") + kafkConsumer.join() + +#the logic to run as process +if __name__ == "__main__": + #Set logging format and level + #logging.basicConfig( + # format='%(asctime)s.%(msecs)s:%(name)s:%(thread)d:%(levelname)s:%(process)d:%(message)s', + # level=logging.INFO) + + #Call the main function + main() + diff --git a/kafkaProducerService.py b/kafkaProducerService.py index 5d10fa7..4c4b7c6 100644 --- a/kafkaProducerService.py +++ b/kafkaProducerService.py @@ -34,7 +34,7 @@ def run(self): while not self.stop_event.is_set(): #Send two messages of type binary to the 'test' Topic producer.send('test', b"test") - producer.send('test', b"\xc2Hola, mundo!") + producer.send('test', b"Hola, mundo!") #Sleep for 3 seconds time.sleep(3) From 1902392b6d08c524ae04f70c2dfc78a01a0a1369 Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Fri, 4 May 2018 16:49:51 -0400 Subject: [PATCH 090/109] added consumer script --- kafkaConsumerService.py | 96 +++++++++++++++++++++++++++++++++++++++++ kafkaProducerService.py | 2 +- 2 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 kafkaConsumerService.py diff --git a/kafkaConsumerService.py b/kafkaConsumerService.py new file mode 100644 index 0000000..a0580eb --- /dev/null +++ b/kafkaConsumerService.py @@ -0,0 +1,96 @@ +import threading, logging, time +import multiprocessing +import os + +from kafka import KafkaConsumer + + +#Thread class that continuously consumes messages from a Kafka message queue/topic +# Input: +# threading.Thread = instance of multiproccessing thread +class Consumer(multiprocessing.Process): + #default initializing function of the thread. + # Input: + # takes self to modify and initialize + def __init__(self): + #initializes thread and passes self. Magic multithreading stuff + multiprocessing.Process.__init__(self) + #Gives the thread an envent called stop_event so it can be interrupted. + self.stop_event = multiprocessing.Event() + + #Function to stop the process + def stop(self): + #Calls even stop_event and sets it. + #This gives context to the thread from the outside and lets you stop it. + self.stop_event.set() + + #The main run function called when you call start. + def run(self): + if hasattr(os, 'getppid'): # only available on Unix + print 'parent process:', os.getppid() + procID = os.getppid() + #Bootstraps an instance of a Kafka producer. + #Initializes the producer and identifies the docker server. + #kafka-spotify is listed in /etc/hosts with the ip of the container + #Input: + # topic to subscribe to: 'test' + # Id to identify the consumer should be unique to the connection + # Servers kafka is advertising as + # Which message rule to subscribe to. 'earliest' will grab the earliest unprocessed message + # Timeout limit + consumer = KafkaConsumer('test', + client_id='python-consumer-%s' % (procID), + bootstrap_servers=['kafka-spotify:9092'], + auto_offset_reset='latest', + consumer_timeout_ms=1000) + + #Alternative way to subscribe to a topic + #consumer.subscribe(['test']) + + #loop until the thread is stopped by checking the stop event + while not self.stop_event.is_set(): + #Loop through ConsumerRecord objects in the consumer object + for message in consumer: + #print the messages to the screen with a note of the thread/client ID + #print("python-consumer-%s processed message: %s" % (procID, message)) + #print the messages to the screen with a note of the thread/client ID, Current Topic, message number, # The value of the message decoded as it is sent as bytecode + print ("python-consumer-%s processed message: %s:%d: value=%s" % (procID, message.topic, + message.offset, message.value.decode('utf-8'))) + #break out of the for loop if the thread was notified of closure + if self.stop_event.is_set(): + break + + #Close the TCP connection to kafka + consumer.close() + + +#Main function called when the app is run +def main(): + #initialize a Consumer object/thread + kafkConsumer = Consumer() + + #Start the thread working. + kafkConsumer.start() + + #sleep for 17 second. If we weren't using threads this would halt the code + time.sleep(20) + + #Call stop to set the thread event so it knows to stop + print("Stopping kafkConsumer") + kafkConsumer.stop() + + #Wait until the thread terminates. Can see the docs for more + #https://docs.python.org/2/library/threading.html?highlight=thread#threading.Thread.join + print("Waiting for execution to halt") + kafkConsumer.join() + +#the logic to run as process +if __name__ == "__main__": + #Set logging format and level + #logging.basicConfig( + # format='%(asctime)s.%(msecs)s:%(name)s:%(thread)d:%(levelname)s:%(process)d:%(message)s', + # level=logging.INFO) + + #Call the main function + main() + diff --git a/kafkaProducerService.py b/kafkaProducerService.py index 5d10fa7..4c4b7c6 100644 --- a/kafkaProducerService.py +++ b/kafkaProducerService.py @@ -34,7 +34,7 @@ def run(self): while not self.stop_event.is_set(): #Send two messages of type binary to the 'test' Topic producer.send('test', b"test") - producer.send('test', b"\xc2Hola, mundo!") + producer.send('test', b"Hola, mundo!") #Sleep for 3 seconds time.sleep(3) From 669d665ab9897851c201f0ba483ae7663bac7151 Mon Sep 17 00:00:00 2001 From: myersr Date: Fri, 4 May 2018 16:49:51 -0400 Subject: [PATCH 091/109] added consumer script --- kafkaConsumerService.py | 96 +++++++++++++++++++++++++++++++++++++++++ kafkaProducerService.py | 2 +- 2 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 kafkaConsumerService.py diff --git a/kafkaConsumerService.py b/kafkaConsumerService.py new file mode 100644 index 0000000..a0580eb --- /dev/null +++ b/kafkaConsumerService.py @@ -0,0 +1,96 @@ +import threading, logging, time +import multiprocessing +import os + +from kafka import KafkaConsumer + + +#Thread class that continuously consumes messages from a Kafka message queue/topic +# Input: +# threading.Thread = instance of multiproccessing thread +class Consumer(multiprocessing.Process): + #default initializing function of the thread. + # Input: + # takes self to modify and initialize + def __init__(self): + #initializes thread and passes self. Magic multithreading stuff + multiprocessing.Process.__init__(self) + #Gives the thread an envent called stop_event so it can be interrupted. + self.stop_event = multiprocessing.Event() + + #Function to stop the process + def stop(self): + #Calls even stop_event and sets it. + #This gives context to the thread from the outside and lets you stop it. + self.stop_event.set() + + #The main run function called when you call start. + def run(self): + if hasattr(os, 'getppid'): # only available on Unix + print 'parent process:', os.getppid() + procID = os.getppid() + #Bootstraps an instance of a Kafka producer. + #Initializes the producer and identifies the docker server. + #kafka-spotify is listed in /etc/hosts with the ip of the container + #Input: + # topic to subscribe to: 'test' + # Id to identify the consumer should be unique to the connection + # Servers kafka is advertising as + # Which message rule to subscribe to. 'earliest' will grab the earliest unprocessed message + # Timeout limit + consumer = KafkaConsumer('test', + client_id='python-consumer-%s' % (procID), + bootstrap_servers=['kafka-spotify:9092'], + auto_offset_reset='latest', + consumer_timeout_ms=1000) + + #Alternative way to subscribe to a topic + #consumer.subscribe(['test']) + + #loop until the thread is stopped by checking the stop event + while not self.stop_event.is_set(): + #Loop through ConsumerRecord objects in the consumer object + for message in consumer: + #print the messages to the screen with a note of the thread/client ID + #print("python-consumer-%s processed message: %s" % (procID, message)) + #print the messages to the screen with a note of the thread/client ID, Current Topic, message number, # The value of the message decoded as it is sent as bytecode + print ("python-consumer-%s processed message: %s:%d: value=%s" % (procID, message.topic, + message.offset, message.value.decode('utf-8'))) + #break out of the for loop if the thread was notified of closure + if self.stop_event.is_set(): + break + + #Close the TCP connection to kafka + consumer.close() + + +#Main function called when the app is run +def main(): + #initialize a Consumer object/thread + kafkConsumer = Consumer() + + #Start the thread working. + kafkConsumer.start() + + #sleep for 17 second. If we weren't using threads this would halt the code + time.sleep(20) + + #Call stop to set the thread event so it knows to stop + print("Stopping kafkConsumer") + kafkConsumer.stop() + + #Wait until the thread terminates. Can see the docs for more + #https://docs.python.org/2/library/threading.html?highlight=thread#threading.Thread.join + print("Waiting for execution to halt") + kafkConsumer.join() + +#the logic to run as process +if __name__ == "__main__": + #Set logging format and level + #logging.basicConfig( + # format='%(asctime)s.%(msecs)s:%(name)s:%(thread)d:%(levelname)s:%(process)d:%(message)s', + # level=logging.INFO) + + #Call the main function + main() + diff --git a/kafkaProducerService.py b/kafkaProducerService.py index 5d10fa7..4c4b7c6 100644 --- a/kafkaProducerService.py +++ b/kafkaProducerService.py @@ -34,7 +34,7 @@ def run(self): while not self.stop_event.is_set(): #Send two messages of type binary to the 'test' Topic producer.send('test', b"test") - producer.send('test', b"\xc2Hola, mundo!") + producer.send('test', b"Hola, mundo!") #Sleep for 3 seconds time.sleep(3) From f11c6189ba710c173521e6d213dbc5ffd19cad49 Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Fri, 4 May 2018 16:49:51 -0400 Subject: [PATCH 092/109] added consumer script --- kafkaConsumerService.py | 96 +++++++++++++++++++++++++++++++++++++++++ kafkaProducerService.py | 2 +- 2 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 kafkaConsumerService.py diff --git a/kafkaConsumerService.py b/kafkaConsumerService.py new file mode 100644 index 0000000..a0580eb --- /dev/null +++ b/kafkaConsumerService.py @@ -0,0 +1,96 @@ +import threading, logging, time +import multiprocessing +import os + +from kafka import KafkaConsumer + + +#Thread class that continuously consumes messages from a Kafka message queue/topic +# Input: +# threading.Thread = instance of multiproccessing thread +class Consumer(multiprocessing.Process): + #default initializing function of the thread. + # Input: + # takes self to modify and initialize + def __init__(self): + #initializes thread and passes self. Magic multithreading stuff + multiprocessing.Process.__init__(self) + #Gives the thread an envent called stop_event so it can be interrupted. + self.stop_event = multiprocessing.Event() + + #Function to stop the process + def stop(self): + #Calls even stop_event and sets it. + #This gives context to the thread from the outside and lets you stop it. + self.stop_event.set() + + #The main run function called when you call start. + def run(self): + if hasattr(os, 'getppid'): # only available on Unix + print 'parent process:', os.getppid() + procID = os.getppid() + #Bootstraps an instance of a Kafka producer. + #Initializes the producer and identifies the docker server. + #kafka-spotify is listed in /etc/hosts with the ip of the container + #Input: + # topic to subscribe to: 'test' + # Id to identify the consumer should be unique to the connection + # Servers kafka is advertising as + # Which message rule to subscribe to. 'earliest' will grab the earliest unprocessed message + # Timeout limit + consumer = KafkaConsumer('test', + client_id='python-consumer-%s' % (procID), + bootstrap_servers=['kafka-spotify:9092'], + auto_offset_reset='latest', + consumer_timeout_ms=1000) + + #Alternative way to subscribe to a topic + #consumer.subscribe(['test']) + + #loop until the thread is stopped by checking the stop event + while not self.stop_event.is_set(): + #Loop through ConsumerRecord objects in the consumer object + for message in consumer: + #print the messages to the screen with a note of the thread/client ID + #print("python-consumer-%s processed message: %s" % (procID, message)) + #print the messages to the screen with a note of the thread/client ID, Current Topic, message number, # The value of the message decoded as it is sent as bytecode + print ("python-consumer-%s processed message: %s:%d: value=%s" % (procID, message.topic, + message.offset, message.value.decode('utf-8'))) + #break out of the for loop if the thread was notified of closure + if self.stop_event.is_set(): + break + + #Close the TCP connection to kafka + consumer.close() + + +#Main function called when the app is run +def main(): + #initialize a Consumer object/thread + kafkConsumer = Consumer() + + #Start the thread working. + kafkConsumer.start() + + #sleep for 17 second. If we weren't using threads this would halt the code + time.sleep(20) + + #Call stop to set the thread event so it knows to stop + print("Stopping kafkConsumer") + kafkConsumer.stop() + + #Wait until the thread terminates. Can see the docs for more + #https://docs.python.org/2/library/threading.html?highlight=thread#threading.Thread.join + print("Waiting for execution to halt") + kafkConsumer.join() + +#the logic to run as process +if __name__ == "__main__": + #Set logging format and level + #logging.basicConfig( + # format='%(asctime)s.%(msecs)s:%(name)s:%(thread)d:%(levelname)s:%(process)d:%(message)s', + # level=logging.INFO) + + #Call the main function + main() + diff --git a/kafkaProducerService.py b/kafkaProducerService.py index 5d10fa7..4c4b7c6 100644 --- a/kafkaProducerService.py +++ b/kafkaProducerService.py @@ -34,7 +34,7 @@ def run(self): while not self.stop_event.is_set(): #Send two messages of type binary to the 'test' Topic producer.send('test', b"test") - producer.send('test', b"\xc2Hola, mundo!") + producer.send('test', b"Hola, mundo!") #Sleep for 3 seconds time.sleep(3) From 73ebbde86b360dc546f831dd91a7dfa95053d8d4 Mon Sep 17 00:00:00 2001 From: myersr Date: Fri, 4 May 2018 16:49:51 -0400 Subject: [PATCH 093/109] added consumer script --- kafkaConsumerService.py | 96 +++++++++++++++++++++++++++++++++++++++++ kafkaProducerService.py | 2 +- 2 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 kafkaConsumerService.py diff --git a/kafkaConsumerService.py b/kafkaConsumerService.py new file mode 100644 index 0000000..a0580eb --- /dev/null +++ b/kafkaConsumerService.py @@ -0,0 +1,96 @@ +import threading, logging, time +import multiprocessing +import os + +from kafka import KafkaConsumer + + +#Thread class that continuously consumes messages from a Kafka message queue/topic +# Input: +# threading.Thread = instance of multiproccessing thread +class Consumer(multiprocessing.Process): + #default initializing function of the thread. + # Input: + # takes self to modify and initialize + def __init__(self): + #initializes thread and passes self. Magic multithreading stuff + multiprocessing.Process.__init__(self) + #Gives the thread an envent called stop_event so it can be interrupted. + self.stop_event = multiprocessing.Event() + + #Function to stop the process + def stop(self): + #Calls even stop_event and sets it. + #This gives context to the thread from the outside and lets you stop it. + self.stop_event.set() + + #The main run function called when you call start. + def run(self): + if hasattr(os, 'getppid'): # only available on Unix + print 'parent process:', os.getppid() + procID = os.getppid() + #Bootstraps an instance of a Kafka producer. + #Initializes the producer and identifies the docker server. + #kafka-spotify is listed in /etc/hosts with the ip of the container + #Input: + # topic to subscribe to: 'test' + # Id to identify the consumer should be unique to the connection + # Servers kafka is advertising as + # Which message rule to subscribe to. 'earliest' will grab the earliest unprocessed message + # Timeout limit + consumer = KafkaConsumer('test', + client_id='python-consumer-%s' % (procID), + bootstrap_servers=['kafka-spotify:9092'], + auto_offset_reset='latest', + consumer_timeout_ms=1000) + + #Alternative way to subscribe to a topic + #consumer.subscribe(['test']) + + #loop until the thread is stopped by checking the stop event + while not self.stop_event.is_set(): + #Loop through ConsumerRecord objects in the consumer object + for message in consumer: + #print the messages to the screen with a note of the thread/client ID + #print("python-consumer-%s processed message: %s" % (procID, message)) + #print the messages to the screen with a note of the thread/client ID, Current Topic, message number, # The value of the message decoded as it is sent as bytecode + print ("python-consumer-%s processed message: %s:%d: value=%s" % (procID, message.topic, + message.offset, message.value.decode('utf-8'))) + #break out of the for loop if the thread was notified of closure + if self.stop_event.is_set(): + break + + #Close the TCP connection to kafka + consumer.close() + + +#Main function called when the app is run +def main(): + #initialize a Consumer object/thread + kafkConsumer = Consumer() + + #Start the thread working. + kafkConsumer.start() + + #sleep for 17 second. If we weren't using threads this would halt the code + time.sleep(20) + + #Call stop to set the thread event so it knows to stop + print("Stopping kafkConsumer") + kafkConsumer.stop() + + #Wait until the thread terminates. Can see the docs for more + #https://docs.python.org/2/library/threading.html?highlight=thread#threading.Thread.join + print("Waiting for execution to halt") + kafkConsumer.join() + +#the logic to run as process +if __name__ == "__main__": + #Set logging format and level + #logging.basicConfig( + # format='%(asctime)s.%(msecs)s:%(name)s:%(thread)d:%(levelname)s:%(process)d:%(message)s', + # level=logging.INFO) + + #Call the main function + main() + diff --git a/kafkaProducerService.py b/kafkaProducerService.py index 5d10fa7..4c4b7c6 100644 --- a/kafkaProducerService.py +++ b/kafkaProducerService.py @@ -34,7 +34,7 @@ def run(self): while not self.stop_event.is_set(): #Send two messages of type binary to the 'test' Topic producer.send('test', b"test") - producer.send('test', b"\xc2Hola, mundo!") + producer.send('test', b"Hola, mundo!") #Sleep for 3 seconds time.sleep(3) From fbbf4e2f4174c816d0a31e6db4a6394f2c1cde02 Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Fri, 4 May 2018 16:49:51 -0400 Subject: [PATCH 094/109] added consumer script --- kafkaConsumerService.py | 96 +++++++++++++++++++++++++++++++++++++++++ kafkaProducerService.py | 2 +- 2 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 kafkaConsumerService.py diff --git a/kafkaConsumerService.py b/kafkaConsumerService.py new file mode 100644 index 0000000..a0580eb --- /dev/null +++ b/kafkaConsumerService.py @@ -0,0 +1,96 @@ +import threading, logging, time +import multiprocessing +import os + +from kafka import KafkaConsumer + + +#Thread class that continuously consumes messages from a Kafka message queue/topic +# Input: +# threading.Thread = instance of multiproccessing thread +class Consumer(multiprocessing.Process): + #default initializing function of the thread. + # Input: + # takes self to modify and initialize + def __init__(self): + #initializes thread and passes self. Magic multithreading stuff + multiprocessing.Process.__init__(self) + #Gives the thread an envent called stop_event so it can be interrupted. + self.stop_event = multiprocessing.Event() + + #Function to stop the process + def stop(self): + #Calls even stop_event and sets it. + #This gives context to the thread from the outside and lets you stop it. + self.stop_event.set() + + #The main run function called when you call start. + def run(self): + if hasattr(os, 'getppid'): # only available on Unix + print 'parent process:', os.getppid() + procID = os.getppid() + #Bootstraps an instance of a Kafka producer. + #Initializes the producer and identifies the docker server. + #kafka-spotify is listed in /etc/hosts with the ip of the container + #Input: + # topic to subscribe to: 'test' + # Id to identify the consumer should be unique to the connection + # Servers kafka is advertising as + # Which message rule to subscribe to. 'earliest' will grab the earliest unprocessed message + # Timeout limit + consumer = KafkaConsumer('test', + client_id='python-consumer-%s' % (procID), + bootstrap_servers=['kafka-spotify:9092'], + auto_offset_reset='latest', + consumer_timeout_ms=1000) + + #Alternative way to subscribe to a topic + #consumer.subscribe(['test']) + + #loop until the thread is stopped by checking the stop event + while not self.stop_event.is_set(): + #Loop through ConsumerRecord objects in the consumer object + for message in consumer: + #print the messages to the screen with a note of the thread/client ID + #print("python-consumer-%s processed message: %s" % (procID, message)) + #print the messages to the screen with a note of the thread/client ID, Current Topic, message number, # The value of the message decoded as it is sent as bytecode + print ("python-consumer-%s processed message: %s:%d: value=%s" % (procID, message.topic, + message.offset, message.value.decode('utf-8'))) + #break out of the for loop if the thread was notified of closure + if self.stop_event.is_set(): + break + + #Close the TCP connection to kafka + consumer.close() + + +#Main function called when the app is run +def main(): + #initialize a Consumer object/thread + kafkConsumer = Consumer() + + #Start the thread working. + kafkConsumer.start() + + #sleep for 17 second. If we weren't using threads this would halt the code + time.sleep(20) + + #Call stop to set the thread event so it knows to stop + print("Stopping kafkConsumer") + kafkConsumer.stop() + + #Wait until the thread terminates. Can see the docs for more + #https://docs.python.org/2/library/threading.html?highlight=thread#threading.Thread.join + print("Waiting for execution to halt") + kafkConsumer.join() + +#the logic to run as process +if __name__ == "__main__": + #Set logging format and level + #logging.basicConfig( + # format='%(asctime)s.%(msecs)s:%(name)s:%(thread)d:%(levelname)s:%(process)d:%(message)s', + # level=logging.INFO) + + #Call the main function + main() + diff --git a/kafkaProducerService.py b/kafkaProducerService.py index 5d10fa7..4c4b7c6 100644 --- a/kafkaProducerService.py +++ b/kafkaProducerService.py @@ -34,7 +34,7 @@ def run(self): while not self.stop_event.is_set(): #Send two messages of type binary to the 'test' Topic producer.send('test', b"test") - producer.send('test', b"\xc2Hola, mundo!") + producer.send('test', b"Hola, mundo!") #Sleep for 3 seconds time.sleep(3) From 557357295fd166712cf6ce23120694f300fec9fe Mon Sep 17 00:00:00 2001 From: myersr Date: Fri, 4 May 2018 16:49:51 -0400 Subject: [PATCH 095/109] added consumer script --- kafkaConsumerService.py | 96 +++++++++++++++++++++++++++++++++++++++++ kafkaProducerService.py | 2 +- 2 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 kafkaConsumerService.py diff --git a/kafkaConsumerService.py b/kafkaConsumerService.py new file mode 100644 index 0000000..a0580eb --- /dev/null +++ b/kafkaConsumerService.py @@ -0,0 +1,96 @@ +import threading, logging, time +import multiprocessing +import os + +from kafka import KafkaConsumer + + +#Thread class that continuously consumes messages from a Kafka message queue/topic +# Input: +# threading.Thread = instance of multiproccessing thread +class Consumer(multiprocessing.Process): + #default initializing function of the thread. + # Input: + # takes self to modify and initialize + def __init__(self): + #initializes thread and passes self. Magic multithreading stuff + multiprocessing.Process.__init__(self) + #Gives the thread an envent called stop_event so it can be interrupted. + self.stop_event = multiprocessing.Event() + + #Function to stop the process + def stop(self): + #Calls even stop_event and sets it. + #This gives context to the thread from the outside and lets you stop it. + self.stop_event.set() + + #The main run function called when you call start. + def run(self): + if hasattr(os, 'getppid'): # only available on Unix + print 'parent process:', os.getppid() + procID = os.getppid() + #Bootstraps an instance of a Kafka producer. + #Initializes the producer and identifies the docker server. + #kafka-spotify is listed in /etc/hosts with the ip of the container + #Input: + # topic to subscribe to: 'test' + # Id to identify the consumer should be unique to the connection + # Servers kafka is advertising as + # Which message rule to subscribe to. 'earliest' will grab the earliest unprocessed message + # Timeout limit + consumer = KafkaConsumer('test', + client_id='python-consumer-%s' % (procID), + bootstrap_servers=['kafka-spotify:9092'], + auto_offset_reset='latest', + consumer_timeout_ms=1000) + + #Alternative way to subscribe to a topic + #consumer.subscribe(['test']) + + #loop until the thread is stopped by checking the stop event + while not self.stop_event.is_set(): + #Loop through ConsumerRecord objects in the consumer object + for message in consumer: + #print the messages to the screen with a note of the thread/client ID + #print("python-consumer-%s processed message: %s" % (procID, message)) + #print the messages to the screen with a note of the thread/client ID, Current Topic, message number, # The value of the message decoded as it is sent as bytecode + print ("python-consumer-%s processed message: %s:%d: value=%s" % (procID, message.topic, + message.offset, message.value.decode('utf-8'))) + #break out of the for loop if the thread was notified of closure + if self.stop_event.is_set(): + break + + #Close the TCP connection to kafka + consumer.close() + + +#Main function called when the app is run +def main(): + #initialize a Consumer object/thread + kafkConsumer = Consumer() + + #Start the thread working. + kafkConsumer.start() + + #sleep for 17 second. If we weren't using threads this would halt the code + time.sleep(20) + + #Call stop to set the thread event so it knows to stop + print("Stopping kafkConsumer") + kafkConsumer.stop() + + #Wait until the thread terminates. Can see the docs for more + #https://docs.python.org/2/library/threading.html?highlight=thread#threading.Thread.join + print("Waiting for execution to halt") + kafkConsumer.join() + +#the logic to run as process +if __name__ == "__main__": + #Set logging format and level + #logging.basicConfig( + # format='%(asctime)s.%(msecs)s:%(name)s:%(thread)d:%(levelname)s:%(process)d:%(message)s', + # level=logging.INFO) + + #Call the main function + main() + diff --git a/kafkaProducerService.py b/kafkaProducerService.py index 5d10fa7..4c4b7c6 100644 --- a/kafkaProducerService.py +++ b/kafkaProducerService.py @@ -34,7 +34,7 @@ def run(self): while not self.stop_event.is_set(): #Send two messages of type binary to the 'test' Topic producer.send('test', b"test") - producer.send('test', b"\xc2Hola, mundo!") + producer.send('test', b"Hola, mundo!") #Sleep for 3 seconds time.sleep(3) From df344ffbed6a343655ab298d449d9fd610865650 Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Fri, 4 May 2018 16:49:51 -0400 Subject: [PATCH 096/109] added consumer script --- kafkaConsumerService.py | 96 +++++++++++++++++++++++++++++++++++++++++ kafkaProducerService.py | 2 +- 2 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 kafkaConsumerService.py diff --git a/kafkaConsumerService.py b/kafkaConsumerService.py new file mode 100644 index 0000000..a0580eb --- /dev/null +++ b/kafkaConsumerService.py @@ -0,0 +1,96 @@ +import threading, logging, time +import multiprocessing +import os + +from kafka import KafkaConsumer + + +#Thread class that continuously consumes messages from a Kafka message queue/topic +# Input: +# threading.Thread = instance of multiproccessing thread +class Consumer(multiprocessing.Process): + #default initializing function of the thread. + # Input: + # takes self to modify and initialize + def __init__(self): + #initializes thread and passes self. Magic multithreading stuff + multiprocessing.Process.__init__(self) + #Gives the thread an envent called stop_event so it can be interrupted. + self.stop_event = multiprocessing.Event() + + #Function to stop the process + def stop(self): + #Calls even stop_event and sets it. + #This gives context to the thread from the outside and lets you stop it. + self.stop_event.set() + + #The main run function called when you call start. + def run(self): + if hasattr(os, 'getppid'): # only available on Unix + print 'parent process:', os.getppid() + procID = os.getppid() + #Bootstraps an instance of a Kafka producer. + #Initializes the producer and identifies the docker server. + #kafka-spotify is listed in /etc/hosts with the ip of the container + #Input: + # topic to subscribe to: 'test' + # Id to identify the consumer should be unique to the connection + # Servers kafka is advertising as + # Which message rule to subscribe to. 'earliest' will grab the earliest unprocessed message + # Timeout limit + consumer = KafkaConsumer('test', + client_id='python-consumer-%s' % (procID), + bootstrap_servers=['kafka-spotify:9092'], + auto_offset_reset='latest', + consumer_timeout_ms=1000) + + #Alternative way to subscribe to a topic + #consumer.subscribe(['test']) + + #loop until the thread is stopped by checking the stop event + while not self.stop_event.is_set(): + #Loop through ConsumerRecord objects in the consumer object + for message in consumer: + #print the messages to the screen with a note of the thread/client ID + #print("python-consumer-%s processed message: %s" % (procID, message)) + #print the messages to the screen with a note of the thread/client ID, Current Topic, message number, # The value of the message decoded as it is sent as bytecode + print ("python-consumer-%s processed message: %s:%d: value=%s" % (procID, message.topic, + message.offset, message.value.decode('utf-8'))) + #break out of the for loop if the thread was notified of closure + if self.stop_event.is_set(): + break + + #Close the TCP connection to kafka + consumer.close() + + +#Main function called when the app is run +def main(): + #initialize a Consumer object/thread + kafkConsumer = Consumer() + + #Start the thread working. + kafkConsumer.start() + + #sleep for 17 second. If we weren't using threads this would halt the code + time.sleep(20) + + #Call stop to set the thread event so it knows to stop + print("Stopping kafkConsumer") + kafkConsumer.stop() + + #Wait until the thread terminates. Can see the docs for more + #https://docs.python.org/2/library/threading.html?highlight=thread#threading.Thread.join + print("Waiting for execution to halt") + kafkConsumer.join() + +#the logic to run as process +if __name__ == "__main__": + #Set logging format and level + #logging.basicConfig( + # format='%(asctime)s.%(msecs)s:%(name)s:%(thread)d:%(levelname)s:%(process)d:%(message)s', + # level=logging.INFO) + + #Call the main function + main() + diff --git a/kafkaProducerService.py b/kafkaProducerService.py index 5d10fa7..4c4b7c6 100644 --- a/kafkaProducerService.py +++ b/kafkaProducerService.py @@ -34,7 +34,7 @@ def run(self): while not self.stop_event.is_set(): #Send two messages of type binary to the 'test' Topic producer.send('test', b"test") - producer.send('test', b"\xc2Hola, mundo!") + producer.send('test', b"Hola, mundo!") #Sleep for 3 seconds time.sleep(3) From b692bce7c2bdf73a1ded733f5d05dfe6f0f9bb36 Mon Sep 17 00:00:00 2001 From: myersr Date: Fri, 4 May 2018 16:49:51 -0400 Subject: [PATCH 097/109] added consumer script --- kafkaConsumerService.py | 96 +++++++++++++++++++++++++++++++++++++++++ kafkaProducerService.py | 2 +- 2 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 kafkaConsumerService.py diff --git a/kafkaConsumerService.py b/kafkaConsumerService.py new file mode 100644 index 0000000..a0580eb --- /dev/null +++ b/kafkaConsumerService.py @@ -0,0 +1,96 @@ +import threading, logging, time +import multiprocessing +import os + +from kafka import KafkaConsumer + + +#Thread class that continuously consumes messages from a Kafka message queue/topic +# Input: +# threading.Thread = instance of multiproccessing thread +class Consumer(multiprocessing.Process): + #default initializing function of the thread. + # Input: + # takes self to modify and initialize + def __init__(self): + #initializes thread and passes self. Magic multithreading stuff + multiprocessing.Process.__init__(self) + #Gives the thread an envent called stop_event so it can be interrupted. + self.stop_event = multiprocessing.Event() + + #Function to stop the process + def stop(self): + #Calls even stop_event and sets it. + #This gives context to the thread from the outside and lets you stop it. + self.stop_event.set() + + #The main run function called when you call start. + def run(self): + if hasattr(os, 'getppid'): # only available on Unix + print 'parent process:', os.getppid() + procID = os.getppid() + #Bootstraps an instance of a Kafka producer. + #Initializes the producer and identifies the docker server. + #kafka-spotify is listed in /etc/hosts with the ip of the container + #Input: + # topic to subscribe to: 'test' + # Id to identify the consumer should be unique to the connection + # Servers kafka is advertising as + # Which message rule to subscribe to. 'earliest' will grab the earliest unprocessed message + # Timeout limit + consumer = KafkaConsumer('test', + client_id='python-consumer-%s' % (procID), + bootstrap_servers=['kafka-spotify:9092'], + auto_offset_reset='latest', + consumer_timeout_ms=1000) + + #Alternative way to subscribe to a topic + #consumer.subscribe(['test']) + + #loop until the thread is stopped by checking the stop event + while not self.stop_event.is_set(): + #Loop through ConsumerRecord objects in the consumer object + for message in consumer: + #print the messages to the screen with a note of the thread/client ID + #print("python-consumer-%s processed message: %s" % (procID, message)) + #print the messages to the screen with a note of the thread/client ID, Current Topic, message number, # The value of the message decoded as it is sent as bytecode + print ("python-consumer-%s processed message: %s:%d: value=%s" % (procID, message.topic, + message.offset, message.value.decode('utf-8'))) + #break out of the for loop if the thread was notified of closure + if self.stop_event.is_set(): + break + + #Close the TCP connection to kafka + consumer.close() + + +#Main function called when the app is run +def main(): + #initialize a Consumer object/thread + kafkConsumer = Consumer() + + #Start the thread working. + kafkConsumer.start() + + #sleep for 17 second. If we weren't using threads this would halt the code + time.sleep(20) + + #Call stop to set the thread event so it knows to stop + print("Stopping kafkConsumer") + kafkConsumer.stop() + + #Wait until the thread terminates. Can see the docs for more + #https://docs.python.org/2/library/threading.html?highlight=thread#threading.Thread.join + print("Waiting for execution to halt") + kafkConsumer.join() + +#the logic to run as process +if __name__ == "__main__": + #Set logging format and level + #logging.basicConfig( + # format='%(asctime)s.%(msecs)s:%(name)s:%(thread)d:%(levelname)s:%(process)d:%(message)s', + # level=logging.INFO) + + #Call the main function + main() + diff --git a/kafkaProducerService.py b/kafkaProducerService.py index 5d10fa7..4c4b7c6 100644 --- a/kafkaProducerService.py +++ b/kafkaProducerService.py @@ -34,7 +34,7 @@ def run(self): while not self.stop_event.is_set(): #Send two messages of type binary to the 'test' Topic producer.send('test', b"test") - producer.send('test', b"\xc2Hola, mundo!") + producer.send('test', b"Hola, mundo!") #Sleep for 3 seconds time.sleep(3) From dc6962bf841d298dc3d643edc59141b99b38d160 Mon Sep 17 00:00:00 2001 From: myersr Date: Fri, 4 May 2018 16:55:33 -0400 Subject: [PATCH 098/109] added stuff --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c009d52..7de4f24 100644 --- a/README.md +++ b/README.md @@ -85,3 +85,4 @@ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file Test if Zookeeper has registered a broker: `echo dump | nc localhost 2181 | grep brokers` and look for an ID To run Spotify container `docker run -p 2181:2181 -p 9092:9092 --env ADVERTISED_HOST=kafka-spotify --env ADVERTISED_PORT=9092 --name kafka-spotify spotify/kafka` instead of the supplied command as I don't have docker-machine. +Please Reach-out From 71823beb8ffb2cac9d64b4446e806e9da4c725fe Mon Sep 17 00:00:00 2001 From: myersr Date: Fri, 4 May 2018 16:55:33 -0400 Subject: [PATCH 099/109] added stuff --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c009d52..7de4f24 100644 --- a/README.md +++ b/README.md @@ -85,3 +85,4 @@ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file Test if Zookeeper has registered a broker: `echo dump | nc localhost 2181 | grep brokers` and look for an ID To run Spotify container `docker run -p 2181:2181 -p 9092:9092 --env ADVERTISED_HOST=kafka-spotify --env ADVERTISED_PORT=9092 --name kafka-spotify spotify/kafka` instead of the supplied command as I don't have docker-machine. +Please Reach-out From a29623f34f37d1a71c4032519f9b8384bb790b0d Mon Sep 17 00:00:00 2001 From: myersr Date: Fri, 4 May 2018 16:55:33 -0400 Subject: [PATCH 100/109] added stuff --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c009d52..7de4f24 100644 --- a/README.md +++ b/README.md @@ -85,3 +85,4 @@ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file Test if Zookeeper has registered a broker: `echo dump | nc localhost 2181 | grep brokers` and look for an ID To run Spotify container `docker run -p 2181:2181 -p 9092:9092 --env ADVERTISED_HOST=kafka-spotify --env ADVERTISED_PORT=9092 --name kafka-spotify spotify/kafka` instead of the supplied command as I don't have docker-machine. +Please Reach-out From cdc4e1a981ed1776f1a5471b30bef15a0967f359 Mon Sep 17 00:00:00 2001 From: myersr Date: Fri, 4 May 2018 16:55:33 -0400 Subject: [PATCH 101/109] added stuff --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c009d52..7de4f24 100644 --- a/README.md +++ b/README.md @@ -85,3 +85,4 @@ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file Test if Zookeeper has registered a broker: `echo dump | nc localhost 2181 | grep brokers` and look for an ID To run Spotify container `docker run -p 2181:2181 -p 9092:9092 --env ADVERTISED_HOST=kafka-spotify --env ADVERTISED_PORT=9092 --name kafka-spotify spotify/kafka` instead of the supplied command as I don't have docker-machine. +Please Reach-out From dfd66685040e1312942c60a25f51ec1c926ccb88 Mon Sep 17 00:00:00 2001 From: myersr Date: Fri, 4 May 2018 16:55:33 -0400 Subject: [PATCH 102/109] added stuff --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c009d52..7de4f24 100644 --- a/README.md +++ b/README.md @@ -85,3 +85,4 @@ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file Test if Zookeeper has registered a broker: `echo dump | nc localhost 2181 | grep brokers` and look for an ID To run Spotify container `docker run -p 2181:2181 -p 9092:9092 --env ADVERTISED_HOST=kafka-spotify --env ADVERTISED_PORT=9092 --name kafka-spotify spotify/kafka` instead of the supplied command as I don't have docker-machine. +Please Reach-out From eb4103cc0d528ff641bafaaf18c6d43f5b0b8058 Mon Sep 17 00:00:00 2001 From: myersr Date: Fri, 4 May 2018 16:55:33 -0400 Subject: [PATCH 103/109] added stuff --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c009d52..7de4f24 100644 --- a/README.md +++ b/README.md @@ -85,3 +85,4 @@ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file Test if Zookeeper has registered a broker: `echo dump | nc localhost 2181 | grep brokers` and look for an ID To run Spotify container `docker run -p 2181:2181 -p 9092:9092 --env ADVERTISED_HOST=kafka-spotify --env ADVERTISED_PORT=9092 --name kafka-spotify spotify/kafka` instead of the supplied command as I don't have docker-machine. +Please Reach-out From 762a853f26ae93a7a66f3df71f12e03f680ead7f Mon Sep 17 00:00:00 2001 From: myersr Date: Fri, 4 May 2018 16:55:33 -0400 Subject: [PATCH 104/109] added stuff --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c009d52..7de4f24 100644 --- a/README.md +++ b/README.md @@ -85,3 +85,4 @@ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file Test if Zookeeper has registered a broker: `echo dump | nc localhost 2181 | grep brokers` and look for an ID To run Spotify container `docker run -p 2181:2181 -p 9092:9092 --env ADVERTISED_HOST=kafka-spotify --env ADVERTISED_PORT=9092 --name kafka-spotify spotify/kafka` instead of the supplied command as I don't have docker-machine. +Please Reach-out From 30034e355ec75682f22187070dec564ce0520059 Mon Sep 17 00:00:00 2001 From: myersr Date: Fri, 4 May 2018 16:55:33 -0400 Subject: [PATCH 105/109] added stuff --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c009d52..7de4f24 100644 --- a/README.md +++ b/README.md @@ -85,3 +85,4 @@ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file Test if Zookeeper has registered a broker: `echo dump | nc localhost 2181 | grep brokers` and look for an ID To run Spotify container `docker run -p 2181:2181 -p 9092:9092 --env ADVERTISED_HOST=kafka-spotify --env ADVERTISED_PORT=9092 --name kafka-spotify spotify/kafka` instead of the supplied command as I don't have docker-machine. +Please Reach-out From 8732a2bbcc9539fe716968f4ecf0f3b81d8435be Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Thu, 10 May 2018 13:02:37 -0400 Subject: [PATCH 106/109] Expanded readme to include Consumer and better intro P --- README.md | 74 ++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 48 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 569eb77..010b6c5 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,35 @@ # Simple kafka/flask server. -Tutorial to come and be linked. - -# Project Title - -A couple examples of a python app that makes use of Kafka in a docker container, Kafka-python python library, and Flask as a webserver. The intent is to have several branches to cover multiple different scenarios utilizing Kafka. - * [kafkaProducerService.py](kafkaProducerService.py) - * Currently a multi-threaded python script that shoots out messages to a kafka consumer. - * Loops for 17 seconds. - * Thread sends 2 messages to the kafka topic 'test' every three seconds. - * only spawns one thread that is a producer - * //TODO is change it to our specific use case for the tutorial. - * [server.py](server.py) - * A flask server - * accepts a post - * submits the post body to a kafka topic 'test' as JSON. + +A couple examples of a python app that makes use of Kafka in a docker container, Kafka-python python library, and Flask as a webserver. The intent is to have several branches to cover multiple different scenarios utilizing Kafka. + +This repository handles a couple Kafka use cases. In an incredibly simple overview: + * a __Producer__ sends a message to __Kafka__ + * __Kafka__ accepts the message and then looks for a consumer + * a __Consumer__ grabs the message from __Kafka__ to process + +One very loose metaphor for a kafka system could be subscribing to a magazine. The Company(Producer) prints the magazine. They have your address but don't know how to get to you. +They deliver a stack of magazines to UPS(Kafka). UPS has no knowledge/use of what is in that magazine, but they know how to get it to the reader's(Consumer) address. UPS is in charge of delivering, tracking, and keeping the magazine safe. +The reader(Consumer) has a mailbox with his address on the side therefore UPS drops off the magazine. + +There are a couple of things to note here. + * Kafka messages are organized by topics + * A _consumer_ and _producer_ both read/write to the same topic + * You can think of a topic like an address in the above scenario. It could be a business with multiple people working at the same location or it could be a single home with 1 occupant. + * Kafka does a ton of stuff in the background that I will not get into in this repo. It handles replication and logging and the likes. The [Official Docs](https://kafka.apache.org/) are a good resource if you would like to learn more. + +This repo provides a basis for 2 different __Producer__ scenarios. + 1. A running python Flask server that accepts a POST request and then opens a connection to Kafka. The server sends the POST body to kafka as JSON and closes the connection after the message is sent. + * [server.py](server.py) + 2. A python script that spawns a thread that opens a connection to Kafka and sends two messages every couple seconds for 17 seconds. + * [kafkaProducerService.py](kafkaProducerService.py) + +This repo provides a basis for 1 __Consumer__ scenario. + 1. A python script to spawn a thread that runs for 20 seconds and opens a connection to listen on a topic. The thread prints all messages to stdout then closes the connection after the 20 seconds. + * [kafkaConsumerService.py](kafkaConsumerService.py) + +This is not a tutorial for using/configuring Kafka. To make things simple, I use the spotify/kafka image that has all the needed functionality inside one container. This also nullifies a lot of the benefits that Kafka can provide. To make it easy, I have included scripts to get the container(s) up and running. The scripts have comments and I implore you to read the commands and be very familiar with what you are running. +//Tutorial to come and be linked. + ## Getting Started @@ -23,13 +39,13 @@ These instructions will get you a copy of the project up and running on your loc What things you need to install the software and how to install them. -These apps require a couple docker containers to run properly. The scripts should be all you need aside from Docker installed. -Only one of the containers is used for the functionality. The other running container is just used to repeat kafka messages and would be replaced in any kind of deployment. For those reasons I didn't create a docker-compose config. One is just a constant output feed. +These apps require a couple docker containers to run properly. The scripts should be all you need aside from Docker to handle the docker set-up. +Only one of the containers is necessary, the one running Kafka/Zookeeper. A script is provided to run a Kafka Consumer that prints kafka messages for testing/debugging. For these reasons I didn't create a docker-compose config. ``` -scripts/createTopicOnExistingKafka.sh - Runs command on the running Kafka container to create a topic. -scripts/listTopicOnExistingKafka.sh - Checks to make sure the topic was created. It runs a command on an existing Docker container -scripts/listenToKafkaTopic.sh - Spans a temporary docker container to consume from the Kafka topic and it prints the message on the screen -scripts/startMasterZookeeperContainer.sh - Creates a docker network. Starts a container in the background that runs zookeeper and Kafka in the same container. +scripts/createTopicOnExistingKafka.sh - Executes a command on the running Kafka container to create a topic. +scripts/listTopicOnExistingKafka.sh - Checks to make sure the topic was created. It runs a command on an existing Docker container and lists all topics. +scripts/listenToKafkaTopic.sh - Spawns a temporary docker container to consume from the Kafka topic. It prints the messages to stdout and exits on interrupt. +scripts/startMasterZookeeperContainer.sh - Creates a docker network. Starts 1 container in the background that runs zookeeper and Kafka. This is the only necessary container and should be created first. ``` Order to run: @@ -42,28 +58,34 @@ You will end with a persistent container running Kafka and Zookeeper in the back ### Installing -A step by step series of examples that tell you have to get a development env running. With python Always suggest a virtual environment. +A step by step series of examples that tell you have to get a development env running. With python I Always suggest a virtual environment. +__The docker container must be running for the producers to connect.__ Install the python packages ``` pip install -r requirements.txt -``` +``` +To run the Consumer run +``` +python kafkaConsumerService.py #Remember this only runs for 20 seconds +``` -Then you can run the apps. +Then you can run the consumers. ``` -python server.py +python server.py #Runs indefinitally but only sends a message when it recieves a POST ``` or ``` -python kafkaProducerService.py +python kafkaProducerService.py #Remember this only runs for 17 seconds ``` + ## Authors * **Roy Myers** - *Initial work* - [rmyers19](https://github.optum.com/rmyers19) From 186bf4364a59cac52a0e65367fb625fc0f274a2d Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Thu, 10 May 2018 13:04:56 -0400 Subject: [PATCH 107/109] updating email --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 010b6c5..0d2b29b 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ python kafkaProducerService.py #Remember this only runs for 17 seconds ## Authors -* **Roy Myers** - *Initial work* - [rmyers19](https://github.optum.com/rmyers19) +* **Roy Myers** - *Initial work* - [myersr](https://github.com/myersr) ## License From be4904cce632b109b07fbf25173baf3661a66f85 Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Thu, 10 May 2018 15:10:40 -0400 Subject: [PATCH 108/109] spacing --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0d2b29b..7c8fda2 100644 --- a/README.md +++ b/README.md @@ -19,9 +19,9 @@ There are a couple of things to note here. This repo provides a basis for 2 different __Producer__ scenarios. 1. A running python Flask server that accepts a POST request and then opens a connection to Kafka. The server sends the POST body to kafka as JSON and closes the connection after the message is sent. - * [server.py](server.py) + * [server.py](server.py) 2. A python script that spawns a thread that opens a connection to Kafka and sends two messages every couple seconds for 17 seconds. - * [kafkaProducerService.py](kafkaProducerService.py) + * [kafkaProducerService.py](kafkaProducerService.py) This repo provides a basis for 1 __Consumer__ scenario. 1. A python script to spawn a thread that runs for 20 seconds and opens a connection to listen on a topic. The thread prints all messages to stdout then closes the connection after the 20 seconds. From e48cfd2580e0f62c20961bc43cba26072a5ce49b Mon Sep 17 00:00:00 2001 From: Roy Myers Date: Thu, 10 May 2018 15:17:35 -0400 Subject: [PATCH 109/109] MD syntax --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7c8fda2..e0a7331 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,9 @@ This repository handles a couple Kafka use cases. In an incredibly simple overvi * a __Consumer__ grabs the message from __Kafka__ to process One very loose metaphor for a kafka system could be subscribing to a magazine. The Company(Producer) prints the magazine. They have your address but don't know how to get to you. -They deliver a stack of magazines to UPS(Kafka). UPS has no knowledge/use of what is in that magazine, but they know how to get it to the reader's(Consumer) address. UPS is in charge of delivering, tracking, and keeping the magazine safe. + +The company delivers a stack of magazines to UPS(Kafka). UPS has no knowledge/use of what is in that magazine, but they know how to get it to the reader's(Consumer) address. UPS is in charge of delivering, tracking, and keeping the magazine safe. + The reader(Consumer) has a mailbox with his address on the side therefore UPS drops off the magazine. There are a couple of things to note here. @@ -25,7 +27,7 @@ This repo provides a basis for 2 different __Producer__ scenarios. This repo provides a basis for 1 __Consumer__ scenario. 1. A python script to spawn a thread that runs for 20 seconds and opens a connection to listen on a topic. The thread prints all messages to stdout then closes the connection after the 20 seconds. - * [kafkaConsumerService.py](kafkaConsumerService.py) + * [kafkaConsumerService.py](kafkaConsumerService.py) This is not a tutorial for using/configuring Kafka. To make things simple, I use the spotify/kafka image that has all the needed functionality inside one container. This also nullifies a lot of the benefits that Kafka can provide. To make it easy, I have included scripts to get the container(s) up and running. The scripts have comments and I implore you to read the commands and be very familiar with what you are running. //Tutorial to come and be linked. @@ -56,7 +58,7 @@ Order to run: You will end with a persistent container running Kafka and Zookeeper in the background, a container printing out to the terminal relaying messages to a Kafka topic. -### Installing +## Installing A step by step series of examples that tell you have to get a development env running. With python I Always suggest a virtual environment.