Add 1.12 folder for MongoDB / MQTT IoT Demo (#81)

* Update links for 1.12
* Update MongoDB connection string
* Update main README with link to MQTT IoT demo on 1.12

README.md

@@ -1,10 +1,11 @@
 Overview of demos in 1.12:
 * [Apache Flink Stream Processing](flink/1.12#fast-data-financial-transaction-processing-with-apache-flink)
+* [Highly Scalable IOT architecture using MQTT and MongoDB](mqtt-iot/1.12#highly-scalable-iot-architecture-using-mosquitto-and-mongodb)
 
 Overview of demos in 1.11:
 * [Apache Flink Stream Processing](flink/1.11#fast-data-financial-transaction-processing-with-apache-flink)
 * [Apache Flink Stream Processing (Kubernetes Edition)](flink-k8s/1.11#fast-data-financial-transaction-processing-with-apache-flink)
-*  [Iot Fast Data Analytics](fastdata-iot/1.11#iot-fast-data-analytics)
+* [Iot Fast Data Analytics](fastdata-iot/1.11#iot-fast-data-analytics)
 * [Continuous Delivery with Gitlab and Jenkins](cicd/1.11#continuous-delivery-with-gitlab-and-jenkins)
 * [Highly Scalable IOT architecture using MQTT and MongoDB](mqtt-iot/1.11#highly-scalable-iot-architecture-using-mosquitto-and-mongodb)
 

mqtt-iot/1.12/demo.json

@@ -0,0 +1,13 @@
+{
+  "mongodb-credentials": {
+    "backupUser": "backup",
+    "backupPassword": "backupuserpassword",
+    "userAdminUser": "useradmin",
+    "userAdminPassword": "useradminpassword",
+    "clusterAdminUser": "clusteradmin",
+    "clusterAdminPassword": "clusteradminpassword",
+    "clusterMonitorUser": "clustermonitor",
+    "clusterMonitorPassword": "monitoruserpassword",
+    "key": "8cNNTVP6GqEOKzhUVDVryxIt04K6kDbXygamH4upPGAO59gzXVQAgX9NwxwqDvpt 094zMkkRWDLzuCgbg3Aj8EFVEM0/W1Nz+XUSTHEn4HiNzCVG4TTHFP6P1PEPswG6 tQMP6bnRXL7uGWmdGhbAxOV/+p6AfNs67MTvfCeH0EaPCgPPXhJft9D0nZ0SPOm9 VvfxG3djnHClIlclkchoIwc1Kw21loyXwuOjX4RkywVDdmFXjKC+l9yxfyt/9Gyh YE0OlS7ozWLiH8zy0MyzBdK+rc0fsxb2/Kb/8/2diC3O3gdVxjneQxaf66+FHVNW mV9/IHDptBHosdWkv0GboW8ZnTXnk0lyY0Jw85JFuTeFBzqPlB37jR0NU/HFm5QT Ld62woaGIWCTuXGb81QHaglPZUBIhEq/b3tahJBmLc+LKd0FUShoupTtPc2FjxbH xD8dZ+L9Uv7NPtSe+o3sTD60Pnsw1wbOrNDrrC+wpwoMy2GbQjXk/d+SRK/CXfuk Z676GKQDivpinhdF58l4OEi+WEN633yuNtNAQDgz+aOVZKN4oLoyR22B1nrea1qW wzZjRw7kpVxcQKiyn+gDmAZZPbctiVqTNHPE5n9LrOcctuLZKpoQk97lvZTSCKfy d32mfx9szZZ/QCfF9Dt7+G5nJUAULigKnQYRi/i86ZTPHSzfun+ZIzYLCzJuZfyS 7E8DMsmv9wCPrPAF/8cOFMWW0o0Na7GZKCJ8U+AMm92R725h4g5ao6+kQPG7vOkY LR8MJzDOqcmAC0M9AwE5UXQl56V6qBNyREx/WGGYS1B5DOfZvVTJNDkoHVIL1upZ geSlACiXQ+M0Rkgo0h8BJUhGY9LTuc6S8qiMBEnhBClg4kA/u4FJ06nlmF3ZpIXT KsVSr9ee3mu0vSr6P52slvAAX+RL3y+JgSlz2kC8oVgCZZdKn7yq9e6yB3zHNMjX 8VIi/UgFmfqCiaAlUT0pt2ZzGuw1L9QUOuNAZfufSkK1ED4V"
+  }
+}

mqtt-iot/1.12/device/Dockerfile

@@ -0,0 +1,10 @@
+FROM python:2
+
+WORKDIR /usr/src/app
+
+COPY requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY device.py .
+
+CMD [ "/bin/bash" ]

mqtt-iot/1.12/device/device.json

@@ -0,0 +1,16 @@
+{
+  "id": "device",
+  "instances": 1,
+  "cpus": 0.1,
+  "mem": 16,
+  "cmd": "./device.py -b mqtt.marathon.l4lb.thisdcos.directory -r 2",
+  "container": {
+    "type": "MESOS",
+    "docker": {
+      "image": "mattjarvis/device",
+      "forcePullImage": true,
+      "privileged": false
+    }
+  },
+  "requirePorts": false
+}

mqtt-iot/1.12/device/device.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+"""
+MQTT generator
+"""
+
+import random
+import time
+import uuid
+import json
+from argparse import ArgumentParser
+import paho.mqtt.client as mqtt
+
+parser = ArgumentParser()
+parser.add_argument("-b", "--broker", dest="broker_address",
+                    required=True, help="MQTT broker address")
+parser.add_argument("-p", "--port", dest="broker_port", default=1883, help="MQTT broker port")
+parser.add_argument("-r", "--rate", dest="sample_rate", default=5, help="Sample rate")
+parser.add_argument("-q", "--qos", dest="qos", default=0, help="MQTT QOS")
+args = parser.parse_args()
+
+uuid = str(uuid.uuid4())
+topic = "device/%s" % uuid
+
+mqttc = mqtt.Client(uuid, False)
+mqttc.connect(args.broker_address, args.broker_port)
+while True:
+    rand = random.randint(20,30)
+    msg = {
+            'uuid': uuid,
+            'value': rand
+    }
+    mqttc.publish(topic, payload=json.dumps(msg), qos=args.qos)
+    time.sleep(float(args.sample_rate))
+mqttc.loop_forever()

mqtt-iot/1.12/device/requirements.txt

@@ -0,0 +1 @@
+paho-mqtt==1.3.1

mqtt-iot/1.12/mongogw/Dockerfile

@@ -0,0 +1,9 @@
+FROM python:2
+
+WORKDIR /usr/src/app
+
+COPY requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt
+COPY mongogw.py .
+
+CMD [ "/bin/bash" ]

mqtt-iot/1.12/mongogw/mongogw.json

@@ -0,0 +1,16 @@
+{
+  "id": "mongogw",
+  "instances": 1,
+  "cpus": 0.1,
+  "mem": 64,
+  "cmd": "./mongogw.py -b mosquitto.marathon.l4lb.thisdcos.directory -m mongo-rs-0-mongod.percona-server-mongodb.autoip.dcos.thisdcos.directory,mongo-rs-1-mongod.percona-server-mongodb.autoip.dcos.thisdcos.directory,mongo-rs-2-mongod.percona-server-mongodb.autoip.dcos.thisdcos.directory:27017 -u mongogw -w 123456",
+  "container": {
+    "type": "MESOS",
+    "docker": {
+      "image": "mattjarvis/mongogw",
+      "forcePullImage": true,
+      "privileged": false
+    }
+  },
+  "requirePorts": false
+}

mqtt-iot/1.12/mongogw/mongogw.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+"""
+MQTT to MongoDB Gateway
+"""
+
+import json
+from argparse import ArgumentParser
+import paho.mqtt.client as mqtt
+import pymongo
+import datetime
+import os
+
+parser = ArgumentParser()
+parser.add_argument("-b", "--broker", dest="broker_address",
+                    required=True, help="MQTT broker address")
+parser.add_argument("-p", "--port", dest="broker_port", default=1883, help="MQTT broker port")
+parser.add_argument("-m", "--mongouri", dest="mongo_uri", required=True, help="MongoDB URI")
+parser.add_argument("-u", "--mongouser", dest="mongo_user", required=True, help="MongoDB user")
+parser.add_argument("-w", "--mongopwd", dest="mongo_password", required=True, help="MongoDB password")
+args = parser.parse_args()
+
+def on_message(client, userdata, message):
+    json_data = json.loads(message.payload)
+    post_data = {
+        'date': datetime.datetime.utcnow(),
+        'deviceUID': json_data['uuid'],
+        'value': json_data['value'],
+        'gatewayID': os.environ['MESOS_TASK_ID']
+    }
+    result = devices.insert_one(post_data)
+    #print('One device: {0}'.format(result.inserted_id))
+
+# MongoDB connection
+mongo_client = pymongo.MongoClient(args.mongo_uri,
+                                   username=args.mongo_user,
+                                   password=args.mongo_password,
+                                   authSource='mongogw',
+                                   authMechanism='SCRAM-SHA-1')
+db = mongo_client.mongogw
+devices = db.devices
+
+# MQTT connection
+mqttc = mqtt.Client("mongogw", False)
+mqttc.on_message=on_message
+mqttc.connect(args.broker_address, args.broker_port)
+mqttc.subscribe("device/#", qos=0)
+mqttc.loop_forever()

mqtt-iot/1.12/mongogw/requirements.txt

@@ -0,0 +1,2 @@
+paho-mqtt==1.3.1
+pymongo==3.7.0

mqtt-iot/1.12/mongouser.json

@@ -0,0 +1,7 @@
+{
+  "user": "mongogw",
+  "pwd": "123456",
+  "roles": [
+    { "db": "mongogw", "role": "readWrite" }
+  ]
+}

mqtt-iot/1.12/mqttpod.json

@@ -0,0 +1,62 @@
+{
+  "id": "/mqtt",
+  "containers": [
+    {
+      "name": "mosquitto",
+      "resources": {
+        "cpus": 0.1,
+        "mem": 64
+      },
+      "image": {
+        "id": "eclipse-mosquitto",
+        "kind": "DOCKER"
+      },
+      "endpoints": [
+        {
+          "name": "mqtt",
+          "containerPort": 1883,
+          "hostPort": 1883,
+          "protocol": [
+            "tcp"
+          ],
+          "labels": {
+            "VIP_0": "/mqtt:1883"
+          }
+        }
+      ]
+    },
+    {
+      "name": "mongogw",
+      "resources": {
+        "cpus": 0.1,
+        "mem": 64
+      },
+      "image": {
+        "id": "mattjarvis/mongogw",
+        "kind": "DOCKER"
+      },
+      "exec": {
+        "command": {
+          "shell": "./mongogw.py -b localhost -m mongo-rs-0-mongod.percona-server-mongodb.autoip.dcos.thisdcos.directory,mongo-rs-1-mongod.percona-server-mongodb.autoip.dcos.thisdcos.directory,mongo-rs-2-mongod.percona-server-mongodb.autoip.dcos.thisdcos.directory:27017 -u mongogw -w 123456"
+        }
+      }
+    }
+  ],
+  "scaling": {
+    "instances": 1,
+    "kind": "fixed"
+  },
+  "networks": [
+    {
+      "name": "dcos",
+      "mode": "container"
+    }
+  ],
+  "volumes": [],
+  "fetch": [],
+  "scheduling": {
+    "placement": {
+      "constraints": []
+    }
+  }
+}