diff --git a/driver/pom.xml b/driver/pom.xml index 014ae8f1bb..ea98270cd4 100644 --- a/driver/pom.xml +++ b/driver/pom.xml @@ -33,6 +33,12 @@ com.github.fge json-schema-validator + + + javax.mail + mailapi + + org.scalatest @@ -66,6 +72,12 @@ spark-sql_${scala.binary.version} provided + + javax.mail + mailapi + 1.4.3 + test + org.apache.spark spark-streaming_${scala.binary.version} diff --git a/examples/data-generators/.gitignore b/examples/data-generators/.gitignore deleted file mode 100644 index 34e1547a39..0000000000 --- a/examples/data-generators/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -target -.idea -*.iml \ No newline at end of file diff --git a/examples/data-generators/ecommerce/ecommerce-policy.json b/examples/data-generators/ecommerce/ecommerce-policy.json deleted file mode 100644 index 6da34b4b9b..0000000000 --- a/examples/data-generators/ecommerce/ecommerce-policy.json +++ /dev/null @@ -1,399 +0,0 @@ -{ - "name": "ecommerce-policy", - "description": "Ecommerce example", - "sparkStreamingWindow": 6000, - "checkpointPath": "checkpoint", - "rawData": { - "enabled": "false", - "path": "myTestParquetPath" - }, - "input": - { - "name": "in", - "type": "RabbitMQ", - "configuration": { - "queue": "logsQueue", - "host": "localhost", - "port": 5672, - "exchangeName": "logsExchange", - "routingKeys": [{"routingKey": "webLogsRoute"},{"routingKey": "purchasesRoute"}] - } - } - , - "transformations": [ - { - "name": "morphline-parser", - "order": 1, - "type": "Morphlines", - "outputFields": [ - "purchaseDateOL", - "userOL", - "totalAmountOL", - "cityOL", - "postalCodeOL", - "stateOL", - "countryOL", - "customerTypeOL", - "paymentTypeOL", - "timestampVL", - "ipHostVL", - "userAgentVL", - "userLogVL", - "responseCodeVL", - "operatingSystemVL" - ], - "configuration": { - "morphline": { - "id": "morphline1", - "importCommands": [ - "org.kitesdk.**" - ], - "commands": [ - { - "readJson": {} - }, - { - "extractJsonPaths": { - "paths": { - "purchaseDateOL": "/purchaseDate", - "userOL": "/user", - "totalAmountOL": "/totalAmount", - "cityOL": "/city", - "postalCodeOL": "/postalCode", - "stateOL": "/state", - "countryOL": "/country", - "customerTypeOL": "/customerType", - "paymentTypeOL": "/paymentType", - "timestampVL": "/timestamp", - "ipHostVL": "/ipHost", - "userAgentVL": "/userAgent", - "userLogVL": "/userLog", - "responseCodeVL": "/responseCode", - "operatingSystemVL": "/operatingSystem" - } - } - }, - { - "removeFields": { - "blacklist": [ - "literal:_attachment_body" - ] - } - } - ] - } - } - }, - { - "name": "purchaseDateOL", - "order": 2, - "inputField": "purchaseDateOL", - "outputFields": [ - "purchaseDateOL" - ], - "type": "DateTime", - "configuration": { - "inputFormat": "unixMillis" - } - }, - { - "name": "timestampVL", - "order": 3, - "inputField": "timestampVL", - "outputFields": [ - "timestampVL" - ], - "type": "DateTime", - "configuration": { - "inputFormat": "unixMillis" - } - } - ], - "cubes": [ - { - "name": "testCube", - "checkpointConfig": { - "timeDimension": "minute", - "granularity": "minute", - "interval": 30000, - "timeAvailability": 60000 - }, - "writer": { - "isAutoCalculatedId": true - }, - "dimensions": [ - { - "name": "purchaseDateOL", - "field": "purchaseDateOL", - "precision": "minute", - "type": "DateTime" - } - ], - "operators": [ - { - "name": "countoperator", - "type": "Count", - "configuration": {} - }, - { - "name": "avgoperator", - "type": "Avg", - "configuration": { - "inputField": "totalAmountOL" - } - }, - { - "name": "sumoperator", - "type": "Sum", - "configuration": { - "inputField": "totalAmountOL" - } - }, - { - "name": "maxoperator", - "type": "Max", - "configuration": { - "inputField": "totalAmountOL" - } - }, - { - "name": "minoperator", - "type": "Min", - "configuration": { - "inputField": "totalAmountOL" - } - } - ] - }, - { - "name": "testCube1", - "checkpointConfig": { - "timeDimension": "minute", - "granularity": "minute", - "interval": 30000, - "timeAvailability": 60000 - }, - "writer": { - "isAutoCalculatedId": true - }, - "dimensions": [ - { - "name": "timestampVL", - "field": "timestampVL", - "type": "DateTime", - "precision": "minute" - } - ], - "operators": [ - { - "name": "countoperator", - "type": "Count", - "configuration": {} - }, - { - "name": "avgoperator", - "type": "Avg", - "configuration": { - "inputField": "totalAmountOL" - } - }, - { - "name": "sumoperator", - "type": "Sum", - "configuration": { - "inputField": "totalAmountOL" - } - }, - { - "name": "maxoperator", - "type": "Max", - "configuration": { - "inputField": "totalAmountOL" - } - }, - { - "name": "minoperator", - "type": "Min", - "configuration": { - "inputField": "totalAmountOL" - } - } - ] - }, - { - "name": "testCube2", - "checkpointConfig": { - "timeDimension": "minute", - "granularity": "minute", - "interval": 30000, - "timeAvailability": 60000 - }, - "writer": { - "isAutoCalculatedId": true - }, - "dimensions": [ - { - "name": "ipHostVL", - "field": "ipHostVL" - }, - { - "name": "timestampVL", - "field": "timestampVL", - "type": "DateTime", - "precision": "minute" - } - ], - "operators": [ - { - "name": "countoperator", - "type": "Count", - "configuration": {} - }, - { - "name": "avgoperator", - "type": "Avg", - "configuration": { - "inputField": "totalAmountOL" - } - }, - { - "name": "sumoperator", - "type": "Sum", - "configuration": { - "inputField": "totalAmountOL" - } - }, - { - "name": "maxoperator", - "type": "Max", - "configuration": { - "inputField": "totalAmountOL" - } - }, - { - "name": "minoperator", - "type": "Min", - "configuration": { - "inputField": "totalAmountOL" - } - } - ] - }, - { - "name": "testCube3", - "checkpointConfig": { - "timeDimension": "minute", - "granularity": "minute", - "interval": 30000, - "timeAvailability": 60000 - }, - "writer": { - "isAutoCalculatedId": true - }, - "dimensions": [ - { - "name": "cityOL", - "field": "cityOL" - } - ], - "operators": [ - { - "name": "countoperator", - "type": "Count", - "configuration": {} - }, - { - "name": "avgoperator", - "type": "Avg", - "configuration": { - "inputField": "totalAmountOL" - } - }, - { - "name": "sumoperator", - "type": "Sum", - "configuration": { - "inputField": "totalAmountOL" - } - }, - { - "name": "maxoperator", - "type": "Max", - "configuration": { - "inputField": "totalAmountOL" - } - }, - { - "name": "minoperator", - "type": "Min", - "configuration": { - "inputField": "totalAmountOL" - } - } - ] - }, - { - "name": "testCube4", - "checkpointConfig": { - "timeDimension": "minute", - "granularity": "minute", - "interval": 30000, - "timeAvailability": 60000 - }, - "writer": { - "isAutoCalculatedId": true - }, - "dimensions": [ - { - "name": "stateOL", - "field": "stateOL" - } - ], - "operators": [ - { - "name": "countoperator", - "type": "Count", - "configuration": {} - }, - { - "name": "avgoperator", - "type": "Avg", - "configuration": { - "inputField": "totalAmountOL" - } - }, - { - "name": "sumoperator", - "type": "Sum", - "configuration": { - "inputField": "totalAmountOL" - } - }, - { - "name": "maxoperator", - "type": "Max", - "configuration": { - "inputField": "totalAmountOL" - } - }, - { - "name": "minoperator", - "type": "Min", - "configuration": { - "inputField": "totalAmountOL" - } - } - ] - } - ], - "outputs": [ - { - "name": "out-elasticsearch", - "type": "ElasticSearch", - "configuration": { - "nodes": [{"node":"localhost","defaultPort":"9200"}], - "indexMapping": "day" - } - } - ] -} \ No newline at end of file diff --git a/examples/data-generators/ecommerce/pom.xml b/examples/data-generators/ecommerce/pom.xml deleted file mode 100644 index 20ba4181a7..0000000000 --- a/examples/data-generators/ecommerce/pom.xml +++ /dev/null @@ -1,92 +0,0 @@ - - - - sparkta - com.stratio.examples - 1.0.0-SNAPSHOT - - 4.0.0 - - ecommerce - - - - com.rabbitmq - amqp-client - 3.5.1 - - - com.google.code.gson - gson - 2.3 - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.3 - - 1.7 - 1.7 - - - - - - - - orderLines - - - - org.databene - maven-benerator-plugin - 0.7.7 - - src/main/resources/benerator-purchases.xml - UTF-8 - - - - - - - visitLog - - - - org.databene - maven-benerator-plugin - 0.7.7 - - src/main/resources/benerator-visitLog.xml - UTF-8 - - - - - - - - \ No newline at end of file diff --git a/examples/data-generators/ecommerce/src/main/java/com/stratio/examples/datagenerator/events/Event.java b/examples/data-generators/ecommerce/src/main/java/com/stratio/examples/datagenerator/events/Event.java deleted file mode 100644 index 9c66569297..0000000000 --- a/examples/data-generators/ecommerce/src/main/java/com/stratio/examples/datagenerator/events/Event.java +++ /dev/null @@ -1,21 +0,0 @@ -/** - * Copyright (C) 2015 Stratio (http://stratio.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.stratio.examples.datagenerator.events; - -public interface Event { - String toJsonOutput(); -} diff --git a/examples/data-generators/ecommerce/src/main/java/com/stratio/examples/datagenerator/events/PurchaseEvent.java b/examples/data-generators/ecommerce/src/main/java/com/stratio/examples/datagenerator/events/PurchaseEvent.java deleted file mode 100644 index 8a57df8e48..0000000000 --- a/examples/data-generators/ecommerce/src/main/java/com/stratio/examples/datagenerator/events/PurchaseEvent.java +++ /dev/null @@ -1,119 +0,0 @@ -/** - * Copyright (C) 2015 Stratio (http://stratio.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.stratio.examples.datagenerator.events; - -import com.google.gson.Gson; -import com.google.gson.JsonObject; - -public class PurchaseEvent implements Event{ - - private final long purchaseDate; - private final String user; - private final float totalAmount; - private final String city; - private final String postalCode; - private final String state; - private final String country; - private final String customerType; - private final String paymentType; - - public PurchaseEvent(long purchaseDate, String user, float totalAmount, String city, String postalCode, String state, String country, String customerType, String paymentType) { - this.purchaseDate = purchaseDate; - this.user = user; - this.totalAmount = totalAmount; - this.city = city; - this.postalCode = postalCode; - this.state = state; - this.country = country; - this.customerType = customerType; - this.paymentType = paymentType; - } - - public String toJsonOutput(){ - - Gson gson = new Gson(); - JsonObject eventObject = new JsonObject(); - eventObject.addProperty("purchaseDate", purchaseDate); - eventObject.addProperty("user", user); - eventObject.addProperty("totalAmount", totalAmount); - eventObject.addProperty("city", city); - eventObject.addProperty("postalCode", postalCode); - eventObject.addProperty("state", state); - eventObject.addProperty("country", country); - eventObject.addProperty("customerType", customerType); - eventObject.addProperty("paymentType", paymentType); - return gson.toJson(eventObject); - } - - public static PurchaseEvent getInstance(Object[] args) { - long purchaseDate = Long.parseLong(args[0].toString()); - String user = args[1].toString(); - float totalAmount = Float.parseFloat(args[2].toString()); - String city = args[3].toString(); - String postalCode = args[4].toString(); - String state = args[5].toString(); - String country = args[6].toString(); - String customerType = args[7].toString(); - String paymentType = args[8].toString(); - - return new PurchaseEvent(purchaseDate, - user, - totalAmount, - city, - postalCode, - state, - country, - customerType, - paymentType); - } - - public long getPurchaseDate() { - return purchaseDate; - } - - public String getUser() { - return user; - } - - public float getTotalAmount() { - return totalAmount; - } - - public String getCity() { - return city; - } - - public String getPostalCode() { - return postalCode; - } - - public String getState() { - return state; - } - - public String getCountry() { - return country; - } - - public String getCustomerType() { - return customerType; - } - - public String getPaymentType() { - return paymentType; - } -} diff --git a/examples/data-generators/ecommerce/src/main/java/com/stratio/examples/datagenerator/events/VisitLogEvent.java b/examples/data-generators/ecommerce/src/main/java/com/stratio/examples/datagenerator/events/VisitLogEvent.java deleted file mode 100644 index 8dc22db9de..0000000000 --- a/examples/data-generators/ecommerce/src/main/java/com/stratio/examples/datagenerator/events/VisitLogEvent.java +++ /dev/null @@ -1,86 +0,0 @@ -/** - * Copyright (C) 2015 Stratio (http://stratio.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.stratio.examples.datagenerator.events; - -import com.google.gson.Gson; -import com.google.gson.JsonObject; - -public class VisitLogEvent implements Event { - - private final long timestamp; - private final String ipHost; - private final String userAgent; - private final String userLog; - private final String responseCode; - private final String operatingSystem; - - public VisitLogEvent(long timestamp, String ipHost, String userAgent, String userLog, String responseCode, String operatingSystem) { - this.timestamp = timestamp; - this.ipHost = ipHost; - this.userAgent = userAgent; - this.userLog = userLog; - this.responseCode = responseCode; - this.operatingSystem = operatingSystem; - } - - public static VisitLogEvent getInstance(Object[] args) { - long timestamp = Long.parseLong(args[0].toString()); - String ipHost = args[1].toString(); - String userAgent = args[2].toString(); - String user = args[3].toString(); - String responseCode = args[4].toString(); - String operatingSystem = args[5].toString(); - return new VisitLogEvent(timestamp, ipHost, userAgent, user, responseCode, operatingSystem); - } - - public String toJsonOutput() { - Gson gson = new Gson(); - JsonObject eventObject = new JsonObject(); - eventObject.addProperty("timestamp", timestamp); - eventObject.addProperty("ipHost", ipHost); - eventObject.addProperty("userAgent", userAgent); - eventObject.addProperty("userLog", userLog); - eventObject.addProperty("responseCode", responseCode); - eventObject.addProperty("operatingSystem", operatingSystem); - - return gson.toJson(eventObject); - } - - public long getTimestamp() { - return timestamp; - } - - public String getIpHost() { - return ipHost; - } - - public String getUserAgent() { - return userAgent; - } - - public String getUserLog() { - return userLog; - } - - public String getResponseCode() { - return responseCode; - } - - public String getOperatingSystem() { - return operatingSystem; - } -} diff --git a/examples/data-generators/ecommerce/src/main/java/com/stratio/examples/datagenerator/producers/Producer.java b/examples/data-generators/ecommerce/src/main/java/com/stratio/examples/datagenerator/producers/Producer.java deleted file mode 100644 index 302cafb230..0000000000 --- a/examples/data-generators/ecommerce/src/main/java/com/stratio/examples/datagenerator/producers/Producer.java +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Copyright (C) 2015 Stratio (http://stratio.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.stratio.examples.datagenerator.producers; - -import com.rabbitmq.client.Channel; -import com.rabbitmq.client.Connection; -import com.rabbitmq.client.ConnectionFactory; -import com.stratio.examples.datagenerator.events.Event; - -import java.io.IOException; - -public abstract class Producer { - - protected final static String EXCHANGE_NAME = "logsExchange"; - protected static final int NUM_BATCH_MESSAGES = 100; - protected int numMessage = 0; - protected ConnectionFactory factory = new ConnectionFactory(); - protected Connection connection; - protected Channel channel; - - public Producer(String host, Integer port) { - factory.setVirtualHost("/"); - factory.setHost(host); - factory.setPort(port); - } - - public void write(Object[] args) throws IOException { - - Event event = getInstance(args); - - if (numMessage == 0) { - connection = factory.newConnection(); - channel = connection.createChannel(); - channel.exchangeDeclare(EXCHANGE_NAME, "direct"); - } - - channel.basicPublish(EXCHANGE_NAME, getRoutingKey(), - null, event.toJsonOutput().getBytes()); - - if (numMessage == NUM_BATCH_MESSAGES) { - channel.close(); - connection.close(); - numMessage = 0; - } else { - numMessage++; - } - } - - protected abstract Event getInstance(Object[] args); - - protected abstract String getRoutingKey(); -} diff --git a/examples/data-generators/ecommerce/src/main/java/com/stratio/examples/datagenerator/producers/PurchaseEventProducer.java b/examples/data-generators/ecommerce/src/main/java/com/stratio/examples/datagenerator/producers/PurchaseEventProducer.java deleted file mode 100644 index a1c3d3dcc1..0000000000 --- a/examples/data-generators/ecommerce/src/main/java/com/stratio/examples/datagenerator/producers/PurchaseEventProducer.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright (C) 2015 Stratio (http://stratio.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.stratio.examples.datagenerator.producers; - -import com.stratio.examples.datagenerator.events.Event; -import com.stratio.examples.datagenerator.events.PurchaseEvent; - -import java.io.IOException; - -public class PurchaseEventProducer extends Producer { - - private final static String ROUTING_KEY = "purchasesRoute"; - - public PurchaseEventProducer(String host, Integer port) throws IOException { - super(host, port); - } - - @Override - protected Event getInstance(Object[] args) { - return PurchaseEvent.getInstance(args); - } - - @Override - protected String getRoutingKey() { - return ROUTING_KEY; - } -} diff --git a/examples/data-generators/ecommerce/src/main/java/com/stratio/examples/datagenerator/producers/VisitLogProducer.java b/examples/data-generators/ecommerce/src/main/java/com/stratio/examples/datagenerator/producers/VisitLogProducer.java deleted file mode 100644 index 2edc7962ff..0000000000 --- a/examples/data-generators/ecommerce/src/main/java/com/stratio/examples/datagenerator/producers/VisitLogProducer.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright (C) 2015 Stratio (http://stratio.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.stratio.examples.datagenerator.producers; - -import com.stratio.examples.datagenerator.events.Event; -import com.stratio.examples.datagenerator.events.VisitLogEvent; - -import java.io.IOException; - -public class VisitLogProducer extends Producer{ - - private final static String ROUTING_KEY = "webLogsRoute"; - - public VisitLogProducer(String host, Integer port) throws IOException { - super(host, port); - } - - @Override - protected Event getInstance(Object[] args) { - return VisitLogEvent.getInstance(args); - } - - @Override - protected String getRoutingKey() { - return ROUTING_KEY; - } -} diff --git a/examples/data-generators/ecommerce/src/main/resources/benerator-purchases.xml b/examples/data-generators/ecommerce/src/main/resources/benerator-purchases.xml deleted file mode 100644 index 01fc3bbbdf..0000000000 --- a/examples/data-generators/ecommerce/src/main/resources/benerator-purchases.xml +++ /dev/null @@ -1,46 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/examples/data-generators/ecommerce/src/main/resources/benerator-visitLog.xml b/examples/data-generators/ecommerce/src/main/resources/benerator-visitLog.xml deleted file mode 100644 index 459d46e8ba..0000000000 --- a/examples/data-generators/ecommerce/src/main/resources/benerator-visitLog.xml +++ /dev/null @@ -1,43 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/examples/data-generators/pom.xml b/examples/data-generators/pom.xml deleted file mode 100644 index e1e1bdf367..0000000000 --- a/examples/data-generators/pom.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - 4.0.0 - - com.stratio.examples - sparkta - pom - 1.0.0-SNAPSHOT - - twitter-to-rabbit - ecommerce - - - - - \ No newline at end of file diff --git a/examples/data-generators/twitter-to-rabbit/pom.xml b/examples/data-generators/twitter-to-rabbit/pom.xml deleted file mode 100644 index 3f07443228..0000000000 --- a/examples/data-generators/twitter-to-rabbit/pom.xml +++ /dev/null @@ -1,68 +0,0 @@ - - - - com.stratio.examples - sparkta - 1.0.0-SNAPSHOT - - 4.0.0 - - twitter-to-rabbit - - - - org.twitter4j - twitter4j-core - 4.0.3 - - - org.twitter4j - twitter4j-async - 4.0.3 - - - org.twitter4j - twitter4j-stream - 4.0.3 - - - org.twitter4j - twitter4j-media-support - 4.0.3 - - - com.rabbitmq - amqp-client - 3.5.1 - - - org.codehaus.jackson - jackson-mapper-asl - 1.8.5 - - - org.slf4j - slf4j-api - 1.7.12 - - - - \ No newline at end of file diff --git a/examples/data-generators/twitter-to-rabbit/src/main/java/com/stratio/examples/twittertorabbit/ReceiverWithRouting.java b/examples/data-generators/twitter-to-rabbit/src/main/java/com/stratio/examples/twittertorabbit/ReceiverWithRouting.java deleted file mode 100644 index e881dd2bbc..0000000000 --- a/examples/data-generators/twitter-to-rabbit/src/main/java/com/stratio/examples/twittertorabbit/ReceiverWithRouting.java +++ /dev/null @@ -1,45 +0,0 @@ -package com.stratio.examples.twittertorabbit; - -import com.rabbitmq.client.Channel; -import com.rabbitmq.client.Connection; -import com.rabbitmq.client.ConnectionFactory; -import com.rabbitmq.client.QueueingConsumer; - -public class ReceiverWithRouting extends TwitterToRabbitMQ { - - /** Pass routingKeys as argument */ - public static void main(String[] argv) - throws java.io.IOException, - java.lang.InterruptedException { - - ConnectionFactory factory = new ConnectionFactory(); - factory.setHost(HOST); - Connection connection = factory.newConnection(); - Channel channel = connection.createChannel(); - - channel.exchangeDeclare(EXCHANGE_NAME, "direct"); - String queueName = channel.queueDeclare().getQueue(); - - if (argv.length < 1){ - System.err.println("Usage: ReceiverWithRouting routingKey1 routingKey2 ..."); - System.exit(1); - } - - for(String routingKeys : argv){ - channel.queueBind(queueName, EXCHANGE_NAME, routingKeys); - } - - System.out.println(" [*] Waiting for messages. To exit press CTRL+C"); - - QueueingConsumer consumer = new QueueingConsumer(channel); - channel.basicConsume(queueName, true, consumer); - - while (true) { - QueueingConsumer.Delivery delivery = consumer.nextDelivery(); - String message = new String(delivery.getBody()); - String routingKey = delivery.getEnvelope().getRoutingKey(); - - System.out.println(" [x] Received '" + routingKey + "':'" + message + "'"); - } - } -} diff --git a/examples/data-generators/twitter-to-rabbit/src/main/java/com/stratio/examples/twittertorabbit/TwitterToRabbitMQ.java b/examples/data-generators/twitter-to-rabbit/src/main/java/com/stratio/examples/twittertorabbit/TwitterToRabbitMQ.java deleted file mode 100644 index 4934d6f6ed..0000000000 --- a/examples/data-generators/twitter-to-rabbit/src/main/java/com/stratio/examples/twittertorabbit/TwitterToRabbitMQ.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.stratio.examples.twittertorabbit; - -import com.rabbitmq.client.Channel; -import org.codehaus.jackson.map.ObjectMapper; - -public abstract class TwitterToRabbitMQ { - protected static final String EXCHANGE_NAME = "twitterExchange"; - protected static final String HOST = "localhost"; - - protected static Channel channel; - protected static ObjectMapper mapper; -} diff --git a/examples/data-generators/twitter-to-rabbit/src/main/java/com/stratio/examples/twittertorabbit/TwitterToRabbitMQSimple.java b/examples/data-generators/twitter-to-rabbit/src/main/java/com/stratio/examples/twittertorabbit/TwitterToRabbitMQSimple.java deleted file mode 100644 index 3c111d14b0..0000000000 --- a/examples/data-generators/twitter-to-rabbit/src/main/java/com/stratio/examples/twittertorabbit/TwitterToRabbitMQSimple.java +++ /dev/null @@ -1,64 +0,0 @@ -package com.stratio.examples.twittertorabbit; - -import com.rabbitmq.client.Connection; -import com.rabbitmq.client.ConnectionFactory; -import org.codehaus.jackson.map.ObjectMapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import twitter4j.*; - -import java.io.IOException; - -public class TwitterToRabbitMQSimple extends TwitterToRabbitMQ { - - private final static Logger LOG = LoggerFactory.getLogger(TwitterToRabbitMQSimple.class); - - /** - * Main entry of this application. - * - * @param args arguments doesn't take effect with this example - */ - public static void main(String[] args) { - - ConnectionFactory factory = new ConnectionFactory(); - try { - Connection conn = factory.newConnection(); - channel = conn.createChannel(); - } catch (IOException e) { - LOG.error("Cannot create RabbitMQ channel", e); - } - - mapper = new ObjectMapper(); - - TwitterStream twitterStream = new TwitterStreamFactory().getInstance(); - StatusListener listener = new StatusListener() { - - public void onStatus(Status status) { - try { - channel.basicPublish("", "test", null, mapper.writeValueAsBytes(status)); - } catch (IOException e) { - LOG.error("Cannot publish status: " + status, e); - } - } - - public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) { - } - - public void onTrackLimitationNotice(int numberOfLimitedStatuses) { - } - - public void onScrubGeo(long userId, long upToStatusId) { - } - - public void onStallWarning(StallWarning warning) { - } - - public void onException(Exception ex) { - LOG.error("Error listening on twitter stream", ex); - } - }; - twitterStream.addListener(listener); - twitterStream.sample(); - } - -} diff --git a/examples/data-generators/twitter-to-rabbit/src/main/java/com/stratio/examples/twittertorabbit/TwitterToRabbitMQWithRouting.java b/examples/data-generators/twitter-to-rabbit/src/main/java/com/stratio/examples/twittertorabbit/TwitterToRabbitMQWithRouting.java deleted file mode 100644 index 196a25dcf0..0000000000 --- a/examples/data-generators/twitter-to-rabbit/src/main/java/com/stratio/examples/twittertorabbit/TwitterToRabbitMQWithRouting.java +++ /dev/null @@ -1,68 +0,0 @@ -package com.stratio.examples.twittertorabbit; - -import com.rabbitmq.client.Connection; -import com.rabbitmq.client.ConnectionFactory; -import org.codehaus.jackson.map.ObjectMapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import twitter4j.*; - -import java.io.IOException; - -public class TwitterToRabbitMQWithRouting extends TwitterToRabbitMQ { - - private final static Logger LOG = LoggerFactory.getLogger(TwitterToRabbitMQWithRouting.class); - - /** - * Main entry of this application. - * - * @param args routing key to publish in - */ - public static void main(String[] args) { - - final String routingKey = args[0]; - - - ConnectionFactory factory = new ConnectionFactory(); - try { - Connection conn = factory.newConnection(); - channel = conn.createChannel(); - channel.exchangeDeclare(EXCHANGE_NAME, "direct"); - } catch (IOException e) { - LOG.error("Cannot create RabbitMQ channel", e); - } - - mapper = new ObjectMapper(); - - TwitterStream twitterStream = new TwitterStreamFactory().getInstance(); - StatusListener listener = new StatusListener() { - - public void onStatus(Status status) { - try { - channel.basicPublish(EXCHANGE_NAME, routingKey, null, mapper.writeValueAsBytes(status)); - } catch (IOException e) { - LOG.error("Cannot publish status: " + status, e); - } - } - - public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) { - } - - public void onTrackLimitationNotice(int numberOfLimitedStatuses) { - } - - public void onScrubGeo(long userId, long upToStatusId) { - } - - public void onStallWarning(StallWarning warning) { - } - - public void onException(Exception ex) { - LOG.error("Error listening on twitter stream", ex); - } - }; - twitterStream.addListener(listener); - twitterStream.sample(); - } - -} diff --git a/examples/data-generators/twitter-to-rabbit/src/main/resources/twitter4j.properties b/examples/data-generators/twitter-to-rabbit/src/main/resources/twitter4j.properties deleted file mode 100644 index 0118d3a091..0000000000 --- a/examples/data-generators/twitter-to-rabbit/src/main/resources/twitter4j.properties +++ /dev/null @@ -1,5 +0,0 @@ -debug=true -oauth.consumerKey=************************** -oauth.consumerSecret=************************** -oauth.accessToken=************************** -oauth.accessTokenSecret=************************** \ No newline at end of file diff --git a/examples/data-generators/twitter-to-rabbit/twitter-policy.json b/examples/data-generators/twitter-to-rabbit/twitter-policy.json deleted file mode 100644 index 35dbcb4e87..0000000000 --- a/examples/data-generators/twitter-to-rabbit/twitter-policy.json +++ /dev/null @@ -1,201 +0,0 @@ -{ - "name": "twitter-policy", - "description": "Twitter example", - "sparkStreamingWindow": 6000, - "rawData": { - "enabled": "false", - - "path": "myTestParquetPath" - }, - "checkpointPath": "checkpoint", - "input": - { - "name": "in", - "type": "RabbitMQ", - "configuration": { - "queue": "test", - "host": "localhost", - "port": "5672", - "exchangeName": "twitterExchange", - "routingKeys": [{"routingKey": "routingKey3"}] - - } - } -, - "transformations": [ - { - "name": "morphline-parser", - "order": 1, - "type": "Morphlines", - "outputFields": [ - "source", - "createdAt", - "text", - "lang", - "favoriteCount", - "retweetCount", - "user_followersCount", - "user_favouritesCount", - "user_id", - "user_screenName", - "user_timeZone" - ], - "configuration": { - "morphline": { - "id": "morphline1", - "importCommands": [ - "org.kitesdk.**" - ], - "commands": [ - { - "readJson": {} - }, - { - "extractJsonPaths": { - "paths": { - "source": "/source", - "createdAt": "/createdAt", - "text": "/text", - "lang": "/lang", - "favoriteCount": "/favoriteCount", - "retweetCount": "/retweetCount", - "user_followersCount": "/user/followersCount", - "user_favouritesCount": "/user/favouritesCount", - "user_id": "/user/id", - "user_screenName": "/user/screenName", - "user_timeZone": "/user/timeZone" - } - } - }, - { - "removeFields": { - "blacklist": [ - "literal:_attachment_body" - ] - } - } - ] - } - } - }, - { - "name": "createdAt", - "order": 2, - "type": "DateTime", - "inputField": "createdAt", - "outputFields": [ - "createdAt" - ], - "configuration": { - "inputFormat": "unixMillis" - } - } - ], - "cubes": [ - { - "name": "testCube", - "checkpointConfig": { - "timeDimension": "minute", - "granularity": "minute", - "interval": 30000, - "timeAvailability": 60000 - }, - "dimensions": [ - { - "name": "createdAt", - "field": "createdAt", - "type": "DateTime", - "precision": "minute" - } - ], - "operators": [ - { - "name": "countoperator", - "type": "Count", - "configuration": {} - }, - { - "name": "maxoperator", - "type": "Max", - "configuration": { - "inputField": "user_followersCount" - } - } - ] - }, - { - "name": "testCube1", - "checkpointConfig": { - "timeDimension": "minute", - "granularity": "minute", - "interval": 30000, - "timeAvailability": 60000 - }, - "dimensions": [ - { - "name": "lang", - "field": "lang" - } - ], - "operators": [ - { - "name": "countoperator", - "type": "Count", - "configuration": {} - }, - { - "name": "maxoperator", - "type": "Max", - "configuration": { - "inputField": "user_followersCount" - } - } - ] - }, - { - "name": "testCube2", - "checkpointConfig": { - "timeDimension": "minute", - "granularity": "minute", - "interval": 30000, - "timeAvailability": 60000 - }, - "dimensions": [ - { - "name": "lang", - "field": "lang" - }, - { - "name": "createdAt", - "field": "createdAt", - "type": "DateTime", - "precision": "minute" - } - ], - "operators": [ - { - "name": "countoperator", - "type": "Count", - "configuration": {} - }, - { - "name": "maxoperator", - "type": "Max", - "configuration": { - "inputField": "user_followersCount" - } - } - ] - } - ], - "outputs": [ - { - "name": "out-mongo", - "type": "MongoDb", - "configuration": { - "hosts": [{"host": "localhost" , "port": "27017" }], - "dbName": "sparkta" - } - } - ] -} \ No newline at end of file diff --git a/serving-api/pom.xml b/serving-api/pom.xml index dff08bee0c..1f57e7fa5a 100644 --- a/serving-api/pom.xml +++ b/serving-api/pom.xml @@ -145,6 +145,36 @@ org.apache.spark spark-streaming_${scala.binary.version} + + + javax.servlet + javax.servlet-api + + + javax.xml.bind + jaxb-api + + + com.sun.xml.bind + jaxb-impl + + + javax.servlet + javax.servlet-api + + + com.esotericsoftware.minlog + minlog + + + org.glassfish.external + management-api + + + org.glassfish.gmbal + gmbal-api-only + + org.apache.spark diff --git a/serving-core/pom.xml b/serving-core/pom.xml index aab62be9b9..51563e52eb 100644 --- a/serving-core/pom.xml +++ b/serving-core/pom.xml @@ -65,6 +65,12 @@ com.github.fge json-schema-validator + + + javax.mail + mailapi + + org.json4s @@ -84,6 +90,12 @@ spark-sql_${scala.binary.version} provided + + javax.mail + mailapi + 1.4.3 + test + org.apache.spark spark-streaming_${scala.binary.version}