Add also support for (non-default) maven repositories.

badthinking · Jun 6, 2019 · ab87ec9 · ab87ec9
1 parent 0e9feb6
commit ab87ec9
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 1 deletion.
diff --git a/src/main/java/io/radanalytics/operator/cluster/KubernetesSparkClusterDeployer.java b/src/main/java/io/radanalytics/operator/cluster/KubernetesSparkClusterDeployer.java
@@ -266,7 +266,12 @@ private ContainerBuilder augmentContainerBuilder(SparkCluster cluster, Container
             command.add("/bin/sh");
             command.add("-c");
             String dependencies = String.join(",", cluster.getMavenDependencies());
-            commandArgs.add("spark-submit --packages " + dependencies +" --conf spark.jars.ivy=/tmp --class no-op 0 > /tmp/spark-submit.log || true; /entrypoint /launch.sh");
+            String repositories = cluster.getMavenRepositories().isEmpty() ? "" : " --repositories " + String.join(",", cluster.getMavenRepositories());
+            commandArgs.add("/entrypoint pwd ; " + logConfig() + " ; spark-submit --packages " +
+                    dependencies +
+                    repositories +
+                    " --conf spark.jars.ivy=/tmp --driver-java-options '-Dlog4j.configuration=file:///tmp/log4j.properties' " +
+                    "--class no-op-ignore-this 0 || true; /entrypoint /launch.sh");
         }
 
         List<String> command = isMaster ? m.getCommand() : w.getCommand();
@@ -280,6 +285,14 @@ private ContainerBuilder augmentContainerBuilder(SparkCluster cluster, Container
         return builder;
     }
 
+    private String logConfig(){
+        String log4jconfig ="'log4j.rootCategory=ERROR, console\\n" +
+                "log4j.appender.console=org.apache.log4j.ConsoleAppender\\n" +
+                "log4j.appender.console.layout=org.apache.log4j.PatternLayout'";
+        return "echo -e " + log4jconfig + " > /tmp/log4j.properties";
+
+    }
+
     private void addLabels( Map<String, String> labels, SparkCluster cluster, boolean isMaster) {
         if (cluster.getLabels() != null) labels.putAll(cluster.getLabels());
         if (isMaster) {

diff --git a/src/main/resources/schema/sparkCluster.json b/src/main/resources/schema/sparkCluster.json
@@ -80,6 +80,12 @@
         "type": "string"
       }
     },
+    "mavenRepositories": {
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
     "customImage": {
       "type": "string"
     },