diff --git a/build.gradle b/build.gradle index d56c36efe..75c100ba0 100644 --- a/build.gradle +++ b/build.gradle @@ -18,7 +18,7 @@ allprojects { joda_time : '2.9.3', slf4j : '1.7.25', guice : '4.2.1', - gadtry : '1.9.2-SNAPSHOT', + gadtry : '1.9.2', guava : '27.0-jre', jackson : '2.9.8', jersey : '2.28', diff --git a/sylph-main/src/main/java/ideal/sylph/main/SylphMaster.java b/sylph-main/src/main/java/ideal/sylph/main/SylphMaster.java index 922171aca..0b9949ef9 100644 --- a/sylph-main/src/main/java/ideal/sylph/main/SylphMaster.java +++ b/sylph-main/src/main/java/ideal/sylph/main/SylphMaster.java @@ -67,7 +67,7 @@ public static void main(String[] args) binder.bind(ControllerApp.class).withSingle() ).aop(new AuthAspect()).initialize(); //----analysis - logger.info("Analyzed App dependencies {}", String.join("\n", app.analysis().printShow())); + logger.info("Analyzed App dependencies {}", String.join("\n", app.analyze().printShow())); app.getInstance(JobEngineManager.class).loadRunners(); app.getInstance(OperatorManager.class).loadPlugins(); diff --git a/sylph-main/src/main/java/ideal/sylph/main/server/SylphContextImpl.java b/sylph-main/src/main/java/ideal/sylph/main/server/SylphContextImpl.java index 4d68ee9b2..465517f27 100644 --- a/sylph-main/src/main/java/ideal/sylph/main/server/SylphContextImpl.java +++ b/sylph-main/src/main/java/ideal/sylph/main/server/SylphContextImpl.java @@ -16,7 +16,7 @@ package ideal.sylph.main.server; import com.fasterxml.jackson.databind.ObjectMapper; -import com.github.harbby.gadtry.easyspi.Module; +import com.github.harbby.gadtry.spi.Module; import ideal.sylph.etl.Plugin; import ideal.sylph.main.service.JobEngineManager; import ideal.sylph.main.service.JobManager; diff --git a/sylph-main/src/main/java/ideal/sylph/main/service/JobEngineManager.java b/sylph-main/src/main/java/ideal/sylph/main/service/JobEngineManager.java index 0f40b994c..ab3ae757a 100644 --- a/sylph-main/src/main/java/ideal/sylph/main/service/JobEngineManager.java +++ b/sylph-main/src/main/java/ideal/sylph/main/service/JobEngineManager.java @@ -17,7 +17,7 @@ import com.github.harbby.gadtry.base.Closeables; import com.github.harbby.gadtry.base.Throwables; -import com.github.harbby.gadtry.easyspi.ModuleLoader; +import com.github.harbby.gadtry.spi.ModuleLoader; import com.github.harbby.gadtry.ioc.Autowired; import com.google.common.collect.ImmutableList; import ideal.sylph.etl.Operator; diff --git a/sylph-main/src/main/java/ideal/sylph/main/service/OperatorManager.java b/sylph-main/src/main/java/ideal/sylph/main/service/OperatorManager.java index 47cf92ddf..869aa3901 100644 --- a/sylph-main/src/main/java/ideal/sylph/main/service/OperatorManager.java +++ b/sylph-main/src/main/java/ideal/sylph/main/service/OperatorManager.java @@ -17,11 +17,11 @@ import com.github.harbby.gadtry.base.Try; import com.github.harbby.gadtry.collection.ImmutableList; -import com.github.harbby.gadtry.easyspi.Module; -import com.github.harbby.gadtry.easyspi.ModuleLoader; -import com.github.harbby.gadtry.easyspi.SecurityClassLoader; -import com.github.harbby.gadtry.easyspi.VolatileClassLoader; import com.github.harbby.gadtry.ioc.Autowired; +import com.github.harbby.gadtry.spi.Module; +import com.github.harbby.gadtry.spi.ModuleLoader; +import com.github.harbby.gadtry.spi.SecurityClassLoader; +import com.github.harbby.gadtry.spi.VolatileClassLoader; import ideal.sylph.etl.Operator; import ideal.sylph.etl.Plugin; import ideal.sylph.etl.api.RealTimePipeline; diff --git a/sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/FlinkContainerFactory.java b/sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/FlinkContainerFactory.java index 81d2ff0f8..9f41d4f28 100644 --- a/sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/FlinkContainerFactory.java +++ b/sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/FlinkContainerFactory.java @@ -105,8 +105,8 @@ public JobContainer createLocalContainer(Job job, String lastRunid) } System.out.print(line); }) - .notDepThisJvmClassPath() - .addUserjars(job.getDepends()) + .notDependParentJvmClassPath() + .addUserJars(job.getDepends()) .build(); YarnConfiguration yarnConfiguration = injector.getInstance(YarnConfiguration.class); return new LocalContainer() diff --git a/sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/FlinkRunner.java b/sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/FlinkRunner.java index e4c3d2f1f..dd59fe98e 100644 --- a/sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/FlinkRunner.java +++ b/sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/FlinkRunner.java @@ -16,8 +16,8 @@ package ideal.sylph.runner.flink; import com.github.harbby.gadtry.collection.ImmutableList; -import com.github.harbby.gadtry.easyspi.DirClassLoader; import com.github.harbby.gadtry.ioc.IocFactory; +import com.github.harbby.gadtry.spi.DynamicClassLoader; import ideal.sylph.runner.flink.engines.FlinkMainClassEngine; import ideal.sylph.runner.flink.engines.FlinkStreamEtlEngine; import ideal.sylph.runner.flink.engines.FlinkStreamSqlEngine; @@ -69,8 +69,8 @@ public void initialize(RunnerContext context) final ClassLoader classLoader = this.getClass().getClassLoader(); - if (classLoader instanceof DirClassLoader) { - ((DirClassLoader) classLoader).addDir(new File(flinkHome, "lib")); + if (classLoader instanceof DynamicClassLoader) { + ((DynamicClassLoader) classLoader).addDir(new File(flinkHome, "lib")); } IocFactory injector = IocFactory.create(binder -> { binder.bind(FlinkMainClassEngine.class).withSingle(); diff --git a/sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/engines/FlinkMainClassEngine.java b/sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/engines/FlinkMainClassEngine.java index 0e7caa8a6..b5e1dd7c4 100644 --- a/sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/engines/FlinkMainClassEngine.java +++ b/sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/engines/FlinkMainClassEngine.java @@ -106,8 +106,8 @@ private static JobGraph compile(String jobId, StringFlow flow, FlinkJobConfig jo return getJobGraphForJarClass(execEnv, mainClass, new String[0]); }) .setClassLoader(classLoader) - .addUserjars(ImmutableList.copy(classLoader.getURLs())) //flink jars + runner jar - .addUserjars(pluginJars) + .addUserJars(ImmutableList.copy(classLoader.getURLs())) //flink jars + runner jar + .addUserJars(pluginJars) .build(); return launcher.startAndGet(); diff --git a/sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/engines/FlinkStreamEtlEngine.java b/sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/engines/FlinkStreamEtlEngine.java index 9126c9200..299328de4 100644 --- a/sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/engines/FlinkStreamEtlEngine.java +++ b/sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/engines/FlinkStreamEtlEngine.java @@ -121,8 +121,8 @@ private static JobGraph compile(String jobId, EtlFlow flow, FlinkJobConfig jobCo return streamGraph.getJobGraph(); }) .setConsole((line) -> System.out.print(new Ansi().fg(YELLOW).a("[" + jobId + "] ").fg(GREEN).a(line).reset().toString())) - .addUserjars(ImmutableList.copy(classLoader.getURLs())) //flink jars + runner jar - .addUserjars(pluginJars) + .addUserJars(ImmutableList.copy(classLoader.getURLs())) //flink jars + runner jar + .addUserJars(pluginJars) .setClassLoader(classLoader) .build(); return launcher.startAndGet(); //setJobConfig(jobGraph, jobConfig, jobClassLoader, jobId); diff --git a/sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/engines/FlinkStreamSqlEngine.java b/sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/engines/FlinkStreamSqlEngine.java index 1c7b83255..fd18dd7a9 100644 --- a/sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/engines/FlinkStreamSqlEngine.java +++ b/sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/engines/FlinkStreamSqlEngine.java @@ -154,8 +154,8 @@ private static JobGraph compile( return streamGraph.getJobGraph(); }) //.notDepThisJvmClassPath() //todo: filter web rest jars 应避免构建作业时平台依赖混入 - .addUserjars(ImmutableList.copy(classLoader.getURLs())) //flink jars + runner jar - .addUserjars(pluginJars) + .addUserJars(ImmutableList.copy(classLoader.getURLs())) //flink jars + runner jar + .addUserJars(pluginJars) .setClassLoader(classLoader) .build(); diff --git a/sylph-runners/spark/src/main/java/ideal/sylph/runner/spark/JobHelper.java b/sylph-runners/spark/src/main/java/ideal/sylph/runner/spark/JobHelper.java index b5d66774c..3dac4072b 100644 --- a/sylph-runners/spark/src/main/java/ideal/sylph/runner/spark/JobHelper.java +++ b/sylph-runners/spark/src/main/java/ideal/sylph/runner/spark/JobHelper.java @@ -93,9 +93,9 @@ public UnaryOperator> loadSink(String driverStr, Map System.out.print(new Ansi().fg(YELLOW).a("[" + jobId + "] ").fg(GREEN).a(line).reset())) - .addUserjars(ImmutableList.copy(classLoader.getURLs())) - .addUserjars(pluginJars) - .notDepThisJvmClassPath() + .addUserJars(ImmutableList.copy(classLoader.getURLs())) + .addUserJars(pluginJars) + .notDependParentJvmClassPath() .setClassLoader(classLoader) .build(); launcher.startAndGet(); @@ -128,9 +128,9 @@ static Serializable build1xJob(String jobId, EtlFlow flow, List pluginJars, return 1; }) .setConsole((line) -> System.out.print(new Ansi().fg(YELLOW).a("[" + jobId + "] ").fg(GREEN).a(line).reset())) - .addUserjars(ImmutableList.copy(classLoader.getURLs())) //flink jars + runner jar - .addUserjars(pluginJars) - .notDepThisJvmClassPath() + .addUserJars(ImmutableList.copy(classLoader.getURLs())) //flink jars + runner jar + .addUserJars(pluginJars) + .notDependParentJvmClassPath() .setClassLoader(classLoader) .build(); launcher.startAndGet(); diff --git a/sylph-runners/spark/src/main/java/ideal/sylph/runner/spark/SparkContainerFactory.java b/sylph-runners/spark/src/main/java/ideal/sylph/runner/spark/SparkContainerFactory.java index 96d7365e9..3314bfc02 100644 --- a/sylph-runners/spark/src/main/java/ideal/sylph/runner/spark/SparkContainerFactory.java +++ b/sylph-runners/spark/src/main/java/ideal/sylph/runner/spark/SparkContainerFactory.java @@ -76,8 +76,8 @@ public JobContainer createLocalContainer(Job job, String lastRunid) } System.out.print(line); }) - .notDepThisJvmClassPath() - .addUserjars(job.getDepends()) + .notDependParentJvmClassPath() + .addUserJars(job.getDepends()) .build(); return new LocalContainer() diff --git a/sylph-runners/spark/src/main/java/ideal/sylph/runner/spark/SparkMainClassEngine.java b/sylph-runners/spark/src/main/java/ideal/sylph/runner/spark/SparkMainClassEngine.java index b361ee254..6b67456a1 100644 --- a/sylph-runners/spark/src/main/java/ideal/sylph/runner/spark/SparkMainClassEngine.java +++ b/sylph-runners/spark/src/main/java/ideal/sylph/runner/spark/SparkMainClassEngine.java @@ -73,8 +73,8 @@ private static Serializable compile(String jobId, StringFlow flow, List plu return ""; }) .setClassLoader(classLoader) - .addUserjars(ImmutableList.copy(classLoader.getURLs())) //flink jars + runner jar - .addUserjars(pluginJars) + .addUserJars(ImmutableList.copy(classLoader.getURLs())) //flink jars + runner jar + .addUserJars(pluginJars) .build(); launcher.startAndGet(); diff --git a/sylph-runners/spark/src/main/java/ideal/sylph/runner/spark/SparkRunner.java b/sylph-runners/spark/src/main/java/ideal/sylph/runner/spark/SparkRunner.java index d5b08b75f..a95cb23db 100644 --- a/sylph-runners/spark/src/main/java/ideal/sylph/runner/spark/SparkRunner.java +++ b/sylph-runners/spark/src/main/java/ideal/sylph/runner/spark/SparkRunner.java @@ -15,8 +15,8 @@ */ package ideal.sylph.runner.spark; -import com.github.harbby.gadtry.easyspi.DirClassLoader; import com.github.harbby.gadtry.ioc.IocFactory; +import com.github.harbby.gadtry.spi.DynamicClassLoader; import ideal.sylph.spi.Runner; import ideal.sylph.spi.RunnerContext; import ideal.sylph.spi.job.ContainerFactory; @@ -45,8 +45,8 @@ public void initialize(RunnerContext context) ClassLoader classLoader = this.getClass().getClassLoader(); - if (classLoader instanceof DirClassLoader) { - ((DirClassLoader) classLoader).addDir(new File(sparkHome, "jars")); + if (classLoader instanceof DynamicClassLoader) { + ((DynamicClassLoader) classLoader).addDir(new File(sparkHome, "jars")); } IocFactory injector = IocFactory.create( diff --git a/sylph-runners/spark/src/main/java/ideal/sylph/runner/spark/SparkStreamingSqlEngine.java b/sylph-runners/spark/src/main/java/ideal/sylph/runner/spark/SparkStreamingSqlEngine.java index d06a1e733..8ecf781ed 100644 --- a/sylph-runners/spark/src/main/java/ideal/sylph/runner/spark/SparkStreamingSqlEngine.java +++ b/sylph-runners/spark/src/main/java/ideal/sylph/runner/spark/SparkStreamingSqlEngine.java @@ -149,10 +149,10 @@ private static Serializable compile(String jobId, SqlFlow sqlFlow, OperatorMetaD appGetter.get(); return true; }) - .addUserjars(ImmutableList.copy(classLoader.getURLs())) //flink jars + runner jar - .addUserjars(pluginJars) + .addUserJars(ImmutableList.copy(classLoader.getURLs())) //flink jars + runner jar + .addUserJars(pluginJars) .setClassLoader(classLoader) - .notDepThisJvmClassPath() + .notDependParentJvmClassPath() .build(); launcher.startAndGet(); diff --git a/sylph-runners/spark/src/main/java/ideal/sylph/runner/spark/StructuredStreamingSqlEngine.java b/sylph-runners/spark/src/main/java/ideal/sylph/runner/spark/StructuredStreamingSqlEngine.java index eb24eba21..6c859dee7 100644 --- a/sylph-runners/spark/src/main/java/ideal/sylph/runner/spark/StructuredStreamingSqlEngine.java +++ b/sylph-runners/spark/src/main/java/ideal/sylph/runner/spark/StructuredStreamingSqlEngine.java @@ -114,10 +114,10 @@ private static Serializable compile(String jobId, SqlFlow sqlFlow, appGetter.get(); return true; }) - .addUserjars(ImmutableList.copy(classLoader.getURLs())) //flink jars + runner jar - .addUserjars(pluginJars) + .addUserJars(ImmutableList.copy(classLoader.getURLs())) //flink jars + runner jar + .addUserJars(pluginJars) .setClassLoader(classLoader) - .notDepThisJvmClassPath() + .notDependParentJvmClassPath() .build(); launcher.startAndGet(); diff --git a/sylph-spi/src/main/java/ideal/sylph/spi/Runner.java b/sylph-spi/src/main/java/ideal/sylph/spi/Runner.java index bb33033cb..4cb903a00 100644 --- a/sylph-spi/src/main/java/ideal/sylph/spi/Runner.java +++ b/sylph-spi/src/main/java/ideal/sylph/spi/Runner.java @@ -16,7 +16,7 @@ package ideal.sylph.spi; import com.github.harbby.gadtry.collection.ImmutableList; -import com.github.harbby.gadtry.easyspi.VolatileClassLoader; +import com.github.harbby.gadtry.spi.VolatileClassLoader; import ideal.sylph.etl.Operator; import ideal.sylph.spi.job.ContainerFactory; import ideal.sylph.spi.job.JobEngineHandle; diff --git a/sylph-spi/src/main/java/ideal/sylph/spi/SylphContext.java b/sylph-spi/src/main/java/ideal/sylph/spi/SylphContext.java index 893a901fb..a526b6978 100644 --- a/sylph-spi/src/main/java/ideal/sylph/spi/SylphContext.java +++ b/sylph-spi/src/main/java/ideal/sylph/spi/SylphContext.java @@ -15,7 +15,7 @@ */ package ideal.sylph.spi; -import com.github.harbby.gadtry.easyspi.Module; +import com.github.harbby.gadtry.spi.Module; import ideal.sylph.etl.Plugin; import ideal.sylph.spi.job.JobContainer; import ideal.sylph.spi.job.JobStore; diff --git a/sylph-web/src/main/java/ideal/sylph/controller/action/PluginManagerResource.java b/sylph-web/src/main/java/ideal/sylph/controller/action/PluginManagerResource.java index 3f2d912d8..99346cf03 100644 --- a/sylph-web/src/main/java/ideal/sylph/controller/action/PluginManagerResource.java +++ b/sylph-web/src/main/java/ideal/sylph/controller/action/PluginManagerResource.java @@ -16,7 +16,7 @@ package ideal.sylph.controller.action; import com.github.harbby.gadtry.collection.MutableMap; -import com.github.harbby.gadtry.easyspi.Module; +import com.github.harbby.gadtry.spi.Module; import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; import ideal.sylph.etl.Plugin;