From 3f4c0901a8040f1c354c2fd3b66b1a953cf5d358 Mon Sep 17 00:00:00 2001 From: Philipp Dallig Date: Mon, 5 Apr 2021 15:43:00 +0200 Subject: [PATCH 1/9] Add pf4j for plugin loading --- pom.xml | 1 + .../zeppelin/integration/MiniZeppelin.java | 6 +- zeppelin-plugins/launcher/cluster/pom.xml | 44 +---- .../launcher/ClusterInterpreterLauncher.java | 24 ++- .../interpreter/launcher/ClusterPlugin.java | 42 +++++ .../ClusterInterpreterLauncherTest.java | 17 +- zeppelin-plugins/launcher/docker/pom.xml | 55 ++---- .../launcher/DockerInterpreterLauncher.java | 21 ++- .../interpreter/launcher/DockerPlugin.java | 42 +++++ .../DockerInterpreterProcessTest.java | 14 +- .../src/test/resources/log4j.properties | 24 +++ zeppelin-plugins/launcher/flink/pom.xml | 41 +---- .../launcher/FlinkInterpreterLauncher.java | 7 +- .../interpreter/launcher/FlinkPlugin.java | 42 +++++ .../launcher/k8s-standard/pom.xml | 106 +++++------ .../interpreter/launcher/K8sPlugin.java | 42 +++++ .../K8sStandardInterpreterLauncher.java | 9 +- .../K8sStandardInterpreterLauncherTest.java | 18 +- zeppelin-plugins/launcher/yarn/pom.xml | 41 +---- .../launcher/YarnInterpreterLauncher.java | 10 +- .../interpreter/launcher/YarnPlugin.java | 42 +++++ zeppelin-plugins/notebookrepo/azure/pom.xml | 23 +-- .../notebook/repo/AzureNotebookRepo.java | 8 +- .../zeppelin/notebook/repo/AzurePlugin.java | 41 +++++ .../notebookrepo/filesystem/pom.xml | 10 +- .../notebook/repo/FileSystemNotebookRepo.java | 2 + .../notebook/repo/FileSystemPlugin.java | 42 +++++ zeppelin-plugins/notebookrepo/gcs/pom.xml | 73 +------- .../notebook/repo/GCSNotebookRepo.java | 2 + .../zeppelin/notebook/repo/GCSPlugin.java | 42 +++++ zeppelin-plugins/notebookrepo/github/pom.xml | 10 +- .../notebook/repo/GitHubNotebookRepo.java | 2 + .../zeppelin/notebook/repo/GithubPlugin.java | 42 +++++ zeppelin-plugins/notebookrepo/mongo/pom.xml | 20 +-- .../notebook/repo/MongoNotebookRepo.java | 2 + .../zeppelin/notebook/repo/MongoPlugin.java | 42 +++++ zeppelin-plugins/notebookrepo/oss/pom.xml | 9 +- .../zeppelin/notebook/repo/AliyunPlugin.java | 42 +++++ .../notebook/repo/OSSNotebookRepo.java | 2 + zeppelin-plugins/notebookrepo/s3/pom.xml | 9 +- .../zeppelin/notebook/repo/AWSPlugin.java | 42 +++++ .../notebook/repo/S3NotebookRepo.java | 5 +- zeppelin-plugins/pom.xml | 146 ++++++++------- .../zeppelin/server/ZeppelinServer.java | 13 +- .../zeppelin/cluster/ZeppelinServerMock.java | 2 - .../zeppelin/rest/AbstractTestRestApi.java | 1 - zeppelin-zengine/pom.xml | 13 ++ .../interpreter/InterpreterSetting.java | 28 ++- .../InterpreterSettingManager.java | 15 +- .../launcher/InterpreterLauncher.java | 8 +- .../launcher/SparkInterpreterLauncher.java | 6 +- .../launcher/StandardInterpreterLauncher.java | 8 +- .../interpreter/recovery/StopInterpreter.java | 6 +- .../notebook/repo/GitNotebookRepo.java | 2 + .../notebook/repo/InMemoryNotebookRepo.java | 2 + .../zeppelin/notebook/repo/NotebookRepo.java | 3 +- .../notebook/repo/NotebookRepoSync.java | 13 +- .../notebook/repo/VFSNotebookRepo.java | 2 + .../zeppelin/plugin/IPluginManager.java} | 28 +-- .../apache/zeppelin/plugin/PluginManager.java | 169 ------------------ .../zeppelin/plugin/ZPluginManager.java | 98 ++++++++++ .../interpreter/AbstractInterpreterTest.java | 7 +- .../InterpreterSettingManagerTest.java | 6 +- .../interpreter/InterpreterSettingTest.java | 14 ++ .../ManagedInterpreterGroupTest.java | 3 + .../launcher/InterpreterLauncherTest.java | 0 .../SparkInterpreterLauncherTest.java | 27 ++- .../StandardInterpreterLauncherTest.java | 14 +- .../NotebookRepoSyncInitializationTest.java | 26 +-- .../notebook/repo/NotebookRepoSyncTest.java | 14 +- .../repo/mock/VFSNotebookRepoMock.java | 10 +- .../zeppelin/plugin/ZPluginManagerTest.java | 54 ++++++ 72 files changed, 1117 insertions(+), 739 deletions(-) create mode 100644 zeppelin-plugins/launcher/cluster/src/main/java/org/apache/zeppelin/interpreter/launcher/ClusterPlugin.java create mode 100644 zeppelin-plugins/launcher/docker/src/main/java/org/apache/zeppelin/interpreter/launcher/DockerPlugin.java create mode 100644 zeppelin-plugins/launcher/docker/src/test/resources/log4j.properties create mode 100644 zeppelin-plugins/launcher/flink/src/main/java/org/apache/zeppelin/interpreter/launcher/FlinkPlugin.java create mode 100644 zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/K8sPlugin.java create mode 100644 zeppelin-plugins/launcher/yarn/src/main/java/org/apache/zeppelin/interpreter/launcher/YarnPlugin.java create mode 100644 zeppelin-plugins/notebookrepo/azure/src/main/java/org/apache/zeppelin/notebook/repo/AzurePlugin.java create mode 100644 zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemPlugin.java create mode 100644 zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSPlugin.java create mode 100644 zeppelin-plugins/notebookrepo/github/src/main/java/org/apache/zeppelin/notebook/repo/GithubPlugin.java create mode 100644 zeppelin-plugins/notebookrepo/mongo/src/main/java/org/apache/zeppelin/notebook/repo/MongoPlugin.java create mode 100644 zeppelin-plugins/notebookrepo/oss/src/main/java/org/apache/zeppelin/notebook/repo/AliyunPlugin.java create mode 100644 zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/AWSPlugin.java rename {zeppelin-interpreter => zeppelin-zengine}/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterLauncher.java (95%) rename zeppelin-zengine/src/{test/java/org/apache/zeppelin/plugin/PluginManagerTest.java => main/java/org/apache/zeppelin/plugin/IPluginManager.java} (63%) delete mode 100644 zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/PluginManager.java create mode 100644 zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/ZPluginManager.java rename {zeppelin-interpreter => zeppelin-zengine}/src/test/java/org/apache/zeppelin/interpreter/launcher/InterpreterLauncherTest.java (100%) create mode 100644 zeppelin-zengine/src/test/java/org/apache/zeppelin/plugin/ZPluginManagerTest.java diff --git a/pom.xml b/pom.xml index 17c6dbbaecb..b8d23af9413 100644 --- a/pom.xml +++ b/pom.xml @@ -134,6 +134,7 @@ 3.6.3 4.1.14 1.6.0 + 3.6.0 2.7.7 2.6.5 diff --git a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/MiniZeppelin.java b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/MiniZeppelin.java index 0931fe82a05..8bc0bd59705 100644 --- a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/MiniZeppelin.java +++ b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/MiniZeppelin.java @@ -24,6 +24,8 @@ import org.apache.zeppelin.interpreter.InterpreterFactory; import org.apache.zeppelin.interpreter.InterpreterSettingManager; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; +import org.apache.zeppelin.plugin.IPluginManager; +import org.apache.zeppelin.plugin.ZPluginManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -59,8 +61,10 @@ public void start(Class clazz) throws IOException { System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir.getAbsolutePath()); System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT.getVarName(), "120000"); conf = ZeppelinConfiguration.create(); + IPluginManager pluginManager = new ZPluginManager(conf); + pluginManager.loadAndStartPlugins(); interpreterSettingManager = new InterpreterSettingManager(conf, - mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class)); + mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class), pluginManager); interpreterFactory = new InterpreterFactory(interpreterSettingManager); } diff --git a/zeppelin-plugins/launcher/cluster/pom.xml b/zeppelin-plugins/launcher/cluster/pom.xml index 55c8e1dec88..1459b0541c1 100644 --- a/zeppelin-plugins/launcher/cluster/pom.xml +++ b/zeppelin-plugins/launcher/cluster/pom.xml @@ -36,53 +36,15 @@ Launcher/ClusterInterpreterLauncher + org.apache.zeppelin.interpreter.launcher.ClusterPlugin org.apache.zeppelin launcher-docker - 0.11.0-SNAPSHOT + ${project.version} + provided - - - - - ${project.basedir}/src/test/resources - - - ${project.basedir}/src/main/resources - - - - - maven-enforcer-plugin - - - enforce - none - - - - - maven-dependency-plugin - - - org.apache.maven.plugins - maven-checkstyle-plugin - - false - - - - - - src/main/resources - - **/*.* - - - - diff --git a/zeppelin-plugins/launcher/cluster/src/main/java/org/apache/zeppelin/interpreter/launcher/ClusterInterpreterLauncher.java b/zeppelin-plugins/launcher/cluster/src/main/java/org/apache/zeppelin/interpreter/launcher/ClusterInterpreterLauncher.java index 397f6ab5157..1910b273421 100644 --- a/zeppelin-plugins/launcher/cluster/src/main/java/org/apache/zeppelin/interpreter/launcher/ClusterInterpreterLauncher.java +++ b/zeppelin-plugins/launcher/cluster/src/main/java/org/apache/zeppelin/interpreter/launcher/ClusterInterpreterLauncher.java @@ -30,6 +30,8 @@ import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterRunningProcess; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils; +import org.apache.zeppelin.plugin.IPluginManager; +import org.pf4j.Extension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,6 +39,8 @@ import java.util.HashMap; import java.util.Map; +import javax.inject.Inject; + import static org.apache.zeppelin.cluster.event.ClusterEvent.CREATE_INTP_PROCESS; import static org.apache.zeppelin.cluster.meta.ClusterMeta.INTP_TSERVER_HOST; import static org.apache.zeppelin.cluster.meta.ClusterMeta.INTP_TSERVER_PORT; @@ -46,22 +50,31 @@ /** * Interpreter Launcher which use cluster to launch the interpreter process. */ +@Extension(points = InterpreterLauncher.class) public class ClusterInterpreterLauncher extends StandardInterpreterLauncher implements ClusterEventListener { private static final Logger LOGGER = LoggerFactory.getLogger(ClusterInterpreterLauncher.class); private InterpreterLaunchContext context; private ClusterManagerServer clusterServer; - public ClusterInterpreterLauncher(ZeppelinConfiguration zConf, RecoveryStorage recoveryStorage) - throws IOException { - super(zConf, recoveryStorage); + private IPluginManager pluginManager; + + @Inject + void setPluginManager(IPluginManager pluginManager) { + LOGGER.info("Injecting PluginManager"); + this.pluginManager = pluginManager; + } + + @Override + public void init(ZeppelinConfiguration zConf, RecoveryStorage recoveryStorage) { + super.init(zConf, recoveryStorage); this.clusterServer = ClusterManagerServer.getInstance(zConf); clusterServer.addClusterEventListeners(ClusterManagerServer.CLUSTER_INTP_EVENT_TOPIC, this); } @Override public InterpreterClient launchDirectly(InterpreterLaunchContext context) throws IOException { - LOGGER.info("Launching Interpreter: " + context.getInterpreterSettingGroup()); + LOGGER.info("Launching Interpreter: {}", context.getInterpreterSettingGroup()); this.context = context; this.properties = context.getProperties(); @@ -224,7 +237,8 @@ private InterpreterClient createInterpreterProcess(InterpreterLaunchContext cont InterpreterClient intpProcess = null; if (isRunningOnDocker(zConf)) { - DockerInterpreterLauncher dockerIntpLauncher = new DockerInterpreterLauncher(zConf, null); + DockerInterpreterLauncher dockerIntpLauncher = (DockerInterpreterLauncher) pluginManager.createInterpreterLauncher("DockerInterpreterLauncher", null); + dockerIntpLauncher.setPluginManager(pluginManager); dockerIntpLauncher.setProperties(context.getProperties()); intpProcess = dockerIntpLauncher.launch(context); } else { diff --git a/zeppelin-plugins/launcher/cluster/src/main/java/org/apache/zeppelin/interpreter/launcher/ClusterPlugin.java b/zeppelin-plugins/launcher/cluster/src/main/java/org/apache/zeppelin/interpreter/launcher/ClusterPlugin.java new file mode 100644 index 00000000000..167accbf9cd --- /dev/null +++ b/zeppelin-plugins/launcher/cluster/src/main/java/org/apache/zeppelin/interpreter/launcher/ClusterPlugin.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.interpreter.launcher; + +import org.pf4j.Plugin; +import org.pf4j.PluginWrapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ClusterPlugin extends Plugin { + + private static final Logger LOGGER = LoggerFactory.getLogger(ClusterPlugin.class); + + public ClusterPlugin(PluginWrapper wrapper) { + super(wrapper); + } + + @Override + public void start() { + LOGGER.info("ClusterPlugin started"); + } + + @Override + public void stop() { + LOGGER.info("ClusterPlugin stopped"); + } +} diff --git a/zeppelin-plugins/launcher/cluster/src/test/java/org/apache/zeppelin/interpreter/launcher/ClusterInterpreterLauncherTest.java b/zeppelin-plugins/launcher/cluster/src/test/java/org/apache/zeppelin/interpreter/launcher/ClusterInterpreterLauncherTest.java index 995da7ecd97..23d6fd28ab7 100644 --- a/zeppelin-plugins/launcher/cluster/src/test/java/org/apache/zeppelin/interpreter/launcher/ClusterInterpreterLauncherTest.java +++ b/zeppelin-plugins/launcher/cluster/src/test/java/org/apache/zeppelin/interpreter/launcher/ClusterInterpreterLauncherTest.java @@ -19,6 +19,8 @@ import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.interpreter.InterpreterOption; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterRunningProcess; +import org.apache.zeppelin.plugin.IPluginManager; +import org.apache.zeppelin.plugin.ZPluginManager; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; @@ -33,7 +35,8 @@ import static org.junit.Assert.assertTrue; public class ClusterInterpreterLauncherTest extends ClusterMockTest { - private static Logger LOGGER = LoggerFactory.getLogger(ClusterInterpreterLauncherTest.class); + private static final Logger LOGGER = LoggerFactory.getLogger(ClusterInterpreterLauncherTest.class); + private IPluginManager pluginManager; @BeforeClass public static void startTest() throws IOException, InterruptedException { @@ -50,6 +53,8 @@ public void setUp() { for (final ZeppelinConfiguration.ConfVars confVar : ZeppelinConfiguration.ConfVars.values()) { System.clearProperty(confVar.getVarName()); } + pluginManager = new ZPluginManager(zconf); + pluginManager.loadAndStartPlugins(); } // TODO(zjffdu) disable this test because this is not a correct unit test, @@ -58,8 +63,7 @@ public void setUp() { public void testConnectExistOnlineIntpProcess() throws IOException { mockIntpProcessMeta("intpGroupId", true); - ClusterInterpreterLauncher launcher - = new ClusterInterpreterLauncher(ClusterMockTest.zconf, null); + ClusterInterpreterLauncher launcher = (ClusterInterpreterLauncher) pluginManager.createInterpreterLauncher("ClusterInterpreterLauncher", null); Properties properties = new Properties(); properties.setProperty( ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT.getVarName(), "5000"); @@ -83,7 +87,7 @@ public void testConnectExistOfflineIntpProcess() throws IOException { mockIntpProcessMeta("intpGroupId2", false); ClusterInterpreterLauncher launcher - = new ClusterInterpreterLauncher(ClusterMockTest.zconf, null); + = (ClusterInterpreterLauncher) pluginManager.createInterpreterLauncher("ClusterInterpreterLauncher", null);; Properties properties = new Properties(); properties.setProperty( ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT.getVarName(), "5000"); @@ -110,7 +114,8 @@ public void testCreateIntpProcessDockerMode() throws IOException { zconf.setRunMode(ZeppelinConfiguration.RUN_MODE.DOCKER); ClusterInterpreterLauncher launcher - = new ClusterInterpreterLauncher(zconf, null); + = (ClusterInterpreterLauncher) pluginManager.createInterpreterLauncher("ClusterInterpreterLauncher", null); + launcher.setPluginManager(pluginManager); Properties properties = new Properties(); properties.setProperty( ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT.getVarName(), "1000"); @@ -129,7 +134,7 @@ public void testCreateIntpProcessLocalMode() throws IOException { zconf.setRunMode(ZeppelinConfiguration.RUN_MODE.LOCAL); ClusterInterpreterLauncher launcher - = new ClusterInterpreterLauncher(zconf, null); + = (ClusterInterpreterLauncher) pluginManager.createInterpreterLauncher("ClusterInterpreterLauncher", null); Properties properties = new Properties(); properties.setProperty( ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT.getVarName(), "1000"); diff --git a/zeppelin-plugins/launcher/docker/pom.xml b/zeppelin-plugins/launcher/docker/pom.xml index ab7ce318b54..9bea21919de 100644 --- a/zeppelin-plugins/launcher/docker/pom.xml +++ b/zeppelin-plugins/launcher/docker/pom.xml @@ -36,6 +36,7 @@ Launcher/DockerInterpreterLauncher + org.apache.zeppelin.interpreter.launcher.DockerPlugin @@ -49,12 +50,24 @@ com.hubspot.jinjava jinjava 2.5.4 + + + org.slf4j + slf4j-api + + com.spotify docker-client shaded 8.15.2 + + + org.slf4j + slf4j-api + + org.apache.commons @@ -64,52 +77,10 @@ org.powermock powermock-api-mockito - ${powermock.version} org.powermock powermock-module-junit4 - ${powermock.version} - - - - - ${project.basedir}/src/test/resources - - - ${project.basedir}/src/main/resources - - - - - maven-enforcer-plugin - - - enforce - none - - - - - maven-dependency-plugin - - - org.apache.maven.plugins - maven-checkstyle-plugin - - false - - - - - - src/main/resources - - **/*.* - - - - diff --git a/zeppelin-plugins/launcher/docker/src/main/java/org/apache/zeppelin/interpreter/launcher/DockerInterpreterLauncher.java b/zeppelin-plugins/launcher/docker/src/main/java/org/apache/zeppelin/interpreter/launcher/DockerInterpreterLauncher.java index 4865aedd35f..5b0b12495fb 100644 --- a/zeppelin-plugins/launcher/docker/src/main/java/org/apache/zeppelin/interpreter/launcher/DockerInterpreterLauncher.java +++ b/zeppelin-plugins/launcher/docker/src/main/java/org/apache/zeppelin/interpreter/launcher/DockerInterpreterLauncher.java @@ -16,25 +16,30 @@ */ package org.apache.zeppelin.interpreter.launcher; -import org.apache.zeppelin.conf.ZeppelinConfiguration; -import org.apache.zeppelin.interpreter.recovery.RecoveryStorage; +import org.apache.zeppelin.plugin.IPluginManager; +import org.pf4j.Extension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.Map; +import javax.inject.Inject; + /** * Interpreter Launcher which use shell script to launch the interpreter process. */ +@Extension public class DockerInterpreterLauncher extends InterpreterLauncher { private static final Logger LOGGER = LoggerFactory.getLogger(DockerInterpreterLauncher.class); private InterpreterLaunchContext context; + private IPluginManager pluginManager; - public DockerInterpreterLauncher(ZeppelinConfiguration zConf, RecoveryStorage recoveryStorage) - throws IOException { - super(zConf, recoveryStorage); + @Inject + void setPluginManager(IPluginManager pluginManager) { + LOGGER.info("Injecting PluginManager"); + this.pluginManager = pluginManager; } @Override @@ -54,11 +59,11 @@ public InterpreterClient launchDirectly(InterpreterLaunchContext context) throws StandardInterpreterLauncher interpreterLauncher = null; if (isSpark()) { - interpreterLauncher = new SparkInterpreterLauncher(zConf, recoveryStorage); + interpreterLauncher = (StandardInterpreterLauncher) pluginManager.createInterpreterLauncher("SparkInterpreterLauncher", recoveryStorage); } else if (isFlink()) { - interpreterLauncher = new FlinkInterpreterLauncher(zConf, recoveryStorage); + interpreterLauncher = (StandardInterpreterLauncher) pluginManager.createInterpreterLauncher("FlinkInterpreterLauncher", recoveryStorage); } else { - interpreterLauncher = new StandardInterpreterLauncher(zConf, recoveryStorage); + interpreterLauncher = (StandardInterpreterLauncher) pluginManager.createInterpreterLauncher("StandardInterpreterLauncher", recoveryStorage); } interpreterLauncher.setProperties(context.getProperties()); Map env = interpreterLauncher.buildEnvFromProperties(context); diff --git a/zeppelin-plugins/launcher/docker/src/main/java/org/apache/zeppelin/interpreter/launcher/DockerPlugin.java b/zeppelin-plugins/launcher/docker/src/main/java/org/apache/zeppelin/interpreter/launcher/DockerPlugin.java new file mode 100644 index 00000000000..7f4236bab81 --- /dev/null +++ b/zeppelin-plugins/launcher/docker/src/main/java/org/apache/zeppelin/interpreter/launcher/DockerPlugin.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.interpreter.launcher; + +import org.pf4j.Plugin; +import org.pf4j.PluginWrapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DockerPlugin extends Plugin { + + private static final Logger LOGGER = LoggerFactory.getLogger(DockerPlugin.class); + + public DockerPlugin(PluginWrapper wrapper) { + super(wrapper); + } + + @Override + public void start() { + LOGGER.info("DockerPlugin started"); + } + + @Override + public void stop() { + LOGGER.info("DockerPlugin stopped"); + } +} diff --git a/zeppelin-plugins/launcher/docker/src/test/java/org/apache/zeppelin/interpreter/launcher/DockerInterpreterProcessTest.java b/zeppelin-plugins/launcher/docker/src/test/java/org/apache/zeppelin/interpreter/launcher/DockerInterpreterProcessTest.java index 4a7a39de6db..839eb84937a 100644 --- a/zeppelin-plugins/launcher/docker/src/test/java/org/apache/zeppelin/interpreter/launcher/DockerInterpreterProcessTest.java +++ b/zeppelin-plugins/launcher/docker/src/test/java/org/apache/zeppelin/interpreter/launcher/DockerInterpreterProcessTest.java @@ -18,6 +18,8 @@ import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.interpreter.InterpreterOption; +import org.apache.zeppelin.plugin.ZPluginManager; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.powermock.api.mockito.PowerMockito; @@ -43,12 +45,20 @@ public class DockerInterpreterProcessTest { private static final Logger LOGGER = LoggerFactory.getLogger(DockerInterpreterProcessTest.class); - protected static ZeppelinConfiguration zconf = ZeppelinConfiguration.create(); + private ZeppelinConfiguration zconf; + private ZPluginManager pluginManager; + + @Before + public void setUp() { + zconf = ZeppelinConfiguration.create(); + pluginManager = new ZPluginManager(zconf); + } @Test public void testCreateIntpProcess() throws IOException { DockerInterpreterLauncher launcher - = new DockerInterpreterLauncher(zconf, null); + = (DockerInterpreterLauncher) pluginManager.createInterpreterLauncher("DockerInterpreterLauncher", null); + launcher.setPluginManager(pluginManager); Properties properties = new Properties(); properties.setProperty( ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT.getVarName(), "5000"); diff --git a/zeppelin-plugins/launcher/docker/src/test/resources/log4j.properties b/zeppelin-plugins/launcher/docker/src/test/resources/log4j.properties new file mode 100644 index 00000000000..47256155227 --- /dev/null +++ b/zeppelin-plugins/launcher/docker/src/test/resources/log4j.properties @@ -0,0 +1,24 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Root logger option +log4j.rootLogger=INFO, stdout + +# Direct log messages to stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%5p [%d] ({%t} %F[%M]:%L) - %m%n diff --git a/zeppelin-plugins/launcher/flink/pom.xml b/zeppelin-plugins/launcher/flink/pom.xml index 979df05ef17..ebe6c4c9eb3 100644 --- a/zeppelin-plugins/launcher/flink/pom.xml +++ b/zeppelin-plugins/launcher/flink/pom.xml @@ -36,45 +36,6 @@ Launcher/FlinkInterpreterLauncher + org.apache.zeppelin.interpreter.launcher.FlinkPlugin - - - - - ${project.basedir}/src/test/resources - - - ${project.basedir}/src/main/resources - - - - - maven-enforcer-plugin - - - enforce - none - - - - - maven-dependency-plugin - - - org.apache.maven.plugins - maven-checkstyle-plugin - - false - - - - - - src/main/resources - - **/*.* - - - - diff --git a/zeppelin-plugins/launcher/flink/src/main/java/org/apache/zeppelin/interpreter/launcher/FlinkInterpreterLauncher.java b/zeppelin-plugins/launcher/flink/src/main/java/org/apache/zeppelin/interpreter/launcher/FlinkInterpreterLauncher.java index b7e5f83a720..a195cc3c9f6 100644 --- a/zeppelin-plugins/launcher/flink/src/main/java/org/apache/zeppelin/interpreter/launcher/FlinkInterpreterLauncher.java +++ b/zeppelin-plugins/launcher/flink/src/main/java/org/apache/zeppelin/interpreter/launcher/FlinkInterpreterLauncher.java @@ -21,7 +21,7 @@ import com.google.common.collect.Sets; import org.apache.commons.lang3.StringUtils; import org.apache.zeppelin.conf.ZeppelinConfiguration; -import org.apache.zeppelin.interpreter.recovery.RecoveryStorage; +import org.pf4j.Extension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,16 +36,13 @@ import java.util.Set; import java.util.stream.Collectors; +@Extension public class FlinkInterpreterLauncher extends StandardInterpreterLauncher { private static final Logger LOGGER = LoggerFactory.getLogger(FlinkInterpreterLauncher.class); private static final Set FLINK_EXECUTION_MODES = Sets.newHashSet( "local", "remote", "yarn", "yarn-application", "kubernetes-application"); - public FlinkInterpreterLauncher(ZeppelinConfiguration zConf, RecoveryStorage recoveryStorage) { - super(zConf, recoveryStorage); - } - @Override public Map buildEnvFromProperties(InterpreterLaunchContext context) throws IOException { diff --git a/zeppelin-plugins/launcher/flink/src/main/java/org/apache/zeppelin/interpreter/launcher/FlinkPlugin.java b/zeppelin-plugins/launcher/flink/src/main/java/org/apache/zeppelin/interpreter/launcher/FlinkPlugin.java new file mode 100644 index 00000000000..4d0f1b7aeb7 --- /dev/null +++ b/zeppelin-plugins/launcher/flink/src/main/java/org/apache/zeppelin/interpreter/launcher/FlinkPlugin.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.interpreter.launcher; + +import org.pf4j.Plugin; +import org.pf4j.PluginWrapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class FlinkPlugin extends Plugin { + + private static final Logger LOGGER = LoggerFactory.getLogger(FlinkPlugin.class); + + public FlinkPlugin(PluginWrapper wrapper) { + super(wrapper); + } + + @Override + public void start() { + LOGGER.info("FlinkPlugin started"); + } + + @Override + public void stop() { + LOGGER.info("FlinkPlugin stopped"); + } +} diff --git a/zeppelin-plugins/launcher/k8s-standard/pom.xml b/zeppelin-plugins/launcher/k8s-standard/pom.xml index 81894cc8247..d60ddab90ad 100644 --- a/zeppelin-plugins/launcher/k8s-standard/pom.xml +++ b/zeppelin-plugins/launcher/k8s-standard/pom.xml @@ -35,16 +35,45 @@ Launcher/K8sStandardInterpreterLauncher + org.apache.zeppelin.interpreter.launcher.K8sPlugin 5.3.2 + 0.3.0 2.5.4 + + + + io.fabric8 + zjsonpatch + ${zjsonpatch.version} + + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + + + + + + io.fabric8 kubernetes-client ${kubernetes.client.version} compile + + + org.slf4j + slf4j-api + + @@ -65,6 +94,15 @@ com.fasterxml.jackson.core jackson-annotations + + + com.google.guava + guava + + + org.slf4j + slf4j-api + @@ -73,63 +111,17 @@ kubernetes-server-mock ${kubernetes.client.version} test + + + + com.squareup.okhttp3 + okhttp + + + com.fasterxml.jackson.core + jackson-databind + + - - - - - maven-enforcer-plugin - - - enforce - none - - - - - - org.apache.maven.plugins - maven-shade-plugin - - - package - - shade - - - - - - - okio - org.apache.zeppelin.shaded.okio - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - - maven-dependency-plugin - - - copy-plugin-dependencies - - true - - - - - - diff --git a/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/K8sPlugin.java b/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/K8sPlugin.java new file mode 100644 index 00000000000..5556ec940c8 --- /dev/null +++ b/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/K8sPlugin.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.interpreter.launcher; + +import org.pf4j.Plugin; +import org.pf4j.PluginWrapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class K8sPlugin extends Plugin { + + private static final Logger LOGGER = LoggerFactory.getLogger(K8sPlugin.class); + + public K8sPlugin(PluginWrapper wrapper) { + super(wrapper); + } + + @Override + public void start() { + LOGGER.info("K8sPlugin started"); + } + + @Override + public void stop() { + LOGGER.info("K8sPlugin stopped"); + } +} diff --git a/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/K8sStandardInterpreterLauncher.java b/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/K8sStandardInterpreterLauncher.java index d8f2b728e20..eb530001256 100644 --- a/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/K8sStandardInterpreterLauncher.java +++ b/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/K8sStandardInterpreterLauncher.java @@ -27,6 +27,7 @@ import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.interpreter.recovery.RecoveryStorage; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils; +import org.pf4j.Extension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,13 +37,15 @@ /** * Interpreter Launcher which use shell script to launch the interpreter process. */ +@Extension public class K8sStandardInterpreterLauncher extends InterpreterLauncher { private static final Logger LOGGER = LoggerFactory.getLogger(K8sStandardInterpreterLauncher.class); - private final KubernetesClient client; + private KubernetesClient client; - public K8sStandardInterpreterLauncher(ZeppelinConfiguration zConf, RecoveryStorage recoveryStorage) { - super(zConf, recoveryStorage); + @Override + public void init(ZeppelinConfiguration zConf, RecoveryStorage recoveryStorage) { + super.init(zConf, recoveryStorage); client = new DefaultKubernetesClient(); } diff --git a/zeppelin-plugins/launcher/k8s-standard/src/test/java/org/apache/zeppelin/interpreter/launcher/K8sStandardInterpreterLauncherTest.java b/zeppelin-plugins/launcher/k8s-standard/src/test/java/org/apache/zeppelin/interpreter/launcher/K8sStandardInterpreterLauncherTest.java index 4afeb0ff5b8..b292d2a49a4 100644 --- a/zeppelin-plugins/launcher/k8s-standard/src/test/java/org/apache/zeppelin/interpreter/launcher/K8sStandardInterpreterLauncherTest.java +++ b/zeppelin-plugins/launcher/k8s-standard/src/test/java/org/apache/zeppelin/interpreter/launcher/K8sStandardInterpreterLauncherTest.java @@ -25,6 +25,8 @@ import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.interpreter.InterpreterOption; +import org.apache.zeppelin.plugin.IPluginManager; +import org.apache.zeppelin.plugin.ZPluginManager; import org.junit.Before; import org.junit.Test; @@ -32,18 +34,22 @@ * In the future, test may use minikube for end-to-end test */ public class K8sStandardInterpreterLauncherTest { + + private IPluginManager pluginManager; @Before public void setUp() { for (final ZeppelinConfiguration.ConfVars confVar : ZeppelinConfiguration.ConfVars.values()) { System.clearProperty(confVar.getVarName()); } + pluginManager = new ZPluginManager(ZeppelinConfiguration.create()); + pluginManager.loadAndStartPlugins(); } @Test public void testK8sLauncher() throws IOException { // given - ZeppelinConfiguration zConf = ZeppelinConfiguration.create(); - K8sStandardInterpreterLauncher launcher = new K8sStandardInterpreterLauncher(zConf, null); + K8sStandardInterpreterLauncher launcher + = (K8sStandardInterpreterLauncher) pluginManager.createInterpreterLauncher("K8sStandardInterpreterLauncher", null); Properties properties = new Properties(); properties.setProperty("ENV_1", "VALUE_1"); properties.setProperty("property_1", "value_1"); @@ -71,8 +77,8 @@ public void testK8sLauncher() throws IOException { @Test public void testK8sLauncherWithSparkAndUserImpersonate() throws IOException { // given - ZeppelinConfiguration zConf = ZeppelinConfiguration.create(); - K8sStandardInterpreterLauncher launcher = new K8sStandardInterpreterLauncher(zConf, null); + K8sStandardInterpreterLauncher launcher + = (K8sStandardInterpreterLauncher) pluginManager.createInterpreterLauncher("K8sStandardInterpreterLauncher", null); Properties properties = new Properties(); properties.setProperty("ENV_1", "VALUE_1"); properties.setProperty("property_1", "value_1"); @@ -104,8 +110,8 @@ public void testK8sLauncherWithSparkAndUserImpersonate() throws IOException { @Test public void testK8sLauncherWithSparkAndWithoutUserImpersonate() throws IOException { // given - ZeppelinConfiguration zConf = ZeppelinConfiguration.create(); - K8sStandardInterpreterLauncher launcher = new K8sStandardInterpreterLauncher(zConf, null); + K8sStandardInterpreterLauncher launcher + = (K8sStandardInterpreterLauncher) pluginManager.createInterpreterLauncher("K8sStandardInterpreterLauncher", null);; Properties properties = new Properties(); properties.setProperty("ENV_1", "VALUE_1"); properties.setProperty("property_1", "value_1"); diff --git a/zeppelin-plugins/launcher/yarn/pom.xml b/zeppelin-plugins/launcher/yarn/pom.xml index 64d579a4f03..6c52ec235a8 100644 --- a/zeppelin-plugins/launcher/yarn/pom.xml +++ b/zeppelin-plugins/launcher/yarn/pom.xml @@ -36,48 +36,9 @@ Launcher/YarnInterpreterLauncher + org.apache.zeppelin.interpreter.launcher.YarnPlugin - - - - ${project.basedir}/src/test/resources - - - ${project.basedir}/src/main/resources - - - - - maven-enforcer-plugin - - - enforce - none - - - - - maven-dependency-plugin - - - org.apache.maven.plugins - maven-checkstyle-plugin - - true - - - - - - src/main/resources - - **/*.* - - - - - hadoop2 diff --git a/zeppelin-plugins/launcher/yarn/src/main/java/org/apache/zeppelin/interpreter/launcher/YarnInterpreterLauncher.java b/zeppelin-plugins/launcher/yarn/src/main/java/org/apache/zeppelin/interpreter/launcher/YarnInterpreterLauncher.java index f07a1eef48f..ddf746ac0c9 100644 --- a/zeppelin-plugins/launcher/yarn/src/main/java/org/apache/zeppelin/interpreter/launcher/YarnInterpreterLauncher.java +++ b/zeppelin-plugins/launcher/yarn/src/main/java/org/apache/zeppelin/interpreter/launcher/YarnInterpreterLauncher.java @@ -17,9 +17,8 @@ package org.apache.zeppelin.interpreter.launcher; -import org.apache.zeppelin.conf.ZeppelinConfiguration; -import org.apache.zeppelin.interpreter.recovery.RecoveryStorage; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils; +import org.pf4j.Extension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,13 +29,10 @@ /** * Launcher for running interpreter in yarn container. */ +@Extension public class YarnInterpreterLauncher extends InterpreterLauncher { - private static Logger LOGGER = LoggerFactory.getLogger(YarnInterpreterLauncher.class); - - public YarnInterpreterLauncher(ZeppelinConfiguration zConf, RecoveryStorage recoveryStorage) { - super(zConf, recoveryStorage); - } + private static final Logger LOGGER = LoggerFactory.getLogger(YarnInterpreterLauncher.class); @Override public InterpreterClient launchDirectly(InterpreterLaunchContext context) throws IOException { diff --git a/zeppelin-plugins/launcher/yarn/src/main/java/org/apache/zeppelin/interpreter/launcher/YarnPlugin.java b/zeppelin-plugins/launcher/yarn/src/main/java/org/apache/zeppelin/interpreter/launcher/YarnPlugin.java new file mode 100644 index 00000000000..257346eeb03 --- /dev/null +++ b/zeppelin-plugins/launcher/yarn/src/main/java/org/apache/zeppelin/interpreter/launcher/YarnPlugin.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.interpreter.launcher; + +import org.pf4j.Plugin; +import org.pf4j.PluginWrapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class YarnPlugin extends Plugin { + + private static final Logger LOGGER = LoggerFactory.getLogger(YarnPlugin.class); + + public YarnPlugin(PluginWrapper wrapper) { + super(wrapper); + } + + @Override + public void start() { + LOGGER.info("YarnPlugin started"); + } + + @Override + public void stop() { + LOGGER.info("YarnPlugin stopped"); + } +} diff --git a/zeppelin-plugins/notebookrepo/azure/pom.xml b/zeppelin-plugins/notebookrepo/azure/pom.xml index 3f7c03c221d..318637fead1 100644 --- a/zeppelin-plugins/notebookrepo/azure/pom.xml +++ b/zeppelin-plugins/notebookrepo/azure/pom.xml @@ -37,6 +37,7 @@ 2.1.4 4.0.0 NotebookRepo/AzureNotebookRepo + org.apache.zeppelin.notebook.repo.AzurePlugin @@ -44,33 +45,23 @@ com.microsoft.azure azure-data-lake-store-sdk ${adl.sdk.version} + + + org.slf4j + slf4j-api + + com.microsoft.azure azure-storage ${azure.storage.version} - - com.fasterxml.jackson.core - jackson-core - org.slf4j slf4j-api - - org.apache.commons - commons-lang3 - - - - - - maven-dependency-plugin - - - diff --git a/zeppelin-plugins/notebookrepo/azure/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java b/zeppelin-plugins/notebookrepo/azure/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java index 490f8189436..43c5c449a4f 100644 --- a/zeppelin-plugins/notebookrepo/azure/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java +++ b/zeppelin-plugins/notebookrepo/azure/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java @@ -38,18 +38,18 @@ import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.notebook.NoteInfo; import org.apache.zeppelin.user.AuthenticationInfo; +import org.pf4j.Extension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Azure storage backend for notebooks */ +@Extension public class AzureNotebookRepo implements NotebookRepo { private static final Logger LOGGER = LoggerFactory.getLogger(AzureNotebookRepo.class); private ZeppelinConfiguration conf; - private String user; - private String shareName; private CloudFileDirectory rootDir; public AzureNotebookRepo() { @@ -59,8 +59,8 @@ public AzureNotebookRepo() { @Override public void init(ZeppelinConfiguration conf) throws IOException { this.conf = conf; - user = conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_AZURE_USER); - shareName = conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_AZURE_SHARE); + String user = conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_AZURE_USER); + String shareName = conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_AZURE_SHARE); try { CloudStorageAccount account = CloudStorageAccount.parse( diff --git a/zeppelin-plugins/notebookrepo/azure/src/main/java/org/apache/zeppelin/notebook/repo/AzurePlugin.java b/zeppelin-plugins/notebookrepo/azure/src/main/java/org/apache/zeppelin/notebook/repo/AzurePlugin.java new file mode 100644 index 00000000000..5738356b300 --- /dev/null +++ b/zeppelin-plugins/notebookrepo/azure/src/main/java/org/apache/zeppelin/notebook/repo/AzurePlugin.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.notebook.repo; + +import org.pf4j.Plugin; +import org.pf4j.PluginWrapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class AzurePlugin extends Plugin { + private static final Logger LOGGER = LoggerFactory.getLogger(AzurePlugin.class); + + public AzurePlugin(PluginWrapper wrapper) { + super(wrapper); + } + + @Override + public void start() { + LOGGER.info("AzurePlugin started"); + } + + @Override + public void stop() { + LOGGER.info("AzurePlugin stopped"); + } +} diff --git a/zeppelin-plugins/notebookrepo/filesystem/pom.xml b/zeppelin-plugins/notebookrepo/filesystem/pom.xml index baa7ae743f6..eb70e5d0d21 100644 --- a/zeppelin-plugins/notebookrepo/filesystem/pom.xml +++ b/zeppelin-plugins/notebookrepo/filesystem/pom.xml @@ -36,18 +36,10 @@ 2.1.4 NotebookRepo/FileSystemNotebookRepo + org.apache.zeppelin.notebook.repo.FileSystemPlugin - - - - maven-dependency-plugin - - - - - hadoop2 diff --git a/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java b/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java index efa92eb19dd..2b7a05f94f3 100644 --- a/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java +++ b/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java @@ -23,6 +23,7 @@ import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.notebook.NoteInfo; import org.apache.zeppelin.user.AuthenticationInfo; +import org.pf4j.Extension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,6 +37,7 @@ * NotebookRepos for hdfs. * */ +@Extension public class FileSystemNotebookRepo implements NotebookRepo { private static final Logger LOGGER = LoggerFactory.getLogger(FileSystemNotebookRepo.class); diff --git a/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemPlugin.java b/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemPlugin.java new file mode 100644 index 00000000000..9642baefd97 --- /dev/null +++ b/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemPlugin.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.notebook.repo; + +import org.pf4j.Plugin; +import org.pf4j.PluginWrapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class FileSystemPlugin extends Plugin { + + private static final Logger LOGGER = LoggerFactory.getLogger(FileSystemPlugin.class); + + public FileSystemPlugin(PluginWrapper wrapper) { + super(wrapper); + } + + @Override + public void start() { + LOGGER.info("FileSystemPlugin started"); + } + + @Override + public void stop() { + LOGGER.info("FileSystemPlugin stopped"); + } +} diff --git a/zeppelin-plugins/notebookrepo/gcs/pom.xml b/zeppelin-plugins/notebookrepo/gcs/pom.xml index d95bf380208..063b9cf4cd9 100644 --- a/zeppelin-plugins/notebookrepo/gcs/pom.xml +++ b/zeppelin-plugins/notebookrepo/gcs/pom.xml @@ -38,6 +38,7 @@ 0.32.0-alpha 0.27 NotebookRepo/GCSNotebookRepo + org.apache.zeppelin.notebook.repo.GCSPlugin @@ -46,36 +47,6 @@ com.google.cloud google-cloud-storage ${gcs.storage.version} - - - com.google.code.findbugs - jsr305 - - - com.google.protobuf - protobuf-java - - - com.google.guava - guava - - - com.google.api - api-common - - - com.google.http-client - google-http-client-jackson2 - - - com.google.http-client - google-http-client - - - org.codehaus.jackson - jackson-core-asl - - @@ -83,16 +54,6 @@ google-cloud-nio ${google.testing.nio.version} test - - - com.google.code.findbugs - jsr305 - - - com.google.guava - guava - - @@ -100,49 +61,19 @@ truth ${google.truth.version} test - - - com.google.guava - guava - - com.google.api api-common 1.2.0 - - - com.google.guava - guava - - - com.google.code.findbugs - jsr305 - - com.google.http-client google-http-client-jackson2 1.23.0 - - - com.google.code.findbugs - jsr305 - - + - - - - - - maven-dependency-plugin - - - diff --git a/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java b/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java index 083f564b050..54171de1059 100644 --- a/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java +++ b/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java @@ -48,6 +48,7 @@ import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.notebook.NoteInfo; import org.apache.zeppelin.user.AuthenticationInfo; +import org.pf4j.Extension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,6 +66,7 @@ * @see * google-auth-library-java. */ +@Extension public class GCSNotebookRepo implements NotebookRepo { private static final Logger LOGGER = LoggerFactory.getLogger(GCSNotebookRepo.class); diff --git a/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSPlugin.java b/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSPlugin.java new file mode 100644 index 00000000000..df908e3186b --- /dev/null +++ b/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSPlugin.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.notebook.repo; + +import org.pf4j.Plugin; +import org.pf4j.PluginWrapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class GCSPlugin extends Plugin { + + private static final Logger LOGGER = LoggerFactory.getLogger(GCSPlugin.class); + + public GCSPlugin(PluginWrapper wrapper) { + super(wrapper); + } + + @Override + public void start() { + LOGGER.info("GCSPlugin started"); + } + + @Override + public void stop() { + LOGGER.info("GCSPlugin stopped"); + } +} diff --git a/zeppelin-plugins/notebookrepo/github/pom.xml b/zeppelin-plugins/notebookrepo/github/pom.xml index 37c4e087417..5469ac5fc9b 100644 --- a/zeppelin-plugins/notebookrepo/github/pom.xml +++ b/zeppelin-plugins/notebookrepo/github/pom.xml @@ -35,14 +35,6 @@ NotebookRepo/GitHubNotebookRepo + org.apache.zeppelin.notebook.repo.GithubPlugin - - - - - maven-dependency-plugin - - - - diff --git a/zeppelin-plugins/notebookrepo/github/src/main/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepo.java b/zeppelin-plugins/notebookrepo/github/src/main/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepo.java index 010d9985044..40381e47f3f 100644 --- a/zeppelin-plugins/notebookrepo/github/src/main/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepo.java +++ b/zeppelin-plugins/notebookrepo/github/src/main/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepo.java @@ -26,6 +26,7 @@ import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.transport.URIish; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; +import org.pf4j.Extension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,6 +48,7 @@ * You should be able to use this integration with all remote git repositories that accept * username + password authentication, not just GitHub. */ +@Extension public class GitHubNotebookRepo extends GitNotebookRepo { private static final Logger LOG = LoggerFactory.getLogger(GitHubNotebookRepo.class); private ZeppelinConfiguration zeppelinConfiguration; diff --git a/zeppelin-plugins/notebookrepo/github/src/main/java/org/apache/zeppelin/notebook/repo/GithubPlugin.java b/zeppelin-plugins/notebookrepo/github/src/main/java/org/apache/zeppelin/notebook/repo/GithubPlugin.java new file mode 100644 index 00000000000..94f684b7507 --- /dev/null +++ b/zeppelin-plugins/notebookrepo/github/src/main/java/org/apache/zeppelin/notebook/repo/GithubPlugin.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.notebook.repo; + +import org.pf4j.Plugin; +import org.pf4j.PluginWrapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class GithubPlugin extends Plugin { + + private static final Logger LOGGER = LoggerFactory.getLogger(GithubPlugin.class); + + public GithubPlugin(PluginWrapper wrapper) { + super(wrapper); + } + + @Override + public void start() { + LOGGER.info("GithubPlugin started"); + } + + @Override + public void stop() { + LOGGER.info("GithubPlugin stopped"); + } +} diff --git a/zeppelin-plugins/notebookrepo/mongo/pom.xml b/zeppelin-plugins/notebookrepo/mongo/pom.xml index 988348b78f3..3d3122d4010 100644 --- a/zeppelin-plugins/notebookrepo/mongo/pom.xml +++ b/zeppelin-plugins/notebookrepo/mongo/pom.xml @@ -37,6 +37,7 @@ NotebookRepo/MongoNotebookRepo + org.apache.zeppelin.notebook.repo.MongoPlugin @@ -50,25 +51,6 @@ de.flapdoodle.embed.mongo 3.2.4 test - - - org.apache.commons - commons-compress - - - org.apache.commons - commons-lang3 - - - - - - - maven-dependency-plugin - - - - diff --git a/zeppelin-plugins/notebookrepo/mongo/src/main/java/org/apache/zeppelin/notebook/repo/MongoNotebookRepo.java b/zeppelin-plugins/notebookrepo/mongo/src/main/java/org/apache/zeppelin/notebook/repo/MongoNotebookRepo.java index 8ed23a583d2..f71202a4618 100644 --- a/zeppelin-plugins/notebookrepo/mongo/src/main/java/org/apache/zeppelin/notebook/repo/MongoNotebookRepo.java +++ b/zeppelin-plugins/notebookrepo/mongo/src/main/java/org/apache/zeppelin/notebook/repo/MongoNotebookRepo.java @@ -24,6 +24,7 @@ import org.bson.Document; import org.bson.conversions.Bson; import org.bson.types.ObjectId; +import org.pf4j.Extension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; @@ -50,6 +51,7 @@ /** * Backend for storing Notebook on MongoDB. */ +@Extension public class MongoNotebookRepo implements NotebookRepo { private static final Logger LOG = LoggerFactory.getLogger(MongoNotebookRepo.class); diff --git a/zeppelin-plugins/notebookrepo/mongo/src/main/java/org/apache/zeppelin/notebook/repo/MongoPlugin.java b/zeppelin-plugins/notebookrepo/mongo/src/main/java/org/apache/zeppelin/notebook/repo/MongoPlugin.java new file mode 100644 index 00000000000..a681c448c97 --- /dev/null +++ b/zeppelin-plugins/notebookrepo/mongo/src/main/java/org/apache/zeppelin/notebook/repo/MongoPlugin.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.notebook.repo; + +import org.pf4j.Plugin; +import org.pf4j.PluginWrapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MongoPlugin extends Plugin { + + private static final Logger LOGGER = LoggerFactory.getLogger(MongoPlugin.class); + + public MongoPlugin(PluginWrapper wrapper) { + super(wrapper); + } + + @Override + public void start() { + LOGGER.info("MongoPlugin started"); + } + + @Override + public void stop() { + LOGGER.info("MongoPlugin stopped"); + } +} diff --git a/zeppelin-plugins/notebookrepo/oss/pom.xml b/zeppelin-plugins/notebookrepo/oss/pom.xml index d5f05950d4e..528362b97ed 100644 --- a/zeppelin-plugins/notebookrepo/oss/pom.xml +++ b/zeppelin-plugins/notebookrepo/oss/pom.xml @@ -36,6 +36,7 @@ 3.8.0 NotebookRepo/OSSNotebookRepo + org.apache.zeppelin.notebook.repo.AliyunPlugin @@ -45,12 +46,4 @@ ${oss.version} - - - - - maven-dependency-plugin - - - diff --git a/zeppelin-plugins/notebookrepo/oss/src/main/java/org/apache/zeppelin/notebook/repo/AliyunPlugin.java b/zeppelin-plugins/notebookrepo/oss/src/main/java/org/apache/zeppelin/notebook/repo/AliyunPlugin.java new file mode 100644 index 00000000000..a97f7e9ccc7 --- /dev/null +++ b/zeppelin-plugins/notebookrepo/oss/src/main/java/org/apache/zeppelin/notebook/repo/AliyunPlugin.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.notebook.repo; + +import org.pf4j.Plugin; +import org.pf4j.PluginWrapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class AliyunPlugin extends Plugin { + + private static final Logger LOGGER = LoggerFactory.getLogger(AliyunPlugin.class); + + public AliyunPlugin(PluginWrapper wrapper) { + super(wrapper); + } + + @Override + public void start() { + LOGGER.info("AliyunPlugin started"); + } + + @Override + public void stop() { + LOGGER.info("AliyunPlugin stopped"); + } +} diff --git a/zeppelin-plugins/notebookrepo/oss/src/main/java/org/apache/zeppelin/notebook/repo/OSSNotebookRepo.java b/zeppelin-plugins/notebookrepo/oss/src/main/java/org/apache/zeppelin/notebook/repo/OSSNotebookRepo.java index 5fddf9904bc..d025751660f 100644 --- a/zeppelin-plugins/notebookrepo/oss/src/main/java/org/apache/zeppelin/notebook/repo/OSSNotebookRepo.java +++ b/zeppelin-plugins/notebookrepo/oss/src/main/java/org/apache/zeppelin/notebook/repo/OSSNotebookRepo.java @@ -23,6 +23,7 @@ import org.apache.zeppelin.notebook.repo.storage.OSSOperator; import org.apache.zeppelin.notebook.repo.storage.RemoteStorageOperator; import org.apache.zeppelin.user.AuthenticationInfo; +import org.pf4j.Extension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,6 +36,7 @@ /** * NotebookRepo for Aliyun OSS (https://cn.aliyun.com/product/oss) */ +@Extension public class OSSNotebookRepo implements NotebookRepoWithVersionControl { private static final Logger LOGGER = LoggerFactory.getLogger(OSSNotebookRepo.class); diff --git a/zeppelin-plugins/notebookrepo/s3/pom.xml b/zeppelin-plugins/notebookrepo/s3/pom.xml index 3d102055b10..d67461b7a2b 100644 --- a/zeppelin-plugins/notebookrepo/s3/pom.xml +++ b/zeppelin-plugins/notebookrepo/s3/pom.xml @@ -36,6 +36,7 @@ 1.12.261 NotebookRepo/S3NotebookRepo + org.apache.zeppelin.notebook.repo.AWSPlugin @@ -140,12 +141,4 @@ - - - - - maven-dependency-plugin - - - diff --git a/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/AWSPlugin.java b/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/AWSPlugin.java new file mode 100644 index 00000000000..9fa8a92400c --- /dev/null +++ b/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/AWSPlugin.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.notebook.repo; + +import org.pf4j.Plugin; +import org.pf4j.PluginWrapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class AWSPlugin extends Plugin { + + private static final Logger LOGGER = LoggerFactory.getLogger(AWSPlugin.class); + + public AWSPlugin(PluginWrapper wrapper) { + super(wrapper); + } + + @Override + public void start() { + LOGGER.info("AWSPlugin started"); + } + + @Override + public void stop() { + LOGGER.info("AWSPlugin stopped"); + } +} diff --git a/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java b/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java index 497203c5faa..76e11698b96 100644 --- a/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java +++ b/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java @@ -36,6 +36,7 @@ import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.notebook.NoteInfo; import org.apache.zeppelin.user.AuthenticationInfo; +import org.pf4j.Extension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,6 +66,7 @@ /** * Backend for storing Notebooks on S3 */ +@Extension public class S3NotebookRepo implements NotebookRepo { private static final Logger LOGGER = LoggerFactory.getLogger(S3NotebookRepo.class); @@ -94,6 +96,7 @@ public S3NotebookRepo() { } + @Override public void init(ZeppelinConfiguration conf) throws IOException { this.conf = conf; bucketName = conf.getS3BucketName(); @@ -114,7 +117,7 @@ public void init(ZeppelinConfiguration conf) throws IOException { } ClientConfiguration cliConf = createClientConfiguration(); - + // see if we should be encrypting data in S3 String kmsKeyID = conf.getS3KMSKeyID(); if (kmsKeyID != null) { diff --git a/zeppelin-plugins/pom.xml b/zeppelin-plugins/pom.xml index befb391eb68..a6e259945d6 100644 --- a/zeppelin-plugins/pom.xml +++ b/zeppelin-plugins/pom.xml @@ -34,6 +34,9 @@ provided + + + @@ -58,17 +61,6 @@ zeppelin-zengine ${project.version} provided - - - com.fasterxml.jackson.core - jackson-core - - - - commons-logging - commons-logging - - @@ -86,70 +78,72 @@ - - - - maven-dependency-plugin - - - copy-plugin-dependencies - package - - copy-dependencies - - - ${project.basedir}/../../../plugins/${plugin.name} - false - false - true - runtime - - - - copy-plugin-artifact - package - - copy - - - ${project.basedir}/../../../plugins/${plugin.name} - false - false - true - - - ${project.groupId} - ${project.artifactId} - ${project.version} - ${project.packaging} - - - - - - - - maven-clean-plugin - - - delete-plugin-dir - - clean - - clean - - - - ${project.basedir}/../../../plugins/${plugin.name} - true - false - - - - - - - - + + + org.apache.maven.plugins + maven-enforcer-plugin + + + enforce + none + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + jar-with-dependencies + + ${project.artifactId}-plugin + ${project.basedir}/../../../plugins/ + false + false + + + true + true + + + ${plugin.name} + ${project.version} + ${plugin.class} + + + + + + make-assembly + package + + single + + + + + + maven-clean-plugin + + + delete-plugin-dir + + clean + + false + clean + + + + ${project.basedir}/../../../plugins + true + false + + + + + + + diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java index ae6846d5548..8b34f5e18f3 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java @@ -86,7 +86,8 @@ import org.apache.zeppelin.notebook.scheduler.NoSchedulerService; import org.apache.zeppelin.notebook.scheduler.QuartzSchedulerService; import org.apache.zeppelin.notebook.scheduler.SchedulerService; -import org.apache.zeppelin.plugin.PluginManager; +import org.apache.zeppelin.plugin.IPluginManager; +import org.apache.zeppelin.plugin.ZPluginManager; import org.apache.zeppelin.rest.exception.WebApplicationExceptionMapper; import org.apache.zeppelin.search.LuceneSearch; import org.apache.zeppelin.search.NoSearchService; @@ -176,6 +177,7 @@ public static void main(String[] args) throws IOException { @Override protected void configure() { Credentials credentials = new Credentials(conf); + bind(ZPluginManager.class).to(IPluginManager.class); bindAsContract(InterpreterFactory.class).in(Singleton.class); bindAsContract(NotebookRepoSync.class).to(NotebookRepo.class).in(Immediate.class); bindAsContract(Helium.class).in(Singleton.class); @@ -228,8 +230,11 @@ protected void configure() { initWebApp(defaultWebApp, conf, sharedServiceLocator, promMetricRegistry); initWebApp(nextWebApp, conf, sharedServiceLocator, promMetricRegistry); + IPluginManager pluginManager = ServiceLocatorUtilities.getService( + sharedServiceLocator, IPluginManager.class.getName()); + pluginManager.loadAndStartPlugins(); // Cluster Manager Server - setupClusterManagerServer(sharedServiceLocator, conf); + setupClusterManagerServer(sharedServiceLocator, conf, pluginManager); // JMX Enable if (conf.isJMXEnabled()) { @@ -438,7 +443,7 @@ private static void setupNotebookServer(WebAppContext webapp, ZeppelinConfigurat }); } - private static void setupClusterManagerServer(ServiceLocator serviceLocator, ZeppelinConfiguration conf) { + private static void setupClusterManagerServer(ServiceLocator serviceLocator, ZeppelinConfiguration conf, IPluginManager pluginManager) { if (conf.isClusterMode()) { LOG.info("Cluster mode is enabled, starting ClusterManagerServer"); ClusterManagerServer clusterManagerServer = ClusterManagerServer.getInstance(conf); @@ -462,7 +467,7 @@ private static void setupClusterManagerServer(ServiceLocator serviceLocator, Zep new Class[] {ZeppelinConfiguration.class, InterpreterSettingManager.class}, new Object[] {conf, intpSettingManager}); recoveryStorage.init(); - PluginManager.get().loadInterpreterLauncher(InterpreterSetting.CLUSTER_INTERPRETER_LAUNCHER_NAME, recoveryStorage); + pluginManager.createInterpreterLauncher(InterpreterSetting.CLUSTER_INTERPRETER_LAUNCHER_NAME, recoveryStorage); } catch (IOException e) { LOG.error(e.getMessage(), e); } diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/cluster/ZeppelinServerMock.java b/zeppelin-server/src/test/java/org/apache/zeppelin/cluster/ZeppelinServerMock.java index 4ea51ef2807..557aa901b41 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/cluster/ZeppelinServerMock.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/cluster/ZeppelinServerMock.java @@ -20,7 +20,6 @@ import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.interpreter.InterpreterSetting; import org.apache.zeppelin.notebook.Notebook; -import org.apache.zeppelin.plugin.PluginManager; import org.apache.zeppelin.rest.AbstractTestRestApi; import org.apache.zeppelin.server.ZeppelinServer; import org.apache.zeppelin.utils.TestUtils; @@ -176,7 +175,6 @@ protected static void shutDown(final boolean deleteConfDir) throws Exception { if (deleteConfDir) { FileUtils.deleteDirectory(confDir); } - PluginManager.reset(); ZeppelinConfiguration.reset(); } } diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java index 210d2fdf691..fdd73fe4048 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java @@ -43,7 +43,6 @@ import org.apache.http.impl.client.HttpClients; import org.apache.http.message.BasicNameValuePair; import org.apache.zeppelin.notebook.Notebook; -import org.apache.zeppelin.plugin.PluginManager; import org.apache.zeppelin.utils.TestUtils; import org.hamcrest.Description; import org.hamcrest.Matcher; diff --git a/zeppelin-zengine/pom.xml b/zeppelin-zengine/pom.xml index da50c7e6580..9c0d1e9673d 100644 --- a/zeppelin-zengine/pom.xml +++ b/zeppelin-zengine/pom.xml @@ -40,6 +40,7 @@ 1.3 2.6.0 4.5.4.201711221230-r + 3.6.0 @@ -76,6 +77,18 @@ slf4j-api + + org.pf4j + pf4j + ${pf4j.version} + + + org.slf4j + slf4j-api + + + + commons-io commons-io diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java index 9ec613cac82..4af34a605da 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java @@ -29,6 +29,8 @@ import com.google.gson.reflect.TypeToken; import com.google.gson.stream.JsonReader; import com.google.gson.stream.JsonWriter; +import javax.inject.Inject; + import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.apache.zeppelin.conf.ZeppelinConfiguration; @@ -45,8 +47,7 @@ import org.apache.zeppelin.interpreter.remote.RemoteInterpreter; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; -import org.apache.zeppelin.notebook.Note; -import org.apache.zeppelin.plugin.PluginManager; +import org.apache.zeppelin.plugin.IPluginManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -133,7 +134,11 @@ public class InterpreterSetting { private transient ApplicationEventListener appEventListener; private transient DependencyResolver dependencyResolver; - private transient ZeppelinConfiguration conf = ZeppelinConfiguration.create(); + @Inject + private transient ZeppelinConfiguration conf; + + @Inject + private transient IPluginManager pluginManager; private transient RecoveryStorage recoveryStorage; private transient RemoteInterpreterEventServer interpreterEventServer; @@ -202,6 +207,11 @@ public Builder setConf(ZeppelinConfiguration conf) { return this; } + public Builder setPluginManager(IPluginManager pluginManager) { + interpreterSetting.pluginManager = pluginManager; + return this; + } + public Builder setDependencyResolver(DependencyResolver dependencyResolver) { interpreterSetting.dependencyResolver = dependencyResolver; return this; @@ -288,10 +298,11 @@ public InterpreterSetting(InterpreterSetting o) { this.interpreterDir = o.getInterpreterDir(); this.interpreterRunner = o.getInterpreterRunner(); this.conf = o.getConf(); + this.pluginManager = o.getPluginManager(); } private InterpreterLauncher createLauncher(Properties properties) throws IOException { - return PluginManager.get().loadInterpreterLauncher( + return pluginManager.createInterpreterLauncher( getLauncherPlugin(properties), recoveryStorage); } @@ -654,11 +665,20 @@ public ZeppelinConfiguration getConf() { return conf; } + public IPluginManager getPluginManager() { + return pluginManager; + } + public InterpreterSetting setConf(ZeppelinConfiguration conf) { this.conf = conf; return this; } + public InterpreterSetting setPluginManager(IPluginManager pluginManager) { + this.pluginManager = pluginManager; + return this; + } + public List getDependencies() { return dependencies; } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java index 72d79a348d9..c0ee39d574a 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java @@ -61,6 +61,8 @@ import org.apache.zeppelin.notebook.Notebook; import org.apache.zeppelin.notebook.Paragraph; import org.apache.zeppelin.notebook.ParagraphTextParser; +import org.apache.zeppelin.plugin.IPluginManager; +import org.apache.zeppelin.plugin.ZPluginManager; import org.apache.zeppelin.resource.Resource; import org.apache.zeppelin.resource.ResourcePool; import org.apache.zeppelin.resource.ResourceSet; @@ -116,6 +118,7 @@ public class InterpreterSettingManager implements NoteEventListener, ClusterEven "editOnDblClick", false); private final ZeppelinConfiguration conf; + private final IPluginManager pluginManager; private final Path interpreterDirPath; /** @@ -155,13 +158,15 @@ public InterpreterSettingManager(ZeppelinConfiguration zeppelinConfiguration, AngularObjectRegistryListener angularObjectRegistryListener, RemoteInterpreterProcessListener remoteInterpreterProcessListener, - ApplicationEventListener appEventListener) + ApplicationEventListener appEventListener, + IPluginManager pluginManager) throws IOException { this(zeppelinConfiguration, new InterpreterOption(), angularObjectRegistryListener, remoteInterpreterProcessListener, appEventListener, - ConfigStorage.getInstance(zeppelinConfiguration)); + ConfigStorage.getInstance(zeppelinConfiguration), + pluginManager); } public InterpreterSettingManager(ZeppelinConfiguration conf, @@ -169,9 +174,11 @@ public InterpreterSettingManager(ZeppelinConfiguration conf, AngularObjectRegistryListener angularObjectRegistryListener, RemoteInterpreterProcessListener remoteInterpreterProcessListener, ApplicationEventListener appEventListener, - ConfigStorage configStorage) + ConfigStorage configStorage, + IPluginManager pluginManager) throws IOException { this.conf = conf; + this.pluginManager = pluginManager; this.defaultOption = defaultOption; this.interpreterDirPath = Paths.get(conf.getInterpreterDir()); LOGGER.debug("InterpreterRootPath: {}", interpreterDirPath); @@ -230,6 +237,7 @@ private void initInterpreterSetting(InterpreterSetting interpreterSetting) { .setDependencyResolver(dependencyResolver) .setRecoveryStorage(recoveryStorage) .setInterpreterEventServer(interpreterEventServer) + .setPluginManager(pluginManager) .postProcessing(); } @@ -532,6 +540,7 @@ private void registerInterpreterSetting(List registeredIn .setInterpreterDir(interpreterDir) .setRunner(runner) .setConf(conf) + .setPluginManager(pluginManager) .setIntepreterSettingManager(this) .create(); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterLauncher.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterLauncher.java similarity index 95% rename from zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterLauncher.java rename to zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterLauncher.java index 2e7b0d65653..754d301b7c7 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterLauncher.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterLauncher.java @@ -22,15 +22,17 @@ import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.interpreter.recovery.RecoveryStorage; +import org.pf4j.ExtensionPoint; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECTION_POOL_SIZE; /** + * * Component to Launch interpreter process. */ -public abstract class InterpreterLauncher { +public abstract class InterpreterLauncher implements ExtensionPoint { private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterLauncher.class); private static final String SPECIAL_CHARACTER="{}()<>&*‘|=?;[]$–#~!.\"%/\\:+,`"; @@ -39,7 +41,7 @@ public abstract class InterpreterLauncher { protected Properties properties; protected RecoveryStorage recoveryStorage; - public InterpreterLauncher(ZeppelinConfiguration zConf, RecoveryStorage recoveryStorage) { + public void init(ZeppelinConfiguration zConf, RecoveryStorage recoveryStorage) { this.zConf = zConf; this.recoveryStorage = recoveryStorage; } @@ -109,7 +111,6 @@ public InterpreterClient launch(InterpreterLaunchContext context) throws IOExcep // launch it via sub class implementation without recovering. return launchDirectly(context); } - /** * launch interpreter process directly without recovering. * @@ -118,5 +119,4 @@ public InterpreterClient launch(InterpreterLaunchContext context) throws IOExcep * @throws IOException */ public abstract InterpreterClient launchDirectly(InterpreterLaunchContext context) throws IOException; - } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java index 85b5fc6f2ac..8f32859ee9d 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java @@ -39,6 +39,7 @@ import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.interpreter.recovery.RecoveryStorage; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils; +import org.pf4j.Extension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,6 +48,7 @@ /** * Spark specific launcher. */ +@Extension public class SparkInterpreterLauncher extends StandardInterpreterLauncher { private static final Logger LOGGER = LoggerFactory.getLogger(SparkInterpreterLauncher.class); @@ -54,10 +56,6 @@ public class SparkInterpreterLauncher extends StandardInterpreterLauncher { private static final String DEFAULT_MASTER = "local[*]"; Optional sparkMaster = Optional.empty(); - public SparkInterpreterLauncher(ZeppelinConfiguration zConf, RecoveryStorage recoveryStorage) { - super(zConf, recoveryStorage); - } - @Override public Map buildEnvFromProperties(InterpreterLaunchContext context) throws IOException { Map env = super.buildEnvFromProperties(context); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncher.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncher.java index 46caee95f02..2af5b933026 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncher.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncher.java @@ -27,6 +27,7 @@ import org.apache.zeppelin.interpreter.remote.ExecRemoteInterpreterProcess; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterRunningProcess; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils; +import org.pf4j.Extension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,12 +38,15 @@ /** * Interpreter Launcher which use shell script to launch the interpreter process. */ +@Extension public class StandardInterpreterLauncher extends InterpreterLauncher { private static final Logger LOGGER = LoggerFactory.getLogger(StandardInterpreterLauncher.class); - public StandardInterpreterLauncher(ZeppelinConfiguration zConf, RecoveryStorage recoveryStorage) { - super(zConf, recoveryStorage); + @Override + public void init(ZeppelinConfiguration zConf, RecoveryStorage recoveryStorage) { + this.zConf = zConf; + this.recoveryStorage = recoveryStorage; } @Override diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/recovery/StopInterpreter.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/recovery/StopInterpreter.java index ee0ceccd0ef..84dc291d2bc 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/recovery/StopInterpreter.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/recovery/StopInterpreter.java @@ -21,6 +21,8 @@ import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.interpreter.InterpreterSettingManager; import org.apache.zeppelin.interpreter.launcher.InterpreterClient; +import org.apache.zeppelin.plugin.IPluginManager; +import org.apache.zeppelin.plugin.ZPluginManager; import org.apache.zeppelin.util.ReflectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,8 +42,10 @@ public class StopInterpreter { public static void main(String[] args) throws IOException { ZeppelinConfiguration zConf = ZeppelinConfiguration.create(); + IPluginManager pluginManager = new ZPluginManager(zConf); + pluginManager.loadAndStartPlugins(); InterpreterSettingManager interpreterSettingManager = - new InterpreterSettingManager(zConf, null, null, null); + new InterpreterSettingManager(zConf, null, null, null, pluginManager); RecoveryStorage recoveryStorage = ReflectionUtils.createClazzInstance(zConf.getRecoveryStorageClass(), new Class[] {ZeppelinConfiguration.class, InterpreterSettingManager.class}, diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java index 8ad7cfed37f..599b0ee75f7 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java @@ -32,6 +32,7 @@ import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.treewalk.filter.PathFilter; +import org.pf4j.Extension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,6 +52,7 @@ * * TODO(bzz): add default .gitignore */ +@Extension public class GitNotebookRepo extends VFSNotebookRepo implements NotebookRepoWithVersionControl { private static final Logger LOGGER = LoggerFactory.getLogger(GitNotebookRepo.class); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/InMemoryNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/InMemoryNotebookRepo.java index 1db4fb863ad..646dea6de39 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/InMemoryNotebookRepo.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/InMemoryNotebookRepo.java @@ -22,6 +22,7 @@ import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.notebook.NoteInfo; import org.apache.zeppelin.user.AuthenticationInfo; +import org.pf4j.Extension; import java.io.IOException; import java.util.Collections; @@ -29,6 +30,7 @@ import java.util.List; import java.util.Map; +@Extension public class InMemoryNotebookRepo implements NotebookRepo { private Map notes = new HashMap<>(); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepo.java index bae5cba8dd3..024758f9c63 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepo.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepo.java @@ -22,6 +22,7 @@ import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.notebook.NoteInfo; import org.apache.zeppelin.user.AuthenticationInfo; +import org.pf4j.ExtensionPoint; import java.io.IOException; import java.util.List; @@ -30,7 +31,7 @@ /** * Notebook repository (persistence layer) abstraction. */ -public interface NotebookRepo { +public interface NotebookRepo extends ExtensionPoint { void init(ZeppelinConfiguration zConf) throws IOException; diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java index 186aa61321a..ab0ce786785 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java @@ -21,11 +21,9 @@ import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.notebook.NoteInfo; -import org.apache.zeppelin.notebook.OldNoteInfo; import org.apache.zeppelin.notebook.Paragraph; -import org.apache.zeppelin.plugin.PluginManager; +import org.apache.zeppelin.plugin.IPluginManager; import org.apache.zeppelin.user.AuthenticationInfo; -import org.apache.zeppelin.util.Util; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,12 +51,15 @@ public class NotebookRepoSync implements NotebookRepoWithVersionControl { private List repos = new ArrayList<>(); private boolean oneWaySync; + private final IPluginManager pluginManager; + /** * @param conf */ @SuppressWarnings("static-access") @Inject - public NotebookRepoSync(ZeppelinConfiguration conf) throws IOException { + public NotebookRepoSync(ZeppelinConfiguration conf, IPluginManager pluginManager) throws IOException { + this.pluginManager = pluginManager; init(conf); } @@ -79,7 +80,7 @@ public void init(ZeppelinConfiguration conf) throws IOException { // init the underlying NotebookRepo for (int i = 0; i < Math.min(storageClassNames.length, getMaxRepoNum()); i++) { - NotebookRepo notebookRepo = PluginManager.get().loadNotebookRepo(storageClassNames[i].trim()); + NotebookRepo notebookRepo = pluginManager.createNotebookRepo(storageClassNames[i].trim()); notebookRepo.init(conf); repos.add(notebookRepo); } @@ -87,7 +88,7 @@ public void init(ZeppelinConfiguration conf) throws IOException { // couldn't initialize any storage, use default if (getRepoCount() == 0) { LOGGER.info("No storage could be initialized, using default {} storage", DEFAULT_STORAGE); - NotebookRepo defaultNotebookRepo = PluginManager.get().loadNotebookRepo(DEFAULT_STORAGE); + NotebookRepo defaultNotebookRepo = pluginManager.createNotebookRepo(DEFAULT_STORAGE); defaultNotebookRepo.init(conf); repos.add(defaultNotebookRepo); } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java index 7ed73f4942b..95923b25f7f 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java @@ -39,12 +39,14 @@ import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.notebook.NoteInfo; import org.apache.zeppelin.user.AuthenticationInfo; +import org.pf4j.Extension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * NotebookRepo implementation based on apache vfs */ +@Extension public class VFSNotebookRepo implements NotebookRepo { private static final Logger LOGGER = LoggerFactory.getLogger(VFSNotebookRepo.class); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/plugin/PluginManagerTest.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/IPluginManager.java similarity index 63% rename from zeppelin-zengine/src/test/java/org/apache/zeppelin/plugin/PluginManagerTest.java rename to zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/IPluginManager.java index d2d54a9abbe..adaf5d33059 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/plugin/PluginManagerTest.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/IPluginManager.java @@ -17,21 +17,23 @@ package org.apache.zeppelin.plugin; -import org.apache.zeppelin.notebook.repo.GitNotebookRepo; -import org.apache.zeppelin.notebook.repo.NotebookRepo; -import org.junit.Test; - import java.io.IOException; -import static org.junit.Assert.assertTrue; - +import org.apache.zeppelin.interpreter.launcher.InterpreterLauncher; +import org.apache.zeppelin.interpreter.recovery.RecoveryStorage; +import org.apache.zeppelin.notebook.repo.NotebookRepo; -public class PluginManagerTest { +/** + * + * Interface a PluginManager + * + * For test use the default {@link ZPluginManager} + * + */ +public interface IPluginManager { + public void loadAndStartPlugins(); + public NotebookRepo createNotebookRepo(String notebookRepoClassName) throws IOException; - @Test - public void testLoadGitNotebookRepo() throws IOException { - NotebookRepo notebookRepo = PluginManager.get() - .loadNotebookRepo("org.apache.zeppelin.notebook.repo.GitNotebookRepo"); - assertTrue(notebookRepo instanceof GitNotebookRepo); - } + public InterpreterLauncher createInterpreterLauncher(String launcherPlugin, + RecoveryStorage recoveryStorage) throws IOException; } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/PluginManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/PluginManager.java deleted file mode 100644 index 7cdc915f18c..00000000000 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/PluginManager.java +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zeppelin.plugin; - -import com.google.common.annotations.VisibleForTesting; -import org.apache.zeppelin.conf.ZeppelinConfiguration; -import org.apache.zeppelin.interpreter.launcher.InterpreterLauncher; -import org.apache.zeppelin.interpreter.launcher.SparkInterpreterLauncher; -import org.apache.zeppelin.interpreter.launcher.StandardInterpreterLauncher; -import org.apache.zeppelin.interpreter.recovery.RecoveryStorage; -import org.apache.zeppelin.notebook.repo.GitNotebookRepo; -import org.apache.zeppelin.notebook.repo.NotebookRepo; -import org.apache.zeppelin.notebook.repo.VFSNotebookRepo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.lang.reflect.InvocationTargetException; -import java.net.URL; -import java.net.URLClassLoader; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Class for loading Plugins. It is singleton and factory class. - * - */ -public class PluginManager { - private static final Logger LOGGER = LoggerFactory.getLogger(PluginManager.class); - - private static PluginManager instance; - - private ZeppelinConfiguration zConf = ZeppelinConfiguration.create(); - private String pluginsDir = zConf.getPluginsDir(); - - private Map cachedLaunchers = new HashMap<>(); - - private List builtinLauncherClassNames = Arrays.asList( - StandardInterpreterLauncher.class.getName(), - SparkInterpreterLauncher.class.getName()); - private List builtinNotebookRepoClassNames = Arrays.asList( - VFSNotebookRepo.class.getName(), - GitNotebookRepo.class.getName()); - - public static synchronized PluginManager get() { - if (instance == null) { - instance = new PluginManager(); - } - return instance; - } - - public NotebookRepo loadNotebookRepo(String notebookRepoClassName) throws IOException { - LOGGER.info("Loading NotebookRepo Plugin: {}", notebookRepoClassName); - if (builtinNotebookRepoClassNames.contains(notebookRepoClassName) || - Boolean.parseBoolean(System.getProperty("zeppelin.isTest", "false"))) { - try { - return (NotebookRepo) (Class.forName(notebookRepoClassName).newInstance()); - } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) { - throw new IOException("Fail to instantiate notebookrepo from classpath directly:" - + notebookRepoClassName, e); - } - } - - String simpleClassName = notebookRepoClassName.substring(notebookRepoClassName.lastIndexOf(".") + 1); - URLClassLoader pluginClassLoader = getPluginClassLoader(pluginsDir, "NotebookRepo", simpleClassName); - if (pluginClassLoader == null) { - return null; - } - NotebookRepo notebookRepo = null; - try { - notebookRepo = (NotebookRepo) (Class.forName(notebookRepoClassName, true, pluginClassLoader)).newInstance(); - } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) { - throw new IOException("Fail to instantiate notebookrepo " + notebookRepoClassName + - " from plugin classpath:" + pluginsDir, e); - } - - return notebookRepo; - } - - private String getOldNotebookRepoClassName(String notebookRepoClassName) { - int pos = notebookRepoClassName.lastIndexOf("."); - return notebookRepoClassName.substring(0, pos) + ".Old" + notebookRepoClassName.substring(pos + 1); - } - - public synchronized InterpreterLauncher loadInterpreterLauncher(String launcherPlugin, - RecoveryStorage recoveryStorage) - throws IOException { - - if (cachedLaunchers.containsKey(launcherPlugin)) { - return cachedLaunchers.get(launcherPlugin); - } - String launcherClassName = "org.apache.zeppelin.interpreter.launcher." + launcherPlugin; - LOGGER.info("Loading Interpreter Launcher Plugin: {}", launcherClassName); - - if (builtinLauncherClassNames.contains(launcherClassName) || - Boolean.parseBoolean(System.getProperty("zeppelin.isTest", "false"))) { - try { - return (InterpreterLauncher) - (Class.forName(launcherClassName)) - .getConstructor(ZeppelinConfiguration.class, RecoveryStorage.class) - .newInstance(zConf, recoveryStorage); - } catch (InstantiationException | IllegalAccessException | ClassNotFoundException - | NoSuchMethodException | InvocationTargetException e) { - throw new IOException("Fail to instantiate InterpreterLauncher from classpath directly:" - + launcherClassName, e); - } - } - - URLClassLoader pluginClassLoader = getPluginClassLoader(pluginsDir, "Launcher", launcherPlugin); - InterpreterLauncher launcher = null; - try { - launcher = (InterpreterLauncher) (Class.forName(launcherClassName, true, pluginClassLoader)) - .getConstructor(ZeppelinConfiguration.class, RecoveryStorage.class) - .newInstance(zConf, recoveryStorage); - } catch (InstantiationException | IllegalAccessException | ClassNotFoundException - | NoSuchMethodException | InvocationTargetException e) { - throw new IOException("Fail to instantiate Launcher " + launcherPlugin + - " from plugin pluginDir: " + pluginsDir, e); - } - - cachedLaunchers.put(launcherPlugin, launcher); - return launcher; - } - - private URLClassLoader getPluginClassLoader(String pluginsDir, - String pluginType, - String pluginName) throws IOException { - - File pluginFolder = new File(pluginsDir + "/" + pluginType + "/" + pluginName); - if (!pluginFolder.exists() || pluginFolder.isFile()) { - LOGGER.warn("PluginFolder {} doesn't exist or is not a directory", pluginFolder.getAbsolutePath()); - return null; - } - List urls = new ArrayList<>(); - for (File file : pluginFolder.listFiles()) { - LOGGER.debug("Add file {} to classpath of plugin: {}", file.getAbsolutePath(), pluginName); - urls.add(file.toURI().toURL()); - } - if (urls.isEmpty()) { - LOGGER.warn("Can not load plugin {}, because the plugin folder {} is empty.", pluginName , pluginFolder); - return null; - } - return new URLClassLoader(urls.toArray(new URL[0])); - } - - @VisibleForTesting - public static void reset() { - instance = null; - } -} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/ZPluginManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/ZPluginManager.java new file mode 100644 index 00000000000..3f4cf09417f --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/ZPluginManager.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.plugin; + +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.interpreter.launcher.InterpreterLauncher; +import org.apache.zeppelin.interpreter.recovery.RecoveryStorage; +import org.apache.zeppelin.notebook.repo.NotebookRepo; +import org.pf4j.DefaultPluginManager; +import org.pf4j.PluginManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; + +import javax.inject.Inject; +import javax.inject.Singleton; + +/** + * Class for loading Plugins + */ +@Singleton +public class ZPluginManager implements IPluginManager { + private static final Logger LOGGER = LoggerFactory.getLogger(ZPluginManager.class); + + private final ZeppelinConfiguration zConf; + private final PluginManager pluginManager; + + @Inject + public ZPluginManager(ZeppelinConfiguration zConf) { + this.zConf = zConf; + Path plugin = Paths.get(zConf.getPluginsDir()); + LOGGER.info("Plugin-Dir: {}", plugin); + this.pluginManager = new DefaultPluginManager(plugin); + } + + @Override + public void loadAndStartPlugins() { + pluginManager.loadPlugins(); + pluginManager.startPlugins(); + LOGGER.info("InterpreterLauncher:"); + List interpreterLaunchers = pluginManager.getExtensions(InterpreterLauncher.class); + for (InterpreterLauncher interpreterLauncher : interpreterLaunchers) { + LOGGER.info(interpreterLauncher.getClass().getSimpleName()); + } + LOGGER.info("NotebookRepo:"); + List notebookRepos = pluginManager.getExtensions(NotebookRepo.class); + for (NotebookRepo notebookRepo : notebookRepos) { + LOGGER.info(notebookRepo.getClass().getName()); + } + } + + @Override + public NotebookRepo createNotebookRepo(String notebookRepoClassName) throws IOException { + LOGGER.info("Loading NotebookRepo Plugin: {}", notebookRepoClassName); + List notebookRepos = pluginManager.getExtensions(NotebookRepo.class); + for (NotebookRepo notebookRepo : notebookRepos) { + if (notebookRepoClassName.equals(notebookRepo.getClass().getName())) { + return notebookRepo; + } + } + throw new IOException("Fail to instantiate notebookrepo " + notebookRepoClassName + " from plugin"); + } + + @Override + public synchronized InterpreterLauncher createInterpreterLauncher(String launcherPlugin, + RecoveryStorage recoveryStorage) + throws IOException { + LOGGER.info("Loading NotebookRepo Plugin: {}", launcherPlugin); + List interpreterLaunchers = pluginManager.getExtensions(InterpreterLauncher.class); + for (InterpreterLauncher interpreterLauncher : interpreterLaunchers) { + if (launcherPlugin.equals(interpreterLauncher.getClass().getSimpleName())) { + interpreterLauncher.init(zConf, recoveryStorage); + return interpreterLauncher; + } + } + throw new IOException("Fail to instantiate launcher " + launcherPlugin + " from plugin"); + } + +} diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/AbstractInterpreterTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/AbstractInterpreterTest.java index df1c6ec1393..89c7b12bd04 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/AbstractInterpreterTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/AbstractInterpreterTest.java @@ -29,6 +29,8 @@ import org.apache.zeppelin.notebook.Notebook; import org.apache.zeppelin.notebook.repo.InMemoryNotebookRepo; import org.apache.zeppelin.notebook.repo.NotebookRepo; +import org.apache.zeppelin.plugin.IPluginManager; +import org.apache.zeppelin.plugin.ZPluginManager; import org.apache.zeppelin.search.LuceneSearch; import org.apache.zeppelin.search.SearchService; import org.apache.zeppelin.user.Credentials; @@ -61,6 +63,7 @@ public abstract class AbstractInterpreterTest { protected File confDir; protected File notebookDir; protected ZeppelinConfiguration conf; + protected IPluginManager pluginManager; @Before public void setUp() throws Exception { @@ -86,11 +89,13 @@ public void setUp() throws Exception { System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_GROUP_DEFAULT.getVarName(), "test"); conf = ZeppelinConfiguration.create(); + pluginManager = new ZPluginManager(conf); + pluginManager.loadAndStartPlugins(); NotebookRepo notebookRepo = new InMemoryNotebookRepo(); NoteManager noteManager = new NoteManager(notebookRepo, conf); AuthorizationService authorizationService = new AuthorizationService(noteManager, conf); interpreterSettingManager = new InterpreterSettingManager(conf, - mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class)); + mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class), pluginManager); interpreterFactory = new InterpreterFactory(interpreterSettingManager); Credentials credentials = new Credentials(conf); notebook = new Notebook(conf, authorizationService, notebookRepo, noteManager, interpreterFactory, interpreterSettingManager, credentials); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerTest.java index def1000ff71..e40003be387 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerTest.java @@ -87,7 +87,7 @@ public void testInitInterpreterSettingManager() throws IOException, RepositoryEx // Load it again InterpreterSettingManager interpreterSettingManager2 = new InterpreterSettingManager(conf, - mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class)); + mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class), pluginManager); assertEquals(6, interpreterSettingManager2.get().size()); interpreterSetting = interpreterSettingManager2.getByName("test"); assertEquals("test", interpreterSetting.getName()); @@ -145,7 +145,7 @@ public void testCreateUpdateRemoveSetting() throws IOException, InterpreterExcep // load it again, it should be saved in interpreter-setting.json. So we can restore it properly InterpreterSettingManager interpreterSettingManager2 = new InterpreterSettingManager(conf, - mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class)); + mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class), pluginManager); assertEquals(7, interpreterSettingManager2.get().size()); interpreterSetting = interpreterSettingManager2.getByName("test3"); assertEquals("test3", interpreterSetting.getName()); @@ -197,7 +197,7 @@ public void testCreateUpdateRemoveSetting() throws IOException, InterpreterExcep // load it again InterpreterSettingManager interpreterSettingManager3 = new InterpreterSettingManager(ZeppelinConfiguration.create(), - mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class)); + mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class), pluginManager); assertEquals(6, interpreterSettingManager3.get().size()); } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingTest.java index de2f41c97db..7e4200ac640 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingTest.java @@ -78,6 +78,7 @@ public void testCreateInterpreters() { .setInterpreterInfos(interpreterInfos) .setOption(interpreterOption) .setIntepreterSettingManager(interpreterSettingManager) + .setConf(conf) .create(); // create default interpreter for user1 and note1 @@ -112,6 +113,7 @@ public void testSharedMode() { .setInterpreterInfos(interpreterInfos) .setOption(interpreterOption) .setIntepreterSettingManager(interpreterSettingManager) + .setConf(conf) .create(); // create default interpreter for user1 and note1 @@ -152,6 +154,7 @@ public void testPerUserScopedMode() { .setInterpreterInfos(interpreterInfos) .setOption(interpreterOption) .setIntepreterSettingManager(interpreterSettingManager) + .setConf(conf) .create(); // create interpreter for user1 and note1 @@ -192,6 +195,7 @@ public void testPerNoteScopedMode() { .setInterpreterInfos(interpreterInfos) .setOption(interpreterOption) .setIntepreterSettingManager(interpreterSettingManager) + .setConf(conf) .create(); // create interpreter for user1 and note1 @@ -232,6 +236,7 @@ public void testPerUserIsolatedMode() { .setInterpreterInfos(interpreterInfos) .setOption(interpreterOption) .setIntepreterSettingManager(interpreterSettingManager) + .setConf(conf) .create(); // create interpreter for user1 and note1 @@ -273,6 +278,7 @@ public void testPerNoteIsolatedMode() { .setInterpreterInfos(interpreterInfos) .setOption(interpreterOption) .setIntepreterSettingManager(interpreterSettingManager) + .setConf(conf) .create(); // create interpreter for user1 and note1 @@ -315,6 +321,7 @@ public void testPerUserIsolatedPerNoteScopedMode() { .setInterpreterInfos(interpreterInfos) .setOption(interpreterOption) .setIntepreterSettingManager(interpreterSettingManager) + .setConf(conf) .create(); // create interpreter for user1 and note1 @@ -371,6 +378,7 @@ public void testPerUserIsolatedPerNoteIsolatedMode() { .setInterpreterInfos(interpreterInfos) .setOption(interpreterOption) .setIntepreterSettingManager(interpreterSettingManager) + .setConf(conf) .create(); // create interpreter for user1 and note1 @@ -433,6 +441,7 @@ public void testPerUserScopedPerNoteScopedMode() { .setInterpreterInfos(interpreterInfos) .setOption(interpreterOption) .setIntepreterSettingManager(interpreterSettingManager) + .setConf(conf) .create(); // create interpreter for user1 and note1 @@ -491,6 +500,7 @@ public void testInterpreterJsonSerializable() { .setGroup("group") .setInterpreterInfos(interpreterInfos) .setOption(interpreterOption) + .setConf(conf) .create(); String json = InterpreterSetting.toJson(interpreterSetting); @@ -525,6 +535,7 @@ public void testIsUserAuthorized() { .setName("id") .setGroup("group") .setOption(interpreterOption) + .setConf(conf) .create(); assertTrue(interpreterSetting.isUserAuthorized(userAndRoles)); @@ -539,6 +550,7 @@ public void testIsUserAuthorized() { .setName("id") .setGroup("group") .setOption(interpreterOption) + .setConf(conf) .create(); assertFalse(interpreterSetting.isUserAuthorized(userAndRoles)); @@ -552,6 +564,7 @@ public void testIsUserAuthorized() { .setName("id") .setGroup("group") .setOption(interpreterOption) + .setConf(conf) .create(); assertTrue(interpreterSetting.isUserAuthorized(userAndRoles)); } @@ -567,6 +580,7 @@ public void testLoadDependency() throws InterruptedException { .setOption(interpreterOption) .setIntepreterSettingManager(interpreterSettingManager) .setDependencyResolver(new DependencyResolver("/tmp")) + .setConf(conf) .create(); // set invalid dependency diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroupTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroupTest.java index fd417cf6da3..6660ae2f6ce 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroupTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroupTest.java @@ -19,6 +19,7 @@ import org.junit.Before; import org.junit.Test; +import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.eclipse.aether.RepositoryException; import java.io.IOException; @@ -44,12 +45,14 @@ public void setUp() throws IOException, RepositoryException { List interpreterInfos = new ArrayList<>(); interpreterInfos.add(interpreterInfo1); interpreterInfos.add(interpreterInfo2); + ZeppelinConfiguration zconf = ZeppelinConfiguration.create(); interpreterSetting = new InterpreterSetting.Builder() .setId("id") .setName("test") .setGroup("test") .setInterpreterInfos(interpreterInfos) .setOption(interpreterOption) + .setConf(zconf) .create(); } diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/launcher/InterpreterLauncherTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/InterpreterLauncherTest.java similarity index 100% rename from zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/launcher/InterpreterLauncherTest.java rename to zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/InterpreterLauncherTest.java diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java index b1d13da736d..0caa615da5c 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java @@ -22,6 +22,8 @@ import org.apache.zeppelin.interpreter.InterpreterOption; import org.apache.zeppelin.interpreter.integration.DownloadUtils; import org.apache.zeppelin.interpreter.remote.ExecRemoteInterpreterProcess; +import org.apache.zeppelin.plugin.IPluginManager; +import org.apache.zeppelin.plugin.ZPluginManager; import org.apache.zeppelin.util.Util; import org.junit.Before; import org.junit.Test; @@ -45,9 +47,13 @@ public class SparkInterpreterLauncherTest { private String sparkHome; private String zeppelinHome; + private ZeppelinConfiguration zConf; + private IPluginManager pluginManager; @Before public void setUp() { + zConf = ZeppelinConfiguration.create(); + pluginManager = new ZPluginManager(zConf); for (final ZeppelinConfiguration.ConfVars confVar : ZeppelinConfiguration.ConfVars.values()) { System.clearProperty(confVar.getVarName()); } @@ -62,8 +68,7 @@ public void setUp() { @Test public void testConnectTimeOut() throws IOException { - ZeppelinConfiguration zConf = ZeppelinConfiguration.create(); - SparkInterpreterLauncher launcher = new SparkInterpreterLauncher(zConf, null); + InterpreterLauncher launcher = pluginManager.createInterpreterLauncher("SparkInterpreterLauncher", null); Properties properties = new Properties(); properties.setProperty("SPARK_HOME", sparkHome); properties.setProperty( @@ -85,8 +90,7 @@ public void testConnectTimeOut() throws IOException { @Test public void testLocalMode() throws IOException { - ZeppelinConfiguration zConf = ZeppelinConfiguration.create(); - SparkInterpreterLauncher launcher = new SparkInterpreterLauncher(zConf, null); + InterpreterLauncher launcher = pluginManager.createInterpreterLauncher("SparkInterpreterLauncher", null); Properties properties = new Properties(); properties.setProperty("SPARK_HOME", sparkHome); properties.setProperty("ENV_1", ""); @@ -114,8 +118,7 @@ public void testLocalMode() throws IOException { @Test public void testYarnClientMode_1() throws IOException { - ZeppelinConfiguration zConf = ZeppelinConfiguration.create(); - SparkInterpreterLauncher launcher = new SparkInterpreterLauncher(zConf, null); + InterpreterLauncher launcher = pluginManager.createInterpreterLauncher("SparkInterpreterLauncher", null); Properties properties = new Properties(); properties.setProperty("SPARK_HOME", sparkHome); properties.setProperty("property_1", "value_1"); @@ -146,8 +149,7 @@ public void testYarnClientMode_1() throws IOException { @Test public void testYarnClientMode_2() throws IOException { - ZeppelinConfiguration zConf = ZeppelinConfiguration.create(); - SparkInterpreterLauncher launcher = new SparkInterpreterLauncher(zConf, null); + InterpreterLauncher launcher = pluginManager.createInterpreterLauncher("SparkInterpreterLauncher", null); Properties properties = new Properties(); properties.setProperty("SPARK_HOME", sparkHome); properties.setProperty("property_1", "value_1"); @@ -180,8 +182,7 @@ public void testYarnClientMode_2() throws IOException { @Test public void testYarnClusterMode_1() throws IOException { - ZeppelinConfiguration zConf = ZeppelinConfiguration.create(); - SparkInterpreterLauncher launcher = new SparkInterpreterLauncher(zConf, null); + InterpreterLauncher launcher = pluginManager.createInterpreterLauncher("SparkInterpreterLauncher", null); Properties properties = new Properties(); properties.setProperty("SPARK_HOME", sparkHome); properties.setProperty("property_1", "value_1"); @@ -220,8 +221,7 @@ public void testYarnClusterMode_1() throws IOException { @Test public void testYarnClusterMode_2() throws IOException { - ZeppelinConfiguration zConf = ZeppelinConfiguration.create(); - SparkInterpreterLauncher launcher = new SparkInterpreterLauncher(zConf, null); + InterpreterLauncher launcher = pluginManager.createInterpreterLauncher("SparkInterpreterLauncher", null); Properties properties = new Properties(); properties.setProperty("SPARK_HOME", sparkHome); properties.setProperty("property_1", "value_1"); @@ -268,8 +268,7 @@ public void testYarnClusterMode_2() throws IOException { @Test public void testYarnClusterMode_3() throws IOException { - ZeppelinConfiguration zConf = ZeppelinConfiguration.create(); - SparkInterpreterLauncher launcher = new SparkInterpreterLauncher(zConf, null); + InterpreterLauncher launcher = pluginManager.createInterpreterLauncher("SparkInterpreterLauncher", null); Properties properties = new Properties(); properties.setProperty("SPARK_HOME", sparkHome); properties.setProperty("property_1", "value_1"); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncherTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncherTest.java index 7321fe42693..184684d3863 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncherTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncherTest.java @@ -20,6 +20,8 @@ import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.interpreter.InterpreterOption; import org.apache.zeppelin.interpreter.remote.ExecRemoteInterpreterProcess; +import org.apache.zeppelin.plugin.IPluginManager; +import org.apache.zeppelin.plugin.ZPluginManager; import org.junit.Before; import org.junit.Test; @@ -30,8 +32,14 @@ import static org.junit.Assert.assertTrue; public class StandardInterpreterLauncherTest { + + private ZeppelinConfiguration zConf; + private IPluginManager pluginManager; + @Before public void setUp() { + zConf = ZeppelinConfiguration.create(); + pluginManager = new ZPluginManager(zConf); for (final ZeppelinConfiguration.ConfVars confVar : ZeppelinConfiguration.ConfVars.values()) { System.clearProperty(confVar.getVarName()); } @@ -39,8 +47,7 @@ public void setUp() { @Test public void testLauncher() throws IOException { - ZeppelinConfiguration zConf = ZeppelinConfiguration.create(); - StandardInterpreterLauncher launcher = new StandardInterpreterLauncher(zConf, null); + InterpreterLauncher launcher = pluginManager.createInterpreterLauncher("StandardInterpreterLauncher", null); Properties properties = new Properties(); properties.setProperty("ENV_1", "VALUE_1"); properties.setProperty("property_1", "value_1"); @@ -64,8 +71,7 @@ public void testLauncher() throws IOException { @Test public void testConnectTimeOut() throws IOException { - ZeppelinConfiguration zConf = ZeppelinConfiguration.create(); - StandardInterpreterLauncher launcher = new StandardInterpreterLauncher(zConf, null); + InterpreterLauncher launcher = pluginManager.createInterpreterLauncher("StandardInterpreterLauncher", null); Properties properties = new Properties(); properties.setProperty( ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT.getVarName(), "10000"); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncInitializationTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncInitializationTest.java index 655b601fafd..d087cca8097 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncInitializationTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncInitializationTest.java @@ -20,6 +20,8 @@ import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; import org.apache.zeppelin.notebook.repo.mock.VFSNotebookRepoMock; +import org.apache.zeppelin.plugin.IPluginManager; +import org.apache.zeppelin.plugin.ZPluginManager; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -45,10 +47,15 @@ public class NotebookRepoSyncInitializationTest { private String unsupportedStorageConf = validFirstStorageClass + "," + validSecondStorageClass + "," + validSecondStorageClass; private String emptyStorageConf = ""; + private ZeppelinConfiguration conf; + private IPluginManager pluginManager; + @Before public void setUp(){ System.setProperty(ConfVars.ZEPPELIN_PLUGINS_DIR.getVarName(), new File("../../../plugins").getAbsolutePath()); System.setProperty("zeppelin.isTest", "true"); + conf = ZeppelinConfiguration.create(); + pluginManager = new ZPluginManager(conf); } @After @@ -61,11 +68,10 @@ public void validInitOneStorageTest() throws IOException { // no need to initialize folder due to one storage // set confs System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), validOneStorageConf); - ZeppelinConfiguration conf = ZeppelinConfiguration.create(); // create repo - NotebookRepoSync notebookRepoSync = new NotebookRepoSync(conf); + NotebookRepoSync notebookRepoSync = new NotebookRepoSync(conf, pluginManager); // check proper initialization of one storage - assertEquals(notebookRepoSync.getRepoCount(), 1); + assertEquals(1, notebookRepoSync.getRepoCount()); assertTrue(notebookRepoSync.getRepo(0) instanceof VFSNotebookRepo); } @@ -89,7 +95,7 @@ public void validInitTwoStorageTest() throws IOException { System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), validTwoStorageConf); ZeppelinConfiguration conf = ZeppelinConfiguration.create(); // create repo - NotebookRepoSync notebookRepoSync = new NotebookRepoSync(conf); + NotebookRepoSync notebookRepoSync = new NotebookRepoSync(conf, pluginManager); // check that both initialized assertEquals(notebookRepoSync.getRepoCount(), 2); assertTrue(notebookRepoSync.getRepo(0) instanceof VFSNotebookRepo); @@ -103,11 +109,11 @@ public void invalidInitTwoStorageTest() throws IOException { ZeppelinConfiguration conf = ZeppelinConfiguration.create(); // create repo try { - NotebookRepoSync notebookRepoSync = new NotebookRepoSync(conf); + NotebookRepoSync notebookRepoSync = new NotebookRepoSync(conf, pluginManager); fail("Should throw exception due to invalid NotebookRepo"); } catch (IOException e) { LOGGER.error(e.getMessage()); - assertTrue(e.getMessage().contains("Fail to instantiate notebookrepo from classpath directly")); + assertTrue(e.getMessage().contains("Fail to instantiate notebookrepo org.apache.zeppelin.notebook.repo.DummyNotebookRepo from plugin")); } } @@ -131,7 +137,7 @@ public void initUnsupportedNumberStoragesTest() throws IOException { System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), unsupportedStorageConf); ZeppelinConfiguration conf = ZeppelinConfiguration.create(); // create repo - NotebookRepoSync notebookRepoSync = new NotebookRepoSync(conf); + NotebookRepoSync notebookRepoSync = new NotebookRepoSync(conf, pluginManager); // check that first two storages initialized instead of three assertEquals(notebookRepoSync.getRepoCount(), 2); assertTrue(notebookRepoSync.getRepo(0) instanceof VFSNotebookRepo); @@ -144,7 +150,7 @@ public void initEmptyStorageTest() throws IOException { System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), emptyStorageConf); ZeppelinConfiguration conf = ZeppelinConfiguration.create(); // create repo - NotebookRepoSync notebookRepoSync = new NotebookRepoSync(conf); + NotebookRepoSync notebookRepoSync = new NotebookRepoSync(conf, pluginManager); // check initialization of one default storage assertEquals(notebookRepoSync.getRepoCount(), 1); assertTrue(notebookRepoSync.getRepo(0) instanceof NotebookRepoWithVersionControl); @@ -156,11 +162,11 @@ public void initOneDummyStorageTest() { ZeppelinConfiguration conf = ZeppelinConfiguration.create(); // create repo try { - NotebookRepoSync notebookRepoSync = new NotebookRepoSync(conf); + NotebookRepoSync notebookRepoSync = new NotebookRepoSync(conf, pluginManager); fail("Should throw exception due to invalid NotebookRepo"); } catch (IOException e) { LOGGER.error(e.getMessage()); - assertTrue(e.getMessage().contains("Fail to instantiate notebookrepo from classpath directly")); + assertTrue(e.getMessage().contains("Fail to instantiate notebookrepo org.apache.zeppelin.notebook.repo.DummyNotebookRepo from plugin")); } } } \ No newline at end of file diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java index 18ed8b63011..a972d03249b 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java @@ -41,6 +41,8 @@ import org.apache.zeppelin.notebook.NoteManager; import org.apache.zeppelin.notebook.Notebook; import org.apache.zeppelin.notebook.Paragraph; +import org.apache.zeppelin.plugin.IPluginManager; +import org.apache.zeppelin.plugin.ZPluginManager; import org.apache.zeppelin.storage.ConfigStorage; import org.apache.zeppelin.user.AuthenticationInfo; import org.apache.zeppelin.user.Credentials; @@ -55,6 +57,7 @@ public class NotebookRepoSyncTest { private File ZEPPELIN_HOME; private ZeppelinConfiguration conf; + private IPluginManager pluginManager; private File mainNotebookDir; private File secNotebookDir; private Notebook notebook; @@ -84,19 +87,20 @@ public void setUp() throws Exception { System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), "org.apache.zeppelin.notebook.repo.VFSNotebookRepo,org.apache.zeppelin.notebook.repo.mock.VFSNotebookRepoMock"); System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC.getVarName(), "false"); System.setProperty(ConfVars.ZEPPELIN_CONFIG_FS_DIR.getVarName(), ZEPPELIN_HOME.getAbsolutePath() + "/conf"); - System.setProperty(ConfVars.ZEPPELIN_PLUGINS_DIR.getVarName(), new File("../../../plugins").getAbsolutePath()); LOG.info("main Note dir : " + mainNotePath); LOG.info("secondary note dir : " + secNotePath); conf = ZeppelinConfiguration.create(); + pluginManager = new ZPluginManager(conf); ConfigStorage.reset(); interpreterSettingManager = new InterpreterSettingManager(conf, - mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class)); + mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class), pluginManager); factory = new InterpreterFactory(interpreterSettingManager); - notebookRepoSync = new NotebookRepoSync(conf); + notebookRepoSync = new NotebookRepoSync(conf, pluginManager); + noteManager = new NoteManager(notebookRepoSync, conf); authorizationService = new AuthorizationService(noteManager, conf); credentials = new Credentials(conf); @@ -255,7 +259,7 @@ public void testOneWaySyncOnReloadedList() throws IOException, SchedulerExceptio System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), mainNotebookDir.getAbsolutePath()); System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC.getVarName(), "true"); conf = ZeppelinConfiguration.create(); - notebookRepoSync = new NotebookRepoSync(conf); + notebookRepoSync = new NotebookRepoSync(conf, pluginManager); notebook = new Notebook(conf, mock(AuthorizationService.class), notebookRepoSync, new NoteManager(notebookRepoSync, conf), factory, interpreterSettingManager, credentials, null); // check that both storage repos are empty @@ -302,7 +306,7 @@ public void testCheckpointOneStorage() throws IOException, SchedulerException { System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), "org.apache.zeppelin.notebook.repo.GitNotebookRepo"); ZeppelinConfiguration vConf = ZeppelinConfiguration.create(); - NotebookRepoSync vRepoSync = new NotebookRepoSync(vConf); + NotebookRepoSync vRepoSync = new NotebookRepoSync(vConf, pluginManager); Notebook vNotebookSync = new Notebook(vConf, mock(AuthorizationService.class), vRepoSync, new NoteManager(vRepoSync, conf), factory, interpreterSettingManager, credentials, null); // one git versioned storage initialized diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/mock/VFSNotebookRepoMock.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/mock/VFSNotebookRepoMock.java index 7f450dfd72e..ffcc43bc1be 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/mock/VFSNotebookRepoMock.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/mock/VFSNotebookRepoMock.java @@ -17,20 +17,22 @@ package org.apache.zeppelin.notebook.repo.mock; import org.apache.zeppelin.conf.ZeppelinConfiguration; -import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.notebook.repo.NotebookRepo; import org.apache.zeppelin.notebook.repo.VFSNotebookRepo; +import org.pf4j.Extension; import java.io.IOException; +@Extension(points = NotebookRepo.class) public class VFSNotebookRepoMock extends VFSNotebookRepo { public VFSNotebookRepoMock() { - String secNotebookDir = ZeppelinConfiguration.create().getNotebookDir() + "_secondary"; - System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), secNotebookDir); } + @Override public void init(ZeppelinConfiguration conf) throws IOException { - super.init(conf); + this.conf = conf; + setNotebookDirectory(conf.getNotebookDir() + "_secondary"); } } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/plugin/ZPluginManagerTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/plugin/ZPluginManagerTest.java new file mode 100644 index 00000000000..957920a9820 --- /dev/null +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/plugin/ZPluginManagerTest.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.plugin; + +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.interpreter.launcher.InterpreterLauncher; +import org.apache.zeppelin.interpreter.launcher.StandardInterpreterLauncher; +import org.apache.zeppelin.notebook.repo.GitNotebookRepo; +import org.apache.zeppelin.notebook.repo.NotebookRepo; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertTrue; + + +public class ZPluginManagerTest { + + private IPluginManager pluginManager; + + @Before + public void setUp() { + ZeppelinConfiguration zConf = ZeppelinConfiguration.create(); + pluginManager = new ZPluginManager(zConf); + pluginManager.loadAndStartPlugins(); + } + + @Test + public void testLoadGitNotebookRepo() throws IOException { + NotebookRepo notebookRepo = pluginManager.createNotebookRepo("org.apache.zeppelin.notebook.repo.GitNotebookRepo"); + assertTrue(notebookRepo instanceof GitNotebookRepo); + } + @Test + public void testStandardInterpreter() throws IOException { + InterpreterLauncher interpreterLauncher = pluginManager.createInterpreterLauncher("StandardInterpreterLauncher", null); + assertTrue(interpreterLauncher instanceof StandardInterpreterLauncher); + } +} From 0888f35d27393afa017679f53f9f8134ac4b7346 Mon Sep 17 00:00:00 2001 From: Philipp Dallig Date: Thu, 14 Apr 2022 10:26:38 +0200 Subject: [PATCH 2/9] fix tests --- .../main/java/org/apache/zeppelin/server/ZeppelinServer.java | 2 +- .../main/java/org/apache/zeppelin/plugin/ZPluginManager.java | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java index 8b34f5e18f3..1b0c48ae69e 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java @@ -177,7 +177,7 @@ public static void main(String[] args) throws IOException { @Override protected void configure() { Credentials credentials = new Credentials(conf); - bind(ZPluginManager.class).to(IPluginManager.class); + bind(ZPluginManager.class).to(IPluginManager.class).in(Singleton.class); bindAsContract(InterpreterFactory.class).in(Singleton.class); bindAsContract(NotebookRepoSync.class).to(NotebookRepo.class).in(Immediate.class); bindAsContract(Helium.class).in(Singleton.class); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/ZPluginManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/ZPluginManager.java index 3f4cf09417f..a37f0da3fb4 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/ZPluginManager.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/ZPluginManager.java @@ -32,12 +32,10 @@ import java.util.List; import javax.inject.Inject; -import javax.inject.Singleton; /** * Class for loading Plugins */ -@Singleton public class ZPluginManager implements IPluginManager { private static final Logger LOGGER = LoggerFactory.getLogger(ZPluginManager.class); @@ -84,7 +82,7 @@ public NotebookRepo createNotebookRepo(String notebookRepoClassName) throws IOEx public synchronized InterpreterLauncher createInterpreterLauncher(String launcherPlugin, RecoveryStorage recoveryStorage) throws IOException { - LOGGER.info("Loading NotebookRepo Plugin: {}", launcherPlugin); + LOGGER.info("Loading launcher Plugin: {}", launcherPlugin); List interpreterLaunchers = pluginManager.getExtensions(InterpreterLauncher.class); for (InterpreterLauncher interpreterLauncher : interpreterLaunchers) { if (launcherPlugin.equals(interpreterLauncher.getClass().getSimpleName())) { From 0e04dbc5a9fd2c0230526a460cbc57683751f1a6 Mon Sep 17 00:00:00 2001 From: Philipp Dallig Date: Thu, 21 Apr 2022 17:33:27 +0200 Subject: [PATCH 3/9] Edit license file --- zeppelin-distribution/src/bin_license/LICENSE | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zeppelin-distribution/src/bin_license/LICENSE b/zeppelin-distribution/src/bin_license/LICENSE index b0c9efc133e..2e902f1bf3e 100644 --- a/zeppelin-distribution/src/bin_license/LICENSE +++ b/zeppelin-distribution/src/bin_license/LICENSE @@ -227,6 +227,7 @@ The following components are provided under Apache License. (Apache 2.0) Dropwizard Jackson Integration for Metrics (io.dropwizard.metrics:metrics-json:4.1.14) - https://github.com/dropwizard/metrics/blob/release/4.1.x/LICENSE (Apache 2.0) Dropwizard Metrics Health Checks (io.dropwizard.metrics:metrics-healthchecks:4.1.14) - https://github.com/dropwizard/metrics/blob/release/4.1.x/LICENSE (Apache 2.0) Dropwizard Metrics Integration with JMX (io.dropwizard.metrics:metrics-jmx:4.1.14) - https://github.com/dropwizard/metrics/blob/release/4.1.x/LICENSE + (Apache 2.0) Plugin Framework for Java (org.pf4j:pf4j:3.6.0) - https://github.com/pf4j/pf4j ======================================================================== MIT licenses @@ -285,6 +286,7 @@ The text of each license is also included at licenses/LICENSE-[project]-[version (The MIT License) ansi-up 2.0.2 (https://github.com/drudru/ansi_up) - https://github.com/drudru/ansi_up#license (The MIT License) bcpkix-jdk15on 1.60 (org.bouncycastle:bcpkix-jdk15on:1.60 https://github.com/bcgit/bc-java) - https://github.com/bcgit/bc-java/blob/master/LICENSE.html (The MIT License) influxdb-client-java 1.4.0 (com.influxdb:influxdb-client-java:1.4.0 https://github.com/influxdata/influxdb-client-java) - https://github.com/influxdata/influxdb-client-java/blob/master/LICENSE + (The MIT License) Java implementation of the SemVer Specification (com.github.zafarkhaja:java-semver:0.9.0) - https://github.com/zafarkhaja/jsemver ======================================================================== BSD-style licenses From 270dcf7be33ca469d10e0d3725bd1991dd55adec Mon Sep 17 00:00:00 2001 From: Philipp Dallig Date: Thu, 21 Apr 2022 18:29:21 +0200 Subject: [PATCH 4/9] Add some documentation --- .../themes/zeppelin/_navigation.html | 1 + docs/development/writing_zeppelin_plugin.md | 42 +++++++++++++++++++ docs/index.md | 1 + 3 files changed, 44 insertions(+) create mode 100644 docs/development/writing_zeppelin_plugin.md diff --git a/docs/_includes/themes/zeppelin/_navigation.html b/docs/_includes/themes/zeppelin/_navigation.html index b741c21832c..48eb099b6ab 100644 --- a/docs/_includes/themes/zeppelin/_navigation.html +++ b/docs/_includes/themes/zeppelin/_navigation.html @@ -166,6 +166,7 @@
  • Extending Zeppelin
  • Writing Zeppelin Interpreter
    • +
  • Writing Zeppelin Plugin
  • Helium (Experimental)
  • Overview
    • diff --git a/docs/development/writing_zeppelin_plugin.md b/docs/development/writing_zeppelin_plugin.md new file mode 100644 index 00000000000..7f7b7252ed3 --- /dev/null +++ b/docs/development/writing_zeppelin_plugin.md @@ -0,0 +1,42 @@ +--- +layout: page +title: "Writing a New Zeppelin Plugin" +description: "Apache Zeppelin Plugin can extend the functionality of the Zeppelin Server." +group: development +--- + +{% include JB/setup %} + +# Writing a New Zeppelin Plugin + +
    + +## What is Apache Zeppelin Plugin + +Apache Zeppelin Plugin can extend the functionality of the Zeppelin zengine. Various extensions can be made to the Zeppelin zengine. For this purpose, an interface must first be implemented in the Zeppelin zengine. The biggest advantage of plugins is the [PluginClassLoader](https://pf4j.org/doc/class-loading.html), which ensures that different versions of the same Java classes do not mix between Zeppelin zengine and the plugins and therefore there are no side effects when running Zeppelin. + +## Make your own Zeppelin Plugin + +Creating a new plugin is quite simple. Just create a new Maven module and make `zengine-plugins-parent` as parent. Create a class with extends `org.pf4j.Plugin`. Plugins are started during Zeppelin Server Startup. +You can also create classes, which implements `org.pf4j.ExtensionPoint` in Zeppelin zengine. + +Current ExtensionPoints: + + - `InterpreterLauncher` + - `NotebookRepo` + +Because Zeppelin does nothing magical you can find also more informations in the [official PF4J documentation](https://pf4j.org/). + + diff --git a/docs/index.md b/docs/index.md index f0cd7355ede..3af8e4741e3 100644 --- a/docs/index.md +++ b/docs/index.md @@ -122,6 +122,7 @@ limitations under the License. #### Developer Guide * Extending Zeppelin * [Writing Zeppelin Interpreter](./development/writing_zeppelin_interpreter.html) + * [Writing Zeppelin Plugin](./development/writing_zeppelin_plugin.html) * [Helium: Overview](./development/helium/overview.html) * [Helium: Writing Application](./development/helium/writing_application.html) * [Helium: Writing Spell](./development/helium/writing_spell.html) From 25af34bf6cd4da5b328532b60bf457765b7fb29f Mon Sep 17 00:00:00 2001 From: Philipp Dallig Date: Thu, 21 Apr 2022 19:00:01 +0200 Subject: [PATCH 5/9] Pass the PluginManager as a zengine object via a interface to the extensions --- .../launcher/ClusterInterpreterLauncher.java | 11 ++++------- .../launcher/ClusterInterpreterLauncherTest.java | 1 - .../launcher/DockerInterpreterLauncher.java | 10 ++++------ .../launcher/DockerInterpreterProcessTest.java | 4 ++-- .../zeppelin/plugin/ExtensionWithPluginManager.java | 12 ++++++++++++ .../org/apache/zeppelin/plugin/ZPluginManager.java | 6 ++++++ 6 files changed, 28 insertions(+), 16 deletions(-) create mode 100644 zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/ExtensionWithPluginManager.java diff --git a/zeppelin-plugins/launcher/cluster/src/main/java/org/apache/zeppelin/interpreter/launcher/ClusterInterpreterLauncher.java b/zeppelin-plugins/launcher/cluster/src/main/java/org/apache/zeppelin/interpreter/launcher/ClusterInterpreterLauncher.java index 1910b273421..3604f0fb99c 100644 --- a/zeppelin-plugins/launcher/cluster/src/main/java/org/apache/zeppelin/interpreter/launcher/ClusterInterpreterLauncher.java +++ b/zeppelin-plugins/launcher/cluster/src/main/java/org/apache/zeppelin/interpreter/launcher/ClusterInterpreterLauncher.java @@ -30,6 +30,7 @@ import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterRunningProcess; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils; +import org.apache.zeppelin.plugin.ExtensionWithPluginManager; import org.apache.zeppelin.plugin.IPluginManager; import org.pf4j.Extension; import org.slf4j.Logger; @@ -39,8 +40,6 @@ import java.util.HashMap; import java.util.Map; -import javax.inject.Inject; - import static org.apache.zeppelin.cluster.event.ClusterEvent.CREATE_INTP_PROCESS; import static org.apache.zeppelin.cluster.meta.ClusterMeta.INTP_TSERVER_HOST; import static org.apache.zeppelin.cluster.meta.ClusterMeta.INTP_TSERVER_PORT; @@ -52,16 +51,15 @@ */ @Extension(points = InterpreterLauncher.class) public class ClusterInterpreterLauncher extends StandardInterpreterLauncher - implements ClusterEventListener { + implements ClusterEventListener, ExtensionWithPluginManager { private static final Logger LOGGER = LoggerFactory.getLogger(ClusterInterpreterLauncher.class); private InterpreterLaunchContext context; private ClusterManagerServer clusterServer; private IPluginManager pluginManager; - @Inject - void setPluginManager(IPluginManager pluginManager) { - LOGGER.info("Injecting PluginManager"); + @Override + public void setPluginManager(IPluginManager pluginManager) { this.pluginManager = pluginManager; } @@ -238,7 +236,6 @@ private InterpreterClient createInterpreterProcess(InterpreterLaunchContext cont InterpreterClient intpProcess = null; if (isRunningOnDocker(zConf)) { DockerInterpreterLauncher dockerIntpLauncher = (DockerInterpreterLauncher) pluginManager.createInterpreterLauncher("DockerInterpreterLauncher", null); - dockerIntpLauncher.setPluginManager(pluginManager); dockerIntpLauncher.setProperties(context.getProperties()); intpProcess = dockerIntpLauncher.launch(context); } else { diff --git a/zeppelin-plugins/launcher/cluster/src/test/java/org/apache/zeppelin/interpreter/launcher/ClusterInterpreterLauncherTest.java b/zeppelin-plugins/launcher/cluster/src/test/java/org/apache/zeppelin/interpreter/launcher/ClusterInterpreterLauncherTest.java index 23d6fd28ab7..8dd90af83d4 100644 --- a/zeppelin-plugins/launcher/cluster/src/test/java/org/apache/zeppelin/interpreter/launcher/ClusterInterpreterLauncherTest.java +++ b/zeppelin-plugins/launcher/cluster/src/test/java/org/apache/zeppelin/interpreter/launcher/ClusterInterpreterLauncherTest.java @@ -115,7 +115,6 @@ public void testCreateIntpProcessDockerMode() throws IOException { ClusterInterpreterLauncher launcher = (ClusterInterpreterLauncher) pluginManager.createInterpreterLauncher("ClusterInterpreterLauncher", null); - launcher.setPluginManager(pluginManager); Properties properties = new Properties(); properties.setProperty( ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT.getVarName(), "1000"); diff --git a/zeppelin-plugins/launcher/docker/src/main/java/org/apache/zeppelin/interpreter/launcher/DockerInterpreterLauncher.java b/zeppelin-plugins/launcher/docker/src/main/java/org/apache/zeppelin/interpreter/launcher/DockerInterpreterLauncher.java index 5b0b12495fb..9f338ca79f2 100644 --- a/zeppelin-plugins/launcher/docker/src/main/java/org/apache/zeppelin/interpreter/launcher/DockerInterpreterLauncher.java +++ b/zeppelin-plugins/launcher/docker/src/main/java/org/apache/zeppelin/interpreter/launcher/DockerInterpreterLauncher.java @@ -16,6 +16,7 @@ */ package org.apache.zeppelin.interpreter.launcher; +import org.apache.zeppelin.plugin.ExtensionWithPluginManager; import org.apache.zeppelin.plugin.IPluginManager; import org.pf4j.Extension; import org.slf4j.Logger; @@ -24,21 +25,18 @@ import java.io.IOException; import java.util.Map; -import javax.inject.Inject; - /** * Interpreter Launcher which use shell script to launch the interpreter process. */ @Extension -public class DockerInterpreterLauncher extends InterpreterLauncher { +public class DockerInterpreterLauncher extends InterpreterLauncher implements ExtensionWithPluginManager { private static final Logger LOGGER = LoggerFactory.getLogger(DockerInterpreterLauncher.class); private InterpreterLaunchContext context; private IPluginManager pluginManager; - @Inject - void setPluginManager(IPluginManager pluginManager) { - LOGGER.info("Injecting PluginManager"); + @Override + public void setPluginManager(IPluginManager pluginManager) { this.pluginManager = pluginManager; } diff --git a/zeppelin-plugins/launcher/docker/src/test/java/org/apache/zeppelin/interpreter/launcher/DockerInterpreterProcessTest.java b/zeppelin-plugins/launcher/docker/src/test/java/org/apache/zeppelin/interpreter/launcher/DockerInterpreterProcessTest.java index 839eb84937a..92ce8631814 100644 --- a/zeppelin-plugins/launcher/docker/src/test/java/org/apache/zeppelin/interpreter/launcher/DockerInterpreterProcessTest.java +++ b/zeppelin-plugins/launcher/docker/src/test/java/org/apache/zeppelin/interpreter/launcher/DockerInterpreterProcessTest.java @@ -18,6 +18,7 @@ import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.interpreter.InterpreterOption; +import org.apache.zeppelin.plugin.IPluginManager; import org.apache.zeppelin.plugin.ZPluginManager; import org.junit.Before; import org.junit.Test; @@ -46,7 +47,7 @@ public class DockerInterpreterProcessTest { private static final Logger LOGGER = LoggerFactory.getLogger(DockerInterpreterProcessTest.class); private ZeppelinConfiguration zconf; - private ZPluginManager pluginManager; + private IPluginManager pluginManager; @Before public void setUp() { @@ -58,7 +59,6 @@ public void setUp() { public void testCreateIntpProcess() throws IOException { DockerInterpreterLauncher launcher = (DockerInterpreterLauncher) pluginManager.createInterpreterLauncher("DockerInterpreterLauncher", null); - launcher.setPluginManager(pluginManager); Properties properties = new Properties(); properties.setProperty( ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT.getVarName(), "5000"); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/ExtensionWithPluginManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/ExtensionWithPluginManager.java new file mode 100644 index 00000000000..6769161311b --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/ExtensionWithPluginManager.java @@ -0,0 +1,12 @@ +package org.apache.zeppelin.plugin; + +/** + * + * Indicates a plugin extension that requires the PluginManager for further operations. Extension that implements this interface + * get the PluginManager passed in right after object creation. + */ +public interface ExtensionWithPluginManager { + + public void setPluginManager(IPluginManager pluginManager); + +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/ZPluginManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/ZPluginManager.java index a37f0da3fb4..abb89803456 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/ZPluginManager.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/ZPluginManager.java @@ -72,6 +72,9 @@ public NotebookRepo createNotebookRepo(String notebookRepoClassName) throws IOEx List notebookRepos = pluginManager.getExtensions(NotebookRepo.class); for (NotebookRepo notebookRepo : notebookRepos) { if (notebookRepoClassName.equals(notebookRepo.getClass().getName())) { + if (notebookRepo instanceof ExtensionWithPluginManager) { + ((ExtensionWithPluginManager) notebookRepo).setPluginManager(this); + } return notebookRepo; } } @@ -86,6 +89,9 @@ public synchronized InterpreterLauncher createInterpreterLauncher(String launche List interpreterLaunchers = pluginManager.getExtensions(InterpreterLauncher.class); for (InterpreterLauncher interpreterLauncher : interpreterLaunchers) { if (launcherPlugin.equals(interpreterLauncher.getClass().getSimpleName())) { + if (interpreterLauncher instanceof ExtensionWithPluginManager) { + ((ExtensionWithPluginManager) interpreterLauncher).setPluginManager(this); + } interpreterLauncher.init(zConf, recoveryStorage); return interpreterLauncher; } From e52868b9271d64e78ae10f4d9422d12701bc3171 Mon Sep 17 00:00:00 2001 From: Philipp Dallig Date: Fri, 13 May 2022 09:20:55 +0200 Subject: [PATCH 6/9] use BasePlugin --- .../interpreter/launcher/ClusterPlugin.java | 20 ++------ .../interpreter/launcher/DockerPlugin.java | 20 ++------ .../interpreter/launcher/FlinkPlugin.java | 20 ++------ .../interpreter/launcher/K8sPlugin.java | 20 ++------ .../interpreter/launcher/YarnPlugin.java | 20 ++------ .../zeppelin/notebook/repo/AzurePlugin.java | 18 ++----- .../notebook/repo/FileSystemPlugin.java | 19 ++----- .../zeppelin/notebook/repo/GCSPlugin.java | 20 ++------ .../zeppelin/notebook/repo/GithubPlugin.java | 20 ++------ .../zeppelin/notebook/repo/MongoPlugin.java | 20 ++------ .../zeppelin/notebook/repo/AliyunPlugin.java | 15 ++---- .../zeppelin/notebook/repo/AWSPlugin.java | 19 ++----- .../apache/zeppelin/plugin/BasePlugin.java | 50 +++++++++++++++++++ 13 files changed, 86 insertions(+), 195 deletions(-) create mode 100644 zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/BasePlugin.java diff --git a/zeppelin-plugins/launcher/cluster/src/main/java/org/apache/zeppelin/interpreter/launcher/ClusterPlugin.java b/zeppelin-plugins/launcher/cluster/src/main/java/org/apache/zeppelin/interpreter/launcher/ClusterPlugin.java index 167accbf9cd..8f8b89da840 100644 --- a/zeppelin-plugins/launcher/cluster/src/main/java/org/apache/zeppelin/interpreter/launcher/ClusterPlugin.java +++ b/zeppelin-plugins/launcher/cluster/src/main/java/org/apache/zeppelin/interpreter/launcher/ClusterPlugin.java @@ -17,26 +17,12 @@ package org.apache.zeppelin.interpreter.launcher; -import org.pf4j.Plugin; +import org.apache.zeppelin.plugin.BasePlugin; import org.pf4j.PluginWrapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -public class ClusterPlugin extends Plugin { - - private static final Logger LOGGER = LoggerFactory.getLogger(ClusterPlugin.class); +public class ClusterPlugin extends BasePlugin { public ClusterPlugin(PluginWrapper wrapper) { - super(wrapper); - } - - @Override - public void start() { - LOGGER.info("ClusterPlugin started"); - } - - @Override - public void stop() { - LOGGER.info("ClusterPlugin stopped"); + super(wrapper, "ClusterPlugin"); } } diff --git a/zeppelin-plugins/launcher/docker/src/main/java/org/apache/zeppelin/interpreter/launcher/DockerPlugin.java b/zeppelin-plugins/launcher/docker/src/main/java/org/apache/zeppelin/interpreter/launcher/DockerPlugin.java index 7f4236bab81..7fc248bb5b1 100644 --- a/zeppelin-plugins/launcher/docker/src/main/java/org/apache/zeppelin/interpreter/launcher/DockerPlugin.java +++ b/zeppelin-plugins/launcher/docker/src/main/java/org/apache/zeppelin/interpreter/launcher/DockerPlugin.java @@ -17,26 +17,12 @@ package org.apache.zeppelin.interpreter.launcher; -import org.pf4j.Plugin; +import org.apache.zeppelin.plugin.BasePlugin; import org.pf4j.PluginWrapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -public class DockerPlugin extends Plugin { - - private static final Logger LOGGER = LoggerFactory.getLogger(DockerPlugin.class); +public class DockerPlugin extends BasePlugin { public DockerPlugin(PluginWrapper wrapper) { - super(wrapper); - } - - @Override - public void start() { - LOGGER.info("DockerPlugin started"); - } - - @Override - public void stop() { - LOGGER.info("DockerPlugin stopped"); + super(wrapper, "DockerPlugin"); } } diff --git a/zeppelin-plugins/launcher/flink/src/main/java/org/apache/zeppelin/interpreter/launcher/FlinkPlugin.java b/zeppelin-plugins/launcher/flink/src/main/java/org/apache/zeppelin/interpreter/launcher/FlinkPlugin.java index 4d0f1b7aeb7..cdbff220723 100644 --- a/zeppelin-plugins/launcher/flink/src/main/java/org/apache/zeppelin/interpreter/launcher/FlinkPlugin.java +++ b/zeppelin-plugins/launcher/flink/src/main/java/org/apache/zeppelin/interpreter/launcher/FlinkPlugin.java @@ -17,26 +17,12 @@ package org.apache.zeppelin.interpreter.launcher; -import org.pf4j.Plugin; +import org.apache.zeppelin.plugin.BasePlugin; import org.pf4j.PluginWrapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -public class FlinkPlugin extends Plugin { - - private static final Logger LOGGER = LoggerFactory.getLogger(FlinkPlugin.class); +public class FlinkPlugin extends BasePlugin { public FlinkPlugin(PluginWrapper wrapper) { - super(wrapper); - } - - @Override - public void start() { - LOGGER.info("FlinkPlugin started"); - } - - @Override - public void stop() { - LOGGER.info("FlinkPlugin stopped"); + super(wrapper, "FlinkPlugin"); } } diff --git a/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/K8sPlugin.java b/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/K8sPlugin.java index 5556ec940c8..aea07d33f76 100644 --- a/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/K8sPlugin.java +++ b/zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/K8sPlugin.java @@ -17,26 +17,12 @@ package org.apache.zeppelin.interpreter.launcher; -import org.pf4j.Plugin; +import org.apache.zeppelin.plugin.BasePlugin; import org.pf4j.PluginWrapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -public class K8sPlugin extends Plugin { - - private static final Logger LOGGER = LoggerFactory.getLogger(K8sPlugin.class); +public class K8sPlugin extends BasePlugin { public K8sPlugin(PluginWrapper wrapper) { - super(wrapper); - } - - @Override - public void start() { - LOGGER.info("K8sPlugin started"); - } - - @Override - public void stop() { - LOGGER.info("K8sPlugin stopped"); + super(wrapper, "K8sPlugin"); } } diff --git a/zeppelin-plugins/launcher/yarn/src/main/java/org/apache/zeppelin/interpreter/launcher/YarnPlugin.java b/zeppelin-plugins/launcher/yarn/src/main/java/org/apache/zeppelin/interpreter/launcher/YarnPlugin.java index 257346eeb03..beca5878c09 100644 --- a/zeppelin-plugins/launcher/yarn/src/main/java/org/apache/zeppelin/interpreter/launcher/YarnPlugin.java +++ b/zeppelin-plugins/launcher/yarn/src/main/java/org/apache/zeppelin/interpreter/launcher/YarnPlugin.java @@ -17,26 +17,12 @@ package org.apache.zeppelin.interpreter.launcher; -import org.pf4j.Plugin; +import org.apache.zeppelin.plugin.BasePlugin; import org.pf4j.PluginWrapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -public class YarnPlugin extends Plugin { - - private static final Logger LOGGER = LoggerFactory.getLogger(YarnPlugin.class); +public class YarnPlugin extends BasePlugin { public YarnPlugin(PluginWrapper wrapper) { - super(wrapper); - } - - @Override - public void start() { - LOGGER.info("YarnPlugin started"); - } - - @Override - public void stop() { - LOGGER.info("YarnPlugin stopped"); + super(wrapper, "YarnPlugin"); } } diff --git a/zeppelin-plugins/notebookrepo/azure/src/main/java/org/apache/zeppelin/notebook/repo/AzurePlugin.java b/zeppelin-plugins/notebookrepo/azure/src/main/java/org/apache/zeppelin/notebook/repo/AzurePlugin.java index 5738356b300..be4ad014854 100644 --- a/zeppelin-plugins/notebookrepo/azure/src/main/java/org/apache/zeppelin/notebook/repo/AzurePlugin.java +++ b/zeppelin-plugins/notebookrepo/azure/src/main/java/org/apache/zeppelin/notebook/repo/AzurePlugin.java @@ -17,25 +17,13 @@ package org.apache.zeppelin.notebook.repo; -import org.pf4j.Plugin; +import org.apache.zeppelin.plugin.BasePlugin; import org.pf4j.PluginWrapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -public class AzurePlugin extends Plugin { - private static final Logger LOGGER = LoggerFactory.getLogger(AzurePlugin.class); +public class AzurePlugin extends BasePlugin { public AzurePlugin(PluginWrapper wrapper) { - super(wrapper); + super(wrapper, "AzurePlugin"); } - @Override - public void start() { - LOGGER.info("AzurePlugin started"); - } - - @Override - public void stop() { - LOGGER.info("AzurePlugin stopped"); - } } diff --git a/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemPlugin.java b/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemPlugin.java index 9642baefd97..f816105af79 100644 --- a/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemPlugin.java +++ b/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemPlugin.java @@ -17,26 +17,13 @@ package org.apache.zeppelin.notebook.repo; -import org.pf4j.Plugin; +import org.apache.zeppelin.plugin.BasePlugin; import org.pf4j.PluginWrapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -public class FileSystemPlugin extends Plugin { - - private static final Logger LOGGER = LoggerFactory.getLogger(FileSystemPlugin.class); +public class FileSystemPlugin extends BasePlugin { public FileSystemPlugin(PluginWrapper wrapper) { - super(wrapper); - } - - @Override - public void start() { - LOGGER.info("FileSystemPlugin started"); + super(wrapper, "FileSystemPlugin"); } - @Override - public void stop() { - LOGGER.info("FileSystemPlugin stopped"); - } } diff --git a/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSPlugin.java b/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSPlugin.java index df908e3186b..ef01ef5f8ed 100644 --- a/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSPlugin.java +++ b/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSPlugin.java @@ -17,26 +17,12 @@ package org.apache.zeppelin.notebook.repo; -import org.pf4j.Plugin; +import org.apache.zeppelin.plugin.BasePlugin; import org.pf4j.PluginWrapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -public class GCSPlugin extends Plugin { - - private static final Logger LOGGER = LoggerFactory.getLogger(GCSPlugin.class); +public class GCSPlugin extends BasePlugin { public GCSPlugin(PluginWrapper wrapper) { - super(wrapper); - } - - @Override - public void start() { - LOGGER.info("GCSPlugin started"); - } - - @Override - public void stop() { - LOGGER.info("GCSPlugin stopped"); + super(wrapper, "GCSPlugin"); } } diff --git a/zeppelin-plugins/notebookrepo/github/src/main/java/org/apache/zeppelin/notebook/repo/GithubPlugin.java b/zeppelin-plugins/notebookrepo/github/src/main/java/org/apache/zeppelin/notebook/repo/GithubPlugin.java index 94f684b7507..91f12f9a607 100644 --- a/zeppelin-plugins/notebookrepo/github/src/main/java/org/apache/zeppelin/notebook/repo/GithubPlugin.java +++ b/zeppelin-plugins/notebookrepo/github/src/main/java/org/apache/zeppelin/notebook/repo/GithubPlugin.java @@ -17,26 +17,12 @@ package org.apache.zeppelin.notebook.repo; -import org.pf4j.Plugin; +import org.apache.zeppelin.plugin.BasePlugin; import org.pf4j.PluginWrapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -public class GithubPlugin extends Plugin { - - private static final Logger LOGGER = LoggerFactory.getLogger(GithubPlugin.class); +public class GithubPlugin extends BasePlugin { public GithubPlugin(PluginWrapper wrapper) { - super(wrapper); - } - - @Override - public void start() { - LOGGER.info("GithubPlugin started"); - } - - @Override - public void stop() { - LOGGER.info("GithubPlugin stopped"); + super(wrapper, "GithubPlugin"); } } diff --git a/zeppelin-plugins/notebookrepo/mongo/src/main/java/org/apache/zeppelin/notebook/repo/MongoPlugin.java b/zeppelin-plugins/notebookrepo/mongo/src/main/java/org/apache/zeppelin/notebook/repo/MongoPlugin.java index a681c448c97..431f5e0c8a6 100644 --- a/zeppelin-plugins/notebookrepo/mongo/src/main/java/org/apache/zeppelin/notebook/repo/MongoPlugin.java +++ b/zeppelin-plugins/notebookrepo/mongo/src/main/java/org/apache/zeppelin/notebook/repo/MongoPlugin.java @@ -17,26 +17,12 @@ package org.apache.zeppelin.notebook.repo; -import org.pf4j.Plugin; +import org.apache.zeppelin.plugin.BasePlugin; import org.pf4j.PluginWrapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -public class MongoPlugin extends Plugin { - - private static final Logger LOGGER = LoggerFactory.getLogger(MongoPlugin.class); +public class MongoPlugin extends BasePlugin { public MongoPlugin(PluginWrapper wrapper) { - super(wrapper); - } - - @Override - public void start() { - LOGGER.info("MongoPlugin started"); - } - - @Override - public void stop() { - LOGGER.info("MongoPlugin stopped"); + super(wrapper, "MongoPlugin"); } } diff --git a/zeppelin-plugins/notebookrepo/oss/src/main/java/org/apache/zeppelin/notebook/repo/AliyunPlugin.java b/zeppelin-plugins/notebookrepo/oss/src/main/java/org/apache/zeppelin/notebook/repo/AliyunPlugin.java index a97f7e9ccc7..c7a9325c867 100644 --- a/zeppelin-plugins/notebookrepo/oss/src/main/java/org/apache/zeppelin/notebook/repo/AliyunPlugin.java +++ b/zeppelin-plugins/notebookrepo/oss/src/main/java/org/apache/zeppelin/notebook/repo/AliyunPlugin.java @@ -17,26 +17,17 @@ package org.apache.zeppelin.notebook.repo; +import org.apache.zeppelin.plugin.BasePlugin; import org.pf4j.Plugin; import org.pf4j.PluginWrapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class AliyunPlugin extends Plugin { +public class AliyunPlugin extends BasePlugin { private static final Logger LOGGER = LoggerFactory.getLogger(AliyunPlugin.class); public AliyunPlugin(PluginWrapper wrapper) { - super(wrapper); - } - - @Override - public void start() { - LOGGER.info("AliyunPlugin started"); - } - - @Override - public void stop() { - LOGGER.info("AliyunPlugin stopped"); + super(wrapper, "AliyunPlugin"); } } diff --git a/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/AWSPlugin.java b/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/AWSPlugin.java index 9fa8a92400c..2a92d13d98d 100644 --- a/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/AWSPlugin.java +++ b/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/AWSPlugin.java @@ -17,26 +17,13 @@ package org.apache.zeppelin.notebook.repo; -import org.pf4j.Plugin; +import org.apache.zeppelin.plugin.BasePlugin; import org.pf4j.PluginWrapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -public class AWSPlugin extends Plugin { - - private static final Logger LOGGER = LoggerFactory.getLogger(AWSPlugin.class); +public class AWSPlugin extends BasePlugin { public AWSPlugin(PluginWrapper wrapper) { - super(wrapper); - } - - @Override - public void start() { - LOGGER.info("AWSPlugin started"); + super(wrapper, "AWSPlugin"); } - @Override - public void stop() { - LOGGER.info("AWSPlugin stopped"); - } } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/BasePlugin.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/BasePlugin.java new file mode 100644 index 00000000000..10bbbbe5c55 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/BasePlugin.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.plugin; + +import org.pf4j.Plugin; +import org.pf4j.PluginWrapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class BasePlugin extends Plugin { + + private static final Logger LOGGER = LoggerFactory.getLogger(BasePlugin.class); + + final String name; + + /** + * + * @param wrapper + * @param name - name of the plugin + */ + public BasePlugin(PluginWrapper wrapper, String name) { + super(wrapper); + this.name = name; + } + + @Override + public void start() { + LOGGER.info("{} started", name); + } + + @Override + public void stop() { + LOGGER.info("{} stopped", name); + } +} From 7d3df8c69838e0ca378fa53f3f3d98f48ae8d119 Mon Sep 17 00:00:00 2001 From: Philipp Dallig Date: Fri, 13 May 2022 09:37:33 +0200 Subject: [PATCH 7/9] Remove duplicate pf4j version --- zeppelin-zengine/pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/zeppelin-zengine/pom.xml b/zeppelin-zengine/pom.xml index 9c0d1e9673d..3180a807943 100644 --- a/zeppelin-zengine/pom.xml +++ b/zeppelin-zengine/pom.xml @@ -40,7 +40,6 @@ 1.3 2.6.0 4.5.4.201711221230-r - 3.6.0 From 5284020bcf8b0a9fe20ad4822436e46f9995f58c Mon Sep 17 00:00:00 2001 From: Philipp Dallig Date: Fri, 13 May 2022 09:37:49 +0200 Subject: [PATCH 8/9] cast to the returned class --- .../interpreter/launcher/DockerInterpreterLauncher.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zeppelin-plugins/launcher/docker/src/main/java/org/apache/zeppelin/interpreter/launcher/DockerInterpreterLauncher.java b/zeppelin-plugins/launcher/docker/src/main/java/org/apache/zeppelin/interpreter/launcher/DockerInterpreterLauncher.java index 9f338ca79f2..cc53baffa49 100644 --- a/zeppelin-plugins/launcher/docker/src/main/java/org/apache/zeppelin/interpreter/launcher/DockerInterpreterLauncher.java +++ b/zeppelin-plugins/launcher/docker/src/main/java/org/apache/zeppelin/interpreter/launcher/DockerInterpreterLauncher.java @@ -57,9 +57,9 @@ public InterpreterClient launchDirectly(InterpreterLaunchContext context) throws StandardInterpreterLauncher interpreterLauncher = null; if (isSpark()) { - interpreterLauncher = (StandardInterpreterLauncher) pluginManager.createInterpreterLauncher("SparkInterpreterLauncher", recoveryStorage); + interpreterLauncher = (SparkInterpreterLauncher) pluginManager.createInterpreterLauncher("SparkInterpreterLauncher", recoveryStorage); } else if (isFlink()) { - interpreterLauncher = (StandardInterpreterLauncher) pluginManager.createInterpreterLauncher("FlinkInterpreterLauncher", recoveryStorage); + interpreterLauncher = (FlinkInterpreterLauncher) pluginManager.createInterpreterLauncher("FlinkInterpreterLauncher", recoveryStorage); } else { interpreterLauncher = (StandardInterpreterLauncher) pluginManager.createInterpreterLauncher("StandardInterpreterLauncher", recoveryStorage); } From 4315bd80a0c2ad14b68888a9daad08dd79eff021 Mon Sep 17 00:00:00 2001 From: Philipp Dallig Date: Fri, 13 May 2022 09:43:03 +0200 Subject: [PATCH 9/9] Fix rat --- .../plugin/ExtensionWithPluginManager.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/ExtensionWithPluginManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/ExtensionWithPluginManager.java index 6769161311b..c364ce75b52 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/ExtensionWithPluginManager.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/ExtensionWithPluginManager.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.zeppelin.plugin; /**