Skip to content

Commit 20f5bc8

Browse files
ludomikulaaq-ikhwa-tech
ludomikula
and
aq-ikhwa-tech
committed
New plugin system
* Add support for SUPER_ADMIN role * Publish server log events * Add handling for audit logs feature * Add handling for geolocation data * Add handling for api delays in case of rate limit * Propagate plugin specific environment variables to plugins * Add environment variable for controlling plugin location * Implemented plugin endpoints security --------- Co-authored-by: Abdul Qadir <[email protected]>
1 parent 4f80cae commit 20f5bc8

File tree

101 files changed

+2919
-905
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

101 files changed

+2919
-905
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,3 +9,4 @@ client/node_modules/
99
client/packages/lowcoder-plugin-demo/.yarn/install-state.gz
1010
client/packages/lowcoder-plugin-demo/yarn.lock
1111
client/packages/lowcoder-plugin-demo/.yarn/cache/@types-node-npm-16.18.68-56f72825c0-094ae9ed80.zip
12+
application-dev.yml

‎deploy/docker/Dockerfile

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,20 +2,14 @@
22
## Build Lowcoder api-service application
33
##
44
FROM maven:3.9-eclipse-temurin-17 AS build-api-service
5+
6+
# Build lowcoder-api
57
COPY ./server/api-service /lowcoder-server
68
WORKDIR /lowcoder-server
79
RUN --mount=type=cache,target=/root/.m2 mvn -f pom.xml clean package -DskipTests
810

911
# Create required folder structure
10-
RUN mkdir -p /lowcoder/api-service/plugins /lowcoder/api-service/config /lowcoder/api-service/logs
11-
12-
# Define lowcoder main jar and plugin jars
13-
ARG JAR_FILE=/lowcoder-server/lowcoder-server/target/lowcoder-server-*.jar
14-
ARG PLUGIN_JARS=/lowcoder-server/lowcoder-plugins/*/target/*.jar
15-
16-
# Copy lowcoder server application and plugins
17-
RUN cp ${JAR_FILE} /lowcoder/api-service/server.jar \
18-
&& cp ${PLUGIN_JARS} /lowcoder/api-service/plugins/
12+
RUN mkdir -p /lowcoder/api-service/config /lowcoder/api-service/logs /lowcoder/plugins
1913

2014
# Copy lowcoder server configuration
2115
COPY server/api-service/lowcoder-server/src/main/resources/selfhost/ce/application.yml /lowcoder/api-service/config/
@@ -43,6 +37,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends gosu \
4337
# Copy lowcoder server configuration
4438
COPY --chown=lowcoder:lowcoder --from=build-api-service /lowcoder/api-service /lowcoder/api-service
4539

40+
# Copy lowcoder api service app, dependencies and libs
41+
COPY --chown=lowcoder:lowcoder --from=build-api-service /lowcoder-server/distribution/target/lowcoder-api-service-bin/app /lowcoder/api-service/app
42+
COPY --chown=lowcoder:lowcoder --from=build-api-service /lowcoder-server/distribution/target/lowcoder-api-service-bin/dependencies /lowcoder/api-service/dependencies
43+
COPY --chown=lowcoder:lowcoder --from=build-api-service /lowcoder-server/distribution/target/lowcoder-api-service-bin/libs /lowcoder/api-service/libs
44+
COPY --chown=lowcoder:lowcoder --from=build-api-service /lowcoder-server/distribution/target/lowcoder-api-service-bin/plugins /lowcoder/api-service/plugins
45+
COPY --chown=lowcoder:lowcoder --from=build-api-service /lowcoder-server/distribution/target/lowcoder-api-service-bin/set-classpath.sh /lowcoder/api-service/set-classpath.sh
46+
4647
EXPOSE 8080
4748
CMD [ "sh" , "/lowcoder/api-service/entrypoint.sh" ]
4849

@@ -202,6 +203,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install --no-instal
202203

203204
# Add lowcoder api-service
204205
COPY --chown=lowcoder:lowcoder --from=lowcoder-ce-api-service /lowcoder/api-service /lowcoder/api-service
206+
RUN mkdir -p /lowcoder/plugins/ && chown lowcoder:lowcoder /lowcoder/plugins/
205207

206208
# Add lowcoder node-service
207209
COPY --chown=lowcoder:lowcoder --from=lowcoder-ce-node-service /lowcoder/node-service /lowcoder/node-service

‎deploy/docker/api-service/entrypoint.sh

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,12 +27,16 @@ ${JAVA_HOME}/bin/java -version
2727
echo
2828

2929
cd /lowcoder/api-service
30+
source set-classpath.sh
31+
3032
exec gosu ${USER_ID}:${GROUP_ID} ${JAVA_HOME}/bin/java \
33+
-Djava.util.prefs.userRoot=/tmp \
3134
-Djava.security.egd=file:/dev/./urandom \
3235
-Dhttps.protocols=TLSv1.1,TLSv1.2 \
3336
-Dlog4j2.formatMsgNoLookups=true \
3437
-Dspring.config.location="file:///lowcoder/api-service/config/application.yml,file:///lowcoder/api-service/config/application-selfhost.yml" \
3538
--add-opens java.base/java.nio=ALL-UNNAMED \
39+
-cp "${LOWCODER_CLASSPATH:=.}" \
3640
${JAVA_OPTS} \
37-
-jar "${APP_JAR}" --spring.webflux.base-path=${CONTEXT_PATH} ${CUSTOM_APP_PROPERTIES}
41+
org.lowcoder.api.ServerApplication --spring.webflux.base-path=${CONTEXT_PATH} ${CUSTOM_APP_PROPERTIES}
3842

‎server/api-service/.gitignore

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,9 @@ dependency-reduced-pom.xml
2323
.run/**
2424
logs/**
2525
tmp/**
26-
/openblocks-server/logs/
2726

27+
# Ignore plugin.properties which are generated dynamically
28+
**/plugin.properties
2829

2930
# to ignore the node_modeules folder
3031
node_modules
@@ -34,5 +35,4 @@ package-lock.json
3435
# test coverage
3536
coverage-summary.json
3637
app/client/cypress/locators/Widgets.json
37-
/openblocks-domain/logs/
38-
application-lowcoder.yml
38+
application-lowcoder.yml

‎server/api-service/PLUGIN.md

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
# Lowcoder plugin system (WIP)
2+
3+
This is an ongoing effort to refactor current plugin system based on pf4j library.
4+
5+
## Reasoning
6+
7+
1. create a cleaner and simpler plugin system with clearly defined purpose(s) (new endpoints, new datasource types, etc..)
8+
2. lowcoder does not need live plugin loading/reloading/unloading/updates, therefore the main feature of pf4j is rendered useless, in fact it adds a lot of complexity due to classloaders used for managing plugins (especially in spring/boot applications)
9+
3. simpler and easier plugin detection - just a jar with a class implementing a common interface (be it a simple pojo project or a complex spring/boot implementation)
10+
11+
## How it works
12+
13+
The main entrypoint for plugin system is in **lowcoder-server** module with class **org.lowcoder.api.framework.configuration.PluginConfiguration**
14+
It creates:
15+
- LowcoderPluginManager bean which is responsible for plugin lifecycle management
16+
- Adds plugin defined endpoints to lowcoder by creating **pluginEndpoints** bean
17+
- TODO: Adds plugin defined datasources to lowcoder by creating **pluginDatasources** bean
18+
19+
### lowcoder-plugin-api library
20+
21+
This library contains APIs for plugin implementations.
22+
It is used by both, lowcoder API server as well as all plugins.
23+
24+
### PluginLoader
25+
26+
The sole purpose of a PluginLoader is to find plugin candidates and load them into VM.
27+
There is currently one implementation that based on paths - **PathBasedPluginLoader**, it:
28+
- looks in folders and subfolders defined in **application.yaml** - entries can point to a folder or specific jar file. If a relative path is supplied, the location of lowcoder API server application jar is used as parent folder (when run in non-packaged state, eg. in IDE, it uses the folder where ServerApplication.class is generated)
29+
30+
```yaml
31+
common:
32+
plugin-dirs:
33+
- plugins
34+
- /some/custom/path/myGreatPlugin.jar
35+
```
36+
- finds all **jar**(s) and inspects them for classes implementing **LowcoderPlugin** interface
37+
- instantiates all LowcoderPlugin implementations
38+
39+
### LowcoderPluginManager
40+
41+
The main job of plugin manager is to:
42+
- register plugins found and instantiated by **PluginLoader**
43+
- start registered plugins by calling **LowcoderPlugin.load()** method
44+
- create and register **RouterFunction**(s) for all loaded plugin endpoints
45+
- TODO: create and register datasources for all loaded plugin datasources
46+
47+
## Plugin project structure
48+
49+
Plugin jar can be structured in any way you like. It can be a plain java project, but also a spring/boot based project or based on any other framework.
50+
51+
It is composed from several parts:
52+
- class(es) implementing **LowcoderPlugin** interface
53+
- class(es) implementing **LowcoderEndpoint** interface, containing endpoint handler functions marked with **@EndpointExtension** annotation. These functions must obey following format:
54+
55+
```java
56+
@EndpointExtension(uri = <endpoint uri>, method = <HTTP method>)
57+
public Mono<ServerResponse> <handler name>(ServerRequest request)
58+
{
59+
... your endpoint logic implementation
60+
}
61+
62+
for example:
63+
64+
@EndpointExtension(uri = "/hello-world", method = Method.GET)
65+
public Mono<ServerResponse> helloWorld(ServerRequest request)
66+
{
67+
return ServerResponse.ok().body(Mono.just(Hello.builder().message("Hello world!").build()), Hello.class);
68+
}
69+
```
70+
- TODO: class(es) impelemting **LowcoderDatasource** interface
71+
72+
### LowcoderPlugin implementations
73+
74+
Methods of interest:
75+
- **pluginId()** - unique plugin ID - if a plugin with such ID is already loaded, subsequent plugins whith this ID will be ignored
76+
- **description()** - short plugin description
77+
- **load(ApplicationContext parentContext)** - is called during plugin startup - this is the place where you should completely initialize your plugin. If initialization fails, return false
78+
- **unload()** - is called during lowcoder API server shutdown - this is the place where you should release all resources
79+
- **endpoints()** - needs to contain all initialized **PluginEndpoints** you want to expose, for example:
80+
81+
```java
82+
@Override
83+
public List<PluginEndpoint> endpoints()
84+
{
85+
List<PluginEndpoint> endpoints = new ArrayList<>();
86+
87+
endpoints.add(new HelloWorldEndpoint());
88+
89+
return endpoints;
90+
}
91+
```
92+
- **pluginInfo()** - should return a record object with additional information about your plugin. It is serialized to JSON as part of the **/plugins** listing (see **"info"** object in this example):
93+
94+
```json
95+
[
96+
{
97+
"id": "example-plugin",
98+
"description": "Example plugin for lowcoder platform",
99+
"info": {}
100+
},
101+
{
102+
"id": "enterprise",
103+
"description": "Lowcoder enterprise plugin",
104+
"info": {
105+
"enabledFeatures": [
106+
"endpointApiUsage"
107+
]
108+
}
109+
}
110+
]
111+
```
112+
113+
## TODOs
114+
115+
1. Implement endpoint security - currently all plugin endpoints are public (probably by adding **security** attribute to **@EndpointExtension** and enforcing it)
116+
117+
118+
## QUESTIONS / CONSIDERATIONS
119+
120+
1. currently the plugin endpoints are prefixed with **/plugin/{pluginId}/** - this is hardcoded, do we want to make it configurable?
121+
122+

‎server/api-service/distribution/pom.xml

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
<project xmlns="http://maven.apache.org/POM/4.0.0"
2+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
3+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
4+
<modelVersion>4.0.0</modelVersion>
5+
<parent>
6+
<groupId>org.lowcoder</groupId>
7+
<artifactId>lowcoder-root</artifactId>
8+
<version>${revision}</version>
9+
</parent>
10+
11+
<artifactId>distribution</artifactId>
12+
<packaging>pom</packaging>
13+
14+
<properties>
15+
<assembly.lib.directory>${project.build.directory}/dependencies</assembly.lib.directory>
16+
</properties>
17+
18+
19+
<!-- Dependency added here only to make sure this module is built after
20+
everything alse was built -->
21+
<dependencies>
22+
<dependency>
23+
<groupId>org.lowcoder</groupId>
24+
<artifactId>lowcoder-sdk</artifactId>
25+
</dependency>
26+
<dependency>
27+
<groupId>org.lowcoder</groupId>
28+
<artifactId>lowcoder-infra</artifactId>
29+
</dependency>
30+
<dependency>
31+
<groupId>org.lowcoder</groupId>
32+
<artifactId>lowcoder-domain</artifactId>
33+
</dependency>
34+
<dependency>
35+
<groupId>org.lowcoder</groupId>
36+
<artifactId>lowcoder-server</artifactId>
37+
</dependency>
38+
</dependencies>
39+
40+
<build>
41+
<finalName>lowcoder-api-service</finalName>
42+
<plugins>
43+
<plugin>
44+
<groupId>org.apache.maven.plugins</groupId>
45+
<artifactId>maven-dependency-plugin</artifactId>
46+
<executions>
47+
<execution>
48+
<id>copy-dependencies</id>
49+
<phase>prepare-package</phase>
50+
<goals>
51+
<goal>copy-dependencies</goal>
52+
</goals>
53+
<configuration>
54+
<outputDirectory>${assembly.lib.directory}</outputDirectory>
55+
<overWriteReleases>false</overWriteReleases>
56+
<overWriteSnapshots>false</overWriteSnapshots>
57+
<overWriteIfNewer>true</overWriteIfNewer>
58+
<prependGroupId>true</prependGroupId>
59+
</configuration>
60+
</execution>
61+
</executions>
62+
</plugin>
63+
<plugin>
64+
<artifactId>maven-assembly-plugin</artifactId>
65+
<executions>
66+
<execution>
67+
<id>distro-assembly</id>
68+
<phase>package</phase>
69+
<goals>
70+
<goal>single</goal>
71+
</goals>
72+
<configuration>
73+
<attach>false</attach>
74+
<descriptors>
75+
<descriptor>src/assembly/bin.xml</descriptor>
76+
</descriptors>
77+
</configuration>
78+
</execution>
79+
</executions>
80+
</plugin>
81+
</plugins>
82+
</build>
83+
84+
</project>

‎server/api-service/distribution/src/assembly/bin.xml

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.2.0"
2+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
3+
xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.2.0 http://maven.apache.org/xsd/assembly-2.2.0.xsd">
4+
<id>bin</id>
5+
<formats>
6+
<format>dir</format>
7+
</formats>
8+
<includeBaseDirectory>false</includeBaseDirectory>
9+
10+
<files>
11+
<file>
12+
<source>src/assembly/set-classpath.sh</source>
13+
<outputDirectory></outputDirectory>
14+
</file>
15+
</files>
16+
<fileSets>
17+
<fileSet>
18+
<directory>${assembly.lib.directory}</directory>
19+
<outputDirectory>dependencies</outputDirectory>
20+
<excludes>
21+
<exclude>${project.groupId}:*</exclude>
22+
</excludes>
23+
</fileSet>
24+
</fileSets>
25+
26+
<moduleSets>
27+
<!-- Main lowcoder API server application -->
28+
<moduleSet>
29+
<useAllReactorProjects>true</useAllReactorProjects>
30+
<includes>
31+
<include>org.lowcoder:lowcoder-server</include>
32+
</includes>
33+
<binaries>
34+
<outputDirectory>app</outputDirectory>
35+
<includeDependencies>false</includeDependencies>
36+
<unpack>false</unpack>
37+
</binaries>
38+
</moduleSet>
39+
40+
<!-- Lowcoder API server dependencies -->
41+
<moduleSet>
42+
<useAllReactorProjects>true</useAllReactorProjects>
43+
<includes>
44+
<include>org.lowcoder:lowcoder-domain</include>
45+
<include>org.lowcoder:lowcoder-infra</include>
46+
<include>org.lowcoder:lowcoder-sdk</include>
47+
</includes>
48+
<binaries>
49+
<outputDirectory>libs</outputDirectory>
50+
<includeDependencies>false</includeDependencies>
51+
<unpack>false</unpack>
52+
</binaries>
53+
</moduleSet>
54+
55+
<!-- Lowcoder plugins -->
56+
<moduleSet>
57+
<useAllReactorProjects>true</useAllReactorProjects>
58+
<includeSubModules>true</includeSubModules>
59+
<includes>
60+
<include>org.lowcoder:*Plugin</include>
61+
</includes>
62+
<excludes>
63+
<exclude>org.lowcoder:sqlBasedPlugin</exclude>
64+
</excludes>
65+
<binaries>
66+
<outputDirectory>plugins</outputDirectory>
67+
<includeDependencies>false</includeDependencies>
68+
<unpack>false</unpack>
69+
</binaries>
70+
</moduleSet>
71+
</moduleSets>
72+
</assembly>

‎server/api-service/distribution/src/assembly/set-classpath.sh

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/bin/bash
2+
3+
#
4+
# Set lowcoder api service classpath for use in startup script
5+
#
6+
export LOWCODER_CLASSPATH="`find libs/ dependencies/ app/ -type f -name "*.jar" | tr '\n' ':' | sed -e 's/:$//'`"
7+
8+
#
9+
# Example usage:
10+
#
11+
# java -cp "${LOWCODER_CLASSPATH}" org.lowcoder.api.ServerApplication

0 commit comments

Comments
 (0)