[INLONG-8413][Doc] Update the description about data Ingestion and Sy…

…nchronization (apache#8414)
ZhuGJ15 · Jul 4, 2023 · 5d86672 · 5d86672
1 parent e0c7f35
commit 5d86672
Show file tree

Hide file tree

Showing 11 changed files with 46 additions and 45 deletions.
diff --git a/README.md b/README.md
@@ -72,38 +72,38 @@ You can use InLong in the following ways：
 - Visualize your operations on [InLong dashboard](https://inlong.apache.org/docs/next/user_guide/dashboard_usage).
 
 ## Supported Data Nodes (Updating)
-| Type         | Name              | Version                      | Architecture          |
-|--------------|-------------------|------------------------------|-----------------------|
-| Extract Node | Auto Push         | None                         | Standard              |
-|              | File              | None                         | Standard              |
-|              | Kafka             | 2.x                          | Lightweight, Standard |
-|              | MongoDB           | >= 3.6                       | Lightweight, Standard |
-|              | MQTT              | >= 3.1                       | Standard              |
-|              | MySQL             | 5.6, 5.7, 8.0.x              | Lightweight, Standard |
-|              | Oracle            | 11,12,19                     | Lightweight           |
-|              | PostgreSQL        | 9.6, 10, 11, 12              | Lightweight, Standard |
-|              | Pulsar            | 2.8.x                        | Lightweight           |
-|              | Redis             | 2.6.x                        | Standard              |
-|              | SQLServer         | 2012, 2014, 2016, 2017, 2019 | Lightweight, Standard |
-| Load Node    | Auto Consumption  | None                         | Standard              |
-|              | ClickHouse        | 20.7+                        | Lightweight, Standard |
-|              | Elasticsearch     | 6.x, 7.x                     | Lightweight, Standard |
-|              | Greenplum         | 4.x, 5.x, 6.x                | Lightweight, Standard |
-|              | HBase             | 2.2.x                        | Lightweight, Standard |
-|              | HDFS              | 2.x, 3.x                     | Lightweight, Standard |
-|              | Hive              | 1.x, 2.x, 3.x                | Lightweight, Standard |
-|              | Iceberg           | 0.12.x                       | Lightweight, Standard |
-|              | Hudi              | 0.12.x                       | Lightweight, Standard |
-|              | Kafka             | 2.x                          | Lightweight, Standard |
-|              | MySQL             | 5.6, 5.7, 8.0.x              | Lightweight, Standard |
-|              | Oracle            | 11, 12, 19                   | Lightweight, Standard |
-|              | PostgreSQL        | 9.6, 10, 11, 12              | Lightweight, Standard |
-|              | SQLServer         | 2012, 2014, 2016, 2017, 2019 | Lightweight, Standard |
-|              | TDSQL-PostgreSQL  | 10.17                        | Lightweight, Standard |
-|              | Doris             | >= 0.13                      | Lightweight, Standard |
-|              | StarRocks         | >= 2.0                       | Lightweight, Standard |
-|              | Kudu              | >= 1.12.0                    | Lightweight, Standard |
-|              | Redis             | >= 3.0                       | Lightweight, Standard |
+| Type         | Name              | Version                      |
+|--------------|-------------------|------------------------------|
+| Extract Node | Auto Push         | None                         |
+|              | File              | None                         |
+|              | Kafka             | 2.x                          |
+|              | MongoDB           | >= 3.6                       |
+|              | MQTT              | >= 3.1                       |
+|              | MySQL             | 5.6, 5.7, 8.0.x              |
+|              | Oracle            | 11,12,19                     |
+|              | PostgreSQL        | 9.6, 10, 11, 12              |
+|              | Pulsar            | 2.8.x                        |
+|              | Redis             | 2.6.x                        |
+|              | SQLServer         | 2012, 2014, 2016, 2017, 2019 |
+| Load Node    | Auto Consumption  | None                         |
+|              | ClickHouse        | 20.7+                        |
+|              | Elasticsearch     | 6.x, 7.x                     |
+|              | Greenplum         | 4.x, 5.x, 6.x                |
+|              | HBase             | 2.2.x                        |
+|              | HDFS              | 2.x, 3.x                     |
+|              | Hive              | 1.x, 2.x, 3.x                |
+|              | Iceberg           | 0.12.x                       |
+|              | Hudi              | 0.12.x                       |
+|              | Kafka             | 2.x                          |
+|              | MySQL             | 5.6, 5.7, 8.0.x              |
+|              | Oracle            | 11, 12, 19                   |
+|              | PostgreSQL        | 9.6, 10, 11, 12              |
+|              | SQLServer         | 2012, 2014, 2016, 2017, 2019 |
+|              | TDSQL-PostgreSQL  | 10.17                        |
+|              | Doris             | >= 0.13                      |
+|              | StarRocks         | >= 2.0                       |
+|              | Kudu              | >= 1.12.0                    |
+|              | Redis             | >= 3.0                       |
 
 ## Build InLong
 More detailed instructions can be found at [Quick Start](https://inlong.apache.org/docs/next/quick_start/how_to_build) section in the documentation.

diff --git a/...anager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java b/...anager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
@@ -25,13 +25,13 @@
 public enum GroupMode {
 
     /**
-     * Standard group init with all components in Inlong Cluster
+     * Standard mode(include Data Ingestion and Synchronization): group init with all components in InLong Cluster
      * StreamSource -> Agent/SDK -> DataProxy -> MQ Cache -> Sort -> StreamSink
      */
     STANDARD("standard"),
 
     /**
-     * DataSync group init with sort in Inlong Cluster
+     * DataSync mode(only Data Synchronization): group init only with sort in InLong Cluster
      * StreamSource -> Sort -> StreamSink
      */
     DATASYNC("datasync");

diff --git a/...ager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupInfo.java b/...ager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupInfo.java
@@ -72,7 +72,7 @@ public abstract class InlongGroupInfo extends BaseInlongGroup {
     @ApiModelProperty(value = "Whether to enable create resource? 0: disable, 1: enable")
     private Integer enableCreateResource = 0;
 
-    @ApiModelProperty(value = "Standard mode: 0, DataSync mode: 1")
+    @ApiModelProperty(value = "Standard mode(include Data Ingestion and Synchronization): 0, DataSync mode(only Data Synchronization): 1")
     private Integer inlongGroupMode;
 
     @ApiModelProperty(value = "Data report type, default is 0.\n"

diff --git a/...nager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupPageRequest.java b/...nager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupPageRequest.java
@@ -58,7 +58,7 @@ public class InlongGroupPageRequest extends PageRequest {
     @ApiModelProperty(value = "The inlong cluster tag list")
     private List<String> clusterTagList;
 
-    @ApiModelProperty(value = "Standard mode: 0, DataSync mode: 1")
+    @ApiModelProperty(value = "Standard mode(include Data Ingestion and Synchronization): 0, DataSync mode(only Data Synchronization): 1")
     private Integer inlongGroupMode;
 
     @ApiModelProperty(value = "Current user", hidden = true)

diff --git a/...r/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupRequest.java b/...r/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupRequest.java
@@ -86,7 +86,7 @@ public abstract class InlongGroupRequest extends BaseInlongGroup {
     @Range(min = 0, max = 1, message = "default is 1, only supports [0: disable, 1: enable]")
     private Integer enableCreateResource = 1;
 
-    @ApiModelProperty(value = "Standard mode: 0, DataSync mode: 1")
+    @ApiModelProperty(value = "Standard mode(include Data Ingestion and Synchronization): 0, DataSync mode(only Data Synchronization): 1")
     @Range(min = 0, max = 1, message = "default is 0, only supports [0: Standard, 1: DataSync]")
     private Integer inlongGroupMode = 0;
 

diff --git a/.../main/java/org/apache/inlong/manager/service/resource/sort/DefaultSortConfigOperator.java b/.../main/java/org/apache/inlong/manager/service/resource/sort/DefaultSortConfigOperator.java
@@ -138,8 +138,8 @@ private GroupInfo getGroupInfo(InlongGroupInfo groupInfo, List<InlongStreamInfo>
             if (InlongConstants.STANDARD_MODE.equals(groupInfo.getInlongGroupMode())) {
                 if (CollectionUtils.isNotEmpty(transformResponses)) {
                     relations = NodeRelationUtils.createNodeRelations(inlongStream);
-                    // in standard mode, replace upstream source node and transform input fields node
-                    // to MQ node (which is inlong stream id)
+                    // in standard mode(include Data Ingestion and Synchronization), replace upstream source node and
+                    // transform input fields node to MQ node (which is InLong stream id)
                     String mqNodeName = sources.get(0).getSourceName();
                     Set<String> nodeNameSet = getInputNodeNames(sources, transformResponses);
                     adjustTransformField(transformResponses, nodeNameSet, mqNodeName);
@@ -181,7 +181,7 @@ private Set<String> getInputNodeNames(List<StreamSource> sources, List<Transform
     /**
      * Set origin node to mq node for transform fields if necessary.
      *
-     * In standard mode for InlongGroup, transform input node must either be
+     * In standard mode(include Data Ingestion and Synchronization) for InlongGroup, transform input node must either be
      * mq source node or transform node, otherwise replace it with mq node name.
      */
     private void adjustTransformField(List<TransformResponse> transforms, Set<String> nodeNameSet, String mqNodeName) {

diff --git a/...r-service/src/main/java/org/apache/inlong/manager/service/source/StreamSourceService.java b/...r-service/src/main/java/org/apache/inlong/manager/service/source/StreamSourceService.java
@@ -81,7 +81,7 @@ public interface StreamSourceService {
      * Get the StreamSource Map by the inlong group info and inlong stream info list.
      * <p/>
      * If the group mode is DATASYNC, means not using any MQ as a cached source, then just get all related sources.
-     * Otherwise, if the group mode is STANDARD, need get the cached MQ sources.
+     * Otherwise, if the group mode is STANDARD(include Data Ingestion and Synchronization), need get the cached MQ sources.
      *
      * @param groupInfo inlong group info
      * @param streamInfos inlong stream info list

diff --git a/...rvice/src/main/java/org/apache/inlong/manager/service/source/StreamSourceServiceImpl.java b/...rvice/src/main/java/org/apache/inlong/manager/service/source/StreamSourceServiceImpl.java
@@ -238,7 +238,8 @@ public Map<String, List<StreamSource>> getSourcesMap(InlongGroupInfo groupInfo,
                     .collect(Collectors.groupingBy(StreamSource::getInlongStreamId, HashMap::new,
                             Collectors.toCollection(ArrayList::new)));
         } else {
-            // if the group mode is STANDARD, needs to get the cached MQ sources
+            // if the group mode is STANDARD(include Data Ingestion and Synchronization),
+            // InLong needs to get the cached MQ sources
             StreamSourceOperator sourceOperator = operatorFactory.getInstance(groupInfo.getMqType());
             result = sourceOperator.getSourcesMap(groupInfo, streamInfos, streamSources);
         }

diff --git a/inlong-manager/manager-test/src/main/resources/h2/apache_inlong_manager.sql b/inlong-manager/manager-test/src/main/resources/h2/apache_inlong_manager.sql
@@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS `inlong_group`
     `max_length`             int(11)               DEFAULT '10240' COMMENT 'The maximum length of a single piece of data, unit: Byte',
     `enable_zookeeper`       tinyint(1)            DEFAULT '0' COMMENT 'Whether to enable the zookeeper, 0-disable, 1-enable',
     `enable_create_resource` tinyint(1)            DEFAULT '1' COMMENT 'Whether to enable create resource? 0-disable, 1-enable',
-    `inlong_group_mode`      tinyint(1)            DEFAULT '0' COMMENT 'Inlong group mode, Standard mode: 0, DataSync mode: 1',
+    `inlong_group_mode`      tinyint(1)            DEFAULT '0' COMMENT 'InLong group mode, Standard mode(include Data Ingestion and Synchronization): 0, DataSync mode(only Data Synchronization): 1',
     `data_report_type`       int(4)                DEFAULT '0' COMMENT 'Data report type. 0: report to DataProxy and respond when the DataProxy received data. 1: report to DataProxy and respond after DataProxy sends data. 2: report to MQ and respond when the MQ received data',
     `inlong_cluster_tag`     varchar(128)          DEFAULT NULL COMMENT 'The cluster tag, which links to inlong_cluster table',
     `ext_params`             mediumtext            DEFAULT NULL COMMENT 'Extended params, will be saved as JSON string',

diff --git a/inlong-manager/manager-web/sql/apache_inlong_manager.sql b/inlong-manager/manager-web/sql/apache_inlong_manager.sql
@@ -41,7 +41,7 @@ CREATE TABLE IF NOT EXISTS `inlong_group`
     `max_length`             int(11)               DEFAULT '10240' COMMENT 'The maximum length of a single piece of data, unit: Byte',
     `enable_zookeeper`       tinyint(1)            DEFAULT '0' COMMENT 'Whether to enable the zookeeper, 0-disable, 1-enable',
     `enable_create_resource` tinyint(1)            DEFAULT '1' COMMENT 'Whether to enable create resource? 0-disable, 1-enable',
-    `inlong_group_mode`      tinyint(1)            DEFAULT '0' COMMENT 'Inlong group mode, Standard mode: 0, DataSync mode: 1',
+    `inlong_group_mode`      tinyint(1)            DEFAULT '0' COMMENT 'InLong group mode, Standard mode(include Data Ingestion and Synchronization): 0, DataSync mode(only Data Synchronization): 1',
     `data_report_type`       int(4)                DEFAULT '0' COMMENT 'Data report type. 0: report to DataProxy and respond when the DataProxy received data. 1: report to DataProxy and respond after DataProxy sends data. 2: report to MQ and respond when the MQ received data',
     `inlong_cluster_tag`     varchar(128)          DEFAULT NULL COMMENT 'The cluster tag, which links to inlong_cluster table',
     `ext_params`             mediumtext            DEFAULT NULL COMMENT 'Extended params, will be saved as JSON string',

diff --git a/inlong-manager/manager-web/sql/changes-1.8.0.sql b/inlong-manager/manager-web/sql/changes-1.8.0.sql
@@ -24,7 +24,7 @@ SET FOREIGN_KEY_CHECKS = 0;
 USE `apache_inlong_manager`;
 
 ALTER TABLE inlong_group
-    CHANGE lightweight inlong_group_mode tinyint(1) DEFAULT 0 NULL COMMENT 'Inlong group mode, Standard mode: 0, DataSync mode: 1';
+    CHANGE lightweight inlong_group_mode tinyint(1) DEFAULT 0 NULL COMMENT 'InLong group mode, Standard mode(include Data Ingestion and Synchronization): 0, DataSync mode(only Data Synchronization): 1';
 
 -- To support multi-tenant management in InLong, see https://github.com/apache/inlong/issues/7914
 CREATE TABLE IF NOT EXISTS `inlong_tenant`