diff --git a/README.md b/README.md index a8abadbef7..ced3449496 100644 --- a/README.md +++ b/README.md @@ -72,38 +72,38 @@ You can use InLong in the following ways： - Visualize your operations on [InLong dashboard](https://inlong.apache.org/docs/next/user_guide/dashboard_usage). ## Supported Data Nodes (Updating) -| Type | Name | Version | Architecture | -|--------------|-------------------|------------------------------|-----------------------| -| Extract Node | Auto Push | None | Standard | -| | File | None | Standard | -| | Kafka | 2.x | Lightweight, Standard | -| | MongoDB | >= 3.6 | Lightweight, Standard | -| | MQTT | >= 3.1 | Standard | -| | MySQL | 5.6, 5.7, 8.0.x | Lightweight, Standard | -| | Oracle | 11,12,19 | Lightweight | -| | PostgreSQL | 9.6, 10, 11, 12 | Lightweight, Standard | -| | Pulsar | 2.8.x | Lightweight | -| | Redis | 2.6.x | Standard | -| | SQLServer | 2012, 2014, 2016, 2017, 2019 | Lightweight, Standard | -| Load Node | Auto Consumption | None | Standard | -| | ClickHouse | 20.7+ | Lightweight, Standard | -| | Elasticsearch | 6.x, 7.x | Lightweight, Standard | -| | Greenplum | 4.x, 5.x, 6.x | Lightweight, Standard | -| | HBase | 2.2.x | Lightweight, Standard | -| | HDFS | 2.x, 3.x | Lightweight, Standard | -| | Hive | 1.x, 2.x, 3.x | Lightweight, Standard | -| | Iceberg | 0.12.x | Lightweight, Standard | -| | Hudi | 0.12.x | Lightweight, Standard | -| | Kafka | 2.x | Lightweight, Standard | -| | MySQL | 5.6, 5.7, 8.0.x | Lightweight, Standard | -| | Oracle | 11, 12, 19 | Lightweight, Standard | -| | PostgreSQL | 9.6, 10, 11, 12 | Lightweight, Standard | -| | SQLServer | 2012, 2014, 2016, 2017, 2019 | Lightweight, Standard | -| | TDSQL-PostgreSQL | 10.17 | Lightweight, Standard | -| | Doris | >= 0.13 | Lightweight, Standard | -| | StarRocks | >= 2.0 | Lightweight, Standard | -| | Kudu | >= 1.12.0 | Lightweight, Standard | -| | Redis | >= 3.0 | Lightweight, Standard | +| Type | Name | Version | +|--------------|-------------------|------------------------------| +| Extract Node | Auto Push | None | +| | File | None | +| | Kafka | 2.x | +| | MongoDB | >= 3.6 | +| | MQTT | >= 3.1 | +| | MySQL | 5.6, 5.7, 8.0.x | +| | Oracle | 11,12,19 | +| | PostgreSQL | 9.6, 10, 11, 12 | +| | Pulsar | 2.8.x | +| | Redis | 2.6.x | +| | SQLServer | 2012, 2014, 2016, 2017, 2019 | +| Load Node | Auto Consumption | None | +| | ClickHouse | 20.7+ | +| | Elasticsearch | 6.x, 7.x | +| | Greenplum | 4.x, 5.x, 6.x | +| | HBase | 2.2.x | +| | HDFS | 2.x, 3.x | +| | Hive | 1.x, 2.x, 3.x | +| | Iceberg | 0.12.x | +| | Hudi | 0.12.x | +| | Kafka | 2.x | +| | MySQL | 5.6, 5.7, 8.0.x | +| | Oracle | 11, 12, 19 | +| | PostgreSQL | 9.6, 10, 11, 12 | +| | SQLServer | 2012, 2014, 2016, 2017, 2019 | +| | TDSQL-PostgreSQL | 10.17 | +| | Doris | >= 0.13 | +| | StarRocks | >= 2.0 | +| | Kudu | >= 1.12.0 | +| | Redis | >= 3.0 | ## Build InLong More detailed instructions can be found at [Quick Start](https://inlong.apache.org/docs/next/quick_start/how_to_build) section in the documentation. diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java index 6e5b358feb..f0db2353b6 100644 --- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java +++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java @@ -25,13 +25,13 @@ public enum GroupMode { /** - * Standard group init with all components in Inlong Cluster + * Standard mode(include Data Ingestion and Synchronization): group init with all components in InLong Cluster * StreamSource -> Agent/SDK -> DataProxy -> MQ Cache -> Sort -> StreamSink */ STANDARD("standard"), /** - * DataSync group init with sort in Inlong Cluster + * DataSync mode(only Data Synchronization): group init only with sort in InLong Cluster * StreamSource -> Sort -> StreamSink */ DATASYNC("datasync"); diff --git a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupInfo.java b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupInfo.java index 956cd214ad..978fe1b473 100644 --- a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupInfo.java +++ b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupInfo.java @@ -72,7 +72,7 @@ public abstract class InlongGroupInfo extends BaseInlongGroup { @ApiModelProperty(value = "Whether to enable create resource? 0: disable, 1: enable") private Integer enableCreateResource = 0; - @ApiModelProperty(value = "Standard mode: 0, DataSync mode: 1") + @ApiModelProperty(value = "Standard mode(include Data Ingestion and Synchronization): 0, DataSync mode(only Data Synchronization): 1") private Integer inlongGroupMode; @ApiModelProperty(value = "Data report type, default is 0.\n" diff --git a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupPageRequest.java b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupPageRequest.java index bf5cb82982..83855aa582 100644 --- a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupPageRequest.java +++ b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupPageRequest.java @@ -58,7 +58,7 @@ public class InlongGroupPageRequest extends PageRequest { @ApiModelProperty(value = "The inlong cluster tag list") private List clusterTagList; - @ApiModelProperty(value = "Standard mode: 0, DataSync mode: 1") + @ApiModelProperty(value = "Standard mode(include Data Ingestion and Synchronization): 0, DataSync mode(only Data Synchronization): 1") private Integer inlongGroupMode; @ApiModelProperty(value = "Current user", hidden = true) diff --git a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupRequest.java b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupRequest.java index ae1f6677ef..a9cc46fc8b 100644 --- a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupRequest.java +++ b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupRequest.java @@ -86,7 +86,7 @@ public abstract class InlongGroupRequest extends BaseInlongGroup { @Range(min = 0, max = 1, message = "default is 1, only supports [0: disable, 1: enable]") private Integer enableCreateResource = 1; - @ApiModelProperty(value = "Standard mode: 0, DataSync mode: 1") + @ApiModelProperty(value = "Standard mode(include Data Ingestion and Synchronization): 0, DataSync mode(only Data Synchronization): 1") @Range(min = 0, max = 1, message = "default is 0, only supports [0: Standard, 1: DataSync]") private Integer inlongGroupMode = 0; diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/DefaultSortConfigOperator.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/DefaultSortConfigOperator.java index b2a38db3dc..f97ff09bf7 100644 --- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/DefaultSortConfigOperator.java +++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/DefaultSortConfigOperator.java @@ -138,8 +138,8 @@ private GroupInfo getGroupInfo(InlongGroupInfo groupInfo, List if (InlongConstants.STANDARD_MODE.equals(groupInfo.getInlongGroupMode())) { if (CollectionUtils.isNotEmpty(transformResponses)) { relations = NodeRelationUtils.createNodeRelations(inlongStream); - // in standard mode, replace upstream source node and transform input fields node - // to MQ node (which is inlong stream id) + // in standard mode(include Data Ingestion and Synchronization), replace upstream source node and + // transform input fields node to MQ node (which is InLong stream id) String mqNodeName = sources.get(0).getSourceName(); Set nodeNameSet = getInputNodeNames(sources, transformResponses); adjustTransformField(transformResponses, nodeNameSet, mqNodeName); @@ -181,7 +181,7 @@ private Set getInputNodeNames(List sources, List transforms, Set nodeNameSet, String mqNodeName) { diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/StreamSourceService.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/StreamSourceService.java index 1b2333fe80..1bcb9f9966 100644 --- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/StreamSourceService.java +++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/StreamSourceService.java @@ -81,7 +81,7 @@ public interface StreamSourceService { * Get the StreamSource Map by the inlong group info and inlong stream info list. *

* If the group mode is DATASYNC, means not using any MQ as a cached source, then just get all related sources. - * Otherwise, if the group mode is STANDARD, need get the cached MQ sources. + * Otherwise, if the group mode is STANDARD(include Data Ingestion and Synchronization), need get the cached MQ sources. * * @param groupInfo inlong group info * @param streamInfos inlong stream info list diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/StreamSourceServiceImpl.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/StreamSourceServiceImpl.java index ee3c4ad70e..c18aef1fe9 100644 --- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/StreamSourceServiceImpl.java +++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/StreamSourceServiceImpl.java @@ -238,7 +238,8 @@ public Map> getSourcesMap(InlongGroupInfo groupInfo, .collect(Collectors.groupingBy(StreamSource::getInlongStreamId, HashMap::new, Collectors.toCollection(ArrayList::new))); } else { - // if the group mode is STANDARD, needs to get the cached MQ sources + // if the group mode is STANDARD(include Data Ingestion and Synchronization), + // InLong needs to get the cached MQ sources StreamSourceOperator sourceOperator = operatorFactory.getInstance(groupInfo.getMqType()); result = sourceOperator.getSourcesMap(groupInfo, streamInfos, streamSources); } diff --git a/inlong-manager/manager-test/src/main/resources/h2/apache_inlong_manager.sql b/inlong-manager/manager-test/src/main/resources/h2/apache_inlong_manager.sql index 1989ccfdea..0f4c8a76c0 100644 --- a/inlong-manager/manager-test/src/main/resources/h2/apache_inlong_manager.sql +++ b/inlong-manager/manager-test/src/main/resources/h2/apache_inlong_manager.sql @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS `inlong_group` `max_length` int(11) DEFAULT '10240' COMMENT 'The maximum length of a single piece of data, unit: Byte', `enable_zookeeper` tinyint(1) DEFAULT '0' COMMENT 'Whether to enable the zookeeper, 0-disable, 1-enable', `enable_create_resource` tinyint(1) DEFAULT '1' COMMENT 'Whether to enable create resource? 0-disable, 1-enable', - `inlong_group_mode` tinyint(1) DEFAULT '0' COMMENT 'Inlong group mode, Standard mode: 0, DataSync mode: 1', + `inlong_group_mode` tinyint(1) DEFAULT '0' COMMENT 'InLong group mode, Standard mode(include Data Ingestion and Synchronization): 0, DataSync mode(only Data Synchronization): 1', `data_report_type` int(4) DEFAULT '0' COMMENT 'Data report type. 0: report to DataProxy and respond when the DataProxy received data. 1: report to DataProxy and respond after DataProxy sends data. 2: report to MQ and respond when the MQ received data', `inlong_cluster_tag` varchar(128) DEFAULT NULL COMMENT 'The cluster tag, which links to inlong_cluster table', `ext_params` mediumtext DEFAULT NULL COMMENT 'Extended params, will be saved as JSON string', diff --git a/inlong-manager/manager-web/sql/apache_inlong_manager.sql b/inlong-manager/manager-web/sql/apache_inlong_manager.sql index c5c1e3a4f4..e131511bee 100644 --- a/inlong-manager/manager-web/sql/apache_inlong_manager.sql +++ b/inlong-manager/manager-web/sql/apache_inlong_manager.sql @@ -41,7 +41,7 @@ CREATE TABLE IF NOT EXISTS `inlong_group` `max_length` int(11) DEFAULT '10240' COMMENT 'The maximum length of a single piece of data, unit: Byte', `enable_zookeeper` tinyint(1) DEFAULT '0' COMMENT 'Whether to enable the zookeeper, 0-disable, 1-enable', `enable_create_resource` tinyint(1) DEFAULT '1' COMMENT 'Whether to enable create resource? 0-disable, 1-enable', - `inlong_group_mode` tinyint(1) DEFAULT '0' COMMENT 'Inlong group mode, Standard mode: 0, DataSync mode: 1', + `inlong_group_mode` tinyint(1) DEFAULT '0' COMMENT 'InLong group mode, Standard mode(include Data Ingestion and Synchronization): 0, DataSync mode(only Data Synchronization): 1', `data_report_type` int(4) DEFAULT '0' COMMENT 'Data report type. 0: report to DataProxy and respond when the DataProxy received data. 1: report to DataProxy and respond after DataProxy sends data. 2: report to MQ and respond when the MQ received data', `inlong_cluster_tag` varchar(128) DEFAULT NULL COMMENT 'The cluster tag, which links to inlong_cluster table', `ext_params` mediumtext DEFAULT NULL COMMENT 'Extended params, will be saved as JSON string', diff --git a/inlong-manager/manager-web/sql/changes-1.8.0.sql b/inlong-manager/manager-web/sql/changes-1.8.0.sql index 50b5991d5c..0a33640d1e 100644 --- a/inlong-manager/manager-web/sql/changes-1.8.0.sql +++ b/inlong-manager/manager-web/sql/changes-1.8.0.sql @@ -24,7 +24,7 @@ SET FOREIGN_KEY_CHECKS = 0; USE `apache_inlong_manager`; ALTER TABLE inlong_group - CHANGE lightweight inlong_group_mode tinyint(1) DEFAULT 0 NULL COMMENT 'Inlong group mode, Standard mode: 0, DataSync mode: 1'; + CHANGE lightweight inlong_group_mode tinyint(1) DEFAULT 0 NULL COMMENT 'InLong group mode, Standard mode(include Data Ingestion and Synchronization): 0, DataSync mode(only Data Synchronization): 1'; -- To support multi-tenant management in InLong, see https://github.com/apache/inlong/issues/7914 CREATE TABLE IF NOT EXISTS `inlong_tenant`