Merge branch '3.0' of https://github.com/taosdata/TDengine into enh/T…

…S-5554-3.0

.github/workflows/taoskeeper-ci.yml

@@ -0,0 +1,58 @@
+name: TaosKeeper CI
+
+on:
+  push:
+    paths:
+      - tools/keeper/**
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    name: Run unit tests
+
+    steps:
+      - name: Checkout the repository
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 1.18
+
+      - name: Install system dependencies
+        run: |
+          sudo apt update -y
+          sudo apt install -y build-essential cmake libgeos-dev
+
+      - name: Install TDengine
+        run: |
+          mkdir debug
+          cd debug
+          cmake .. -DBUILD_HTTP=false -DBUILD_JDBC=false -DBUILD_TOOLS=false -DBUILD_TEST=off -DBUILD_KEEPER=true
+          make -j 4
+          sudo make install
+          which taosd
+          which taosadapter
+          which taoskeeper
+
+      - name: Start taosd
+        run: |
+          cp /etc/taos/taos.cfg ./
+          sudo echo "supportVnodes 256" >> taos.cfg
+          nohup sudo taosd -c taos.cfg &
+
+      - name: Start taosadapter
+        run: nohup sudo taosadapter &
+
+      - name: Run tests with coverage
+        working-directory: tools/keeper
+        run: |
+          go mod tidy
+          go test -v -coverpkg=./... -coverprofile=coverage.out ./...
+          go tool cover -func=coverage.out
+
+      - name: Clean up
+        if: always()
+        run: |
+          if pgrep taosd; then sudo pkill taosd; fi
+          if pgrep taosadapter; then sudo pkill taosadapter; fi

cmake/cmake.define

@@ -1,6 +1,7 @@
 cmake_minimum_required(VERSION 3.0)
 set(CMAKE_VERBOSE_MAKEFILE FALSE)
 set(TD_BUILD_TAOSA_INTERNAL FALSE)
+set(TD_BUILD_KEEPER_INTERNAL FALSE)
 
 # set output directory
 SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib)
@@ -57,6 +58,19 @@ IF(TD_BUILD_HTTP)
     ADD_DEFINITIONS(-DHTTP_EMBEDDED)
 ENDIF()
 
+IF("${BUILD_KEEPER}" STREQUAL "")
+    SET(TD_BUILD_KEEPER FALSE)
+ELSEIF(${BUILD_KEEPER} MATCHES "false")
+    SET(TD_BUILD_KEEPER FALSE)
+ELSEIF(${BUILD_KEEPER} MATCHES "true")
+    SET(TD_BUILD_KEEPER TRUE)
+ELSEIF(${BUILD_KEEPER} MATCHES "internal")
+    SET(TD_BUILD_KEEPER FALSE)
+    SET(TD_BUILD_KEEPER_INTERNAL TRUE)
+ELSE()
+    SET(TD_BUILD_KEEPER FALSE)
+ENDIF()
+
 IF("${BUILD_TOOLS}" STREQUAL "")
     IF(TD_LINUX)
         IF(TD_ARM_32)

docs/en/14-reference/03-taos-sql/21-node.md

@@ -27,11 +27,15 @@ The preceding SQL command shows all dnodes in the cluster with the ID, endpoint,
 ## Delete a DNODE
 
 ```sql
-DROP DNODE dnode_id
+DROP DNODE dnode_id [force] [unsafe]
 ```
 
 Note that deleting a dnode does not stop its process. You must stop the process after the dnode is deleted.
 
+Only online node is allowed to be deleted. Drop is executed with force option if the offline node need to be deleted.
+
+Drop is executed with unsafe option if the node with single replica is offline, and the data on it is not able to be restored.
+
 ## Modify Dnode Configuration
 
 ```sql

docs/examples/java/src/main/java/com/taos/example/ConsumerLoopFull.java

@@ -1,8 +1,9 @@
 package com.taos.example;
 
-import com.alibaba.fastjson.JSON;
+import com.fasterxml.jackson.core.JsonProcessingException;
 import com.taosdata.jdbc.TSDBDriver;
 import com.taosdata.jdbc.tmq.*;
+import com.taosdata.jdbc.utils.JsonUtil;
 
 import java.sql.*;
 import java.time.Duration;
@@ -60,7 +61,7 @@ public static TaosConsumer<ResultBean> getConsumer() throws Exception {
 // ANCHOR_END: create_consumer
     }
 
-    public static void pollExample(TaosConsumer<ResultBean> consumer) throws SQLException {
+    public static void pollExample(TaosConsumer<ResultBean> consumer) throws SQLException, JsonProcessingException {
 // ANCHOR: poll_data_code_piece
         List<String> topics = Collections.singletonList("topic_meters");
         try {
@@ -73,7 +74,7 @@ public static void pollExample(TaosConsumer<ResultBean> consumer) throws SQLExce
                 for (ConsumerRecord<ResultBean> record : records) {
                     ResultBean bean = record.value();
                     // Add your data processing logic here
-                    System.out.println("data: " + JSON.toJSONString(bean));
+                    System.out.println("data: " + JsonUtil.getObjectMapper().writeValueAsString(bean));
                 }
             }
         } catch (Exception ex) {
@@ -91,15 +92,15 @@ public static void pollExample(TaosConsumer<ResultBean> consumer) throws SQLExce
 // ANCHOR_END: poll_data_code_piece
     }
 
-    public static void seekExample(TaosConsumer<ResultBean> consumer) throws SQLException {
+    public static void seekExample(TaosConsumer<ResultBean> consumer) throws SQLException, JsonProcessingException {
 // ANCHOR: consumer_seek
         List<String> topics = Collections.singletonList("topic_meters");
         try {
             // subscribe to the topics
             consumer.subscribe(topics);
             System.out.println("Subscribe topics successfully.");
             Set<TopicPartition> assignment = consumer.assignment();
-            System.out.println("Now assignment: " + JSON.toJSONString(assignment));
+            System.out.println("Now assignment: " + JsonUtil.getObjectMapper().writeValueAsString(assignment));
 
             ConsumerRecords<ResultBean> records = ConsumerRecords.emptyRecord();
             // make sure we have got some data
@@ -125,7 +126,7 @@ public static void seekExample(TaosConsumer<ResultBean> consumer) throws SQLExce
     }
 
 
-    public static void commitExample(TaosConsumer<ResultBean> consumer) throws SQLException {
+    public static void commitExample(TaosConsumer<ResultBean> consumer) throws SQLException, JsonProcessingException {
 // ANCHOR: commit_code_piece
         List<String> topics = Collections.singletonList("topic_meters");
         try {
@@ -135,7 +136,7 @@ public static void commitExample(TaosConsumer<ResultBean> consumer) throws SQLEx
                 for (ConsumerRecord<ResultBean> record : records) {
                     ResultBean bean = record.value();
                     // Add your data processing logic here
-                    System.out.println("data: " + JSON.toJSONString(bean));
+                    System.out.println("data: " + JsonUtil.getObjectMapper().writeValueAsString(bean));
                 }
                 if (!records.isEmpty()) {
                     // after processing the data, commit the offset manually

docs/examples/java/src/main/java/com/taos/example/ConsumerLoopImp.java

@@ -1,7 +1,7 @@
 package com.taos.example;
 
-import com.alibaba.fastjson.JSON;
 import com.taosdata.jdbc.TSDBDriver;
+import com.taosdata.jdbc.utils.JsonUtil;
 
 import java.sql.Connection;
 import java.sql.DriverManager;
@@ -31,7 +31,11 @@ public static void main(String[] args) throws SQLException, InterruptedException
         final AbsConsumerLoop consumerLoop = new AbsConsumerLoop() {
             @Override
             public void process(ResultBean result) {
-                System.out.println("data: " + JSON.toJSONString(result));
+                try{
+                    System.out.println("data: " + JsonUtil.getObjectMapper().writeValueAsString(result));
+                } catch (Exception e) {
+                    throw new RuntimeException(e);
+                }
             }
         };
 

docs/examples/java/src/main/java/com/taos/example/WsConsumerLoopFull.java

@@ -1,8 +1,9 @@
 package com.taos.example;
 
-import com.alibaba.fastjson.JSON;
+import com.fasterxml.jackson.core.JsonProcessingException;
 import com.taosdata.jdbc.TSDBDriver;
 import com.taosdata.jdbc.tmq.*;
+import com.taosdata.jdbc.utils.JsonUtil;
 
 import java.sql.*;
 import java.time.Duration;
@@ -60,7 +61,7 @@ public static TaosConsumer<ResultBean> getConsumer() throws Exception {
 // ANCHOR_END: create_consumer
     }
 
-    public static void pollExample(TaosConsumer<ResultBean> consumer) throws SQLException {
+    public static void pollExample(TaosConsumer<ResultBean> consumer) throws SQLException, JsonProcessingException {
 // ANCHOR: poll_data_code_piece
         List<String> topics = Collections.singletonList("topic_meters");
         try {
@@ -73,7 +74,7 @@ public static void pollExample(TaosConsumer<ResultBean> consumer) throws SQLExce
                 for (ConsumerRecord<ResultBean> record : records) {
                     ResultBean bean = record.value();
                     // Add your data processing logic here
-                    System.out.println("data: " + JSON.toJSONString(bean));
+                    System.out.println("data: " + JsonUtil.getObjectMapper().writeValueAsString(bean));
                 }
             }
         } catch (Exception ex) {
@@ -91,15 +92,15 @@ public static void pollExample(TaosConsumer<ResultBean> consumer) throws SQLExce
 // ANCHOR_END: poll_data_code_piece
     }
 
-    public static void seekExample(TaosConsumer<ResultBean> consumer) throws SQLException {
+    public static void seekExample(TaosConsumer<ResultBean> consumer) throws SQLException, JsonProcessingException {
 // ANCHOR: consumer_seek
         List<String> topics = Collections.singletonList("topic_meters");
         try {
             // subscribe to the topics
             consumer.subscribe(topics);
             System.out.println("Subscribe topics successfully.");
             Set<TopicPartition> assignment = consumer.assignment();
-            System.out.println("Now assignment: " + JSON.toJSONString(assignment));
+            System.out.println("Now assignment: " + JsonUtil.getObjectMapper().writeValueAsString(assignment));
 
             ConsumerRecords<ResultBean> records = ConsumerRecords.emptyRecord();
             // make sure we have got some data
@@ -125,7 +126,7 @@ public static void seekExample(TaosConsumer<ResultBean> consumer) throws SQLExce
     }
 
 
-    public static void commitExample(TaosConsumer<ResultBean> consumer) throws SQLException {
+    public static void commitExample(TaosConsumer<ResultBean> consumer) throws SQLException, JsonProcessingException {
 // ANCHOR: commit_code_piece
         List<String> topics = Collections.singletonList("topic_meters");
         try {
@@ -135,7 +136,7 @@ public static void commitExample(TaosConsumer<ResultBean> consumer) throws SQLEx
                 for (ConsumerRecord<ResultBean> record : records) {
                     ResultBean bean = record.value();
                     // Add your data processing logic here
-                    System.out.println("data: " + JSON.toJSONString(bean));
+                    System.out.println("data: " + JsonUtil.getObjectMapper().writeValueAsString(bean));
                 }
                 if (!records.isEmpty()) {
                     // after processing the data, commit the offset manually

docs/examples/java/src/main/java/com/taos/example/WsConsumerLoopImp.java

@@ -1,7 +1,7 @@
 package com.taos.example;
 
-import com.alibaba.fastjson.JSON;
 import com.taosdata.jdbc.TSDBDriver;
+import com.taosdata.jdbc.utils.JsonUtil;
 
 import java.sql.Connection;
 import java.sql.DriverManager;
@@ -28,7 +28,11 @@ public static void main(String[] args) throws SQLException, InterruptedException
         final AbsConsumerLoop consumerLoop = new AbsConsumerLoop() {
             @Override
             public void process(ResultBean result) {
-                System.out.println("data: " + JSON.toJSONString(result));
+                try{
+                    System.out.println("data: " + JsonUtil.getObjectMapper().writeValueAsString(result));
+                } catch (Exception e) {
+                    throw new RuntimeException(e);
+                }
             }
         };
 

docs/examples/java/src/main/java/com/taos/example/highvolume/DataBaseMonitor.java

@@ -13,6 +13,9 @@ public class DataBaseMonitor {
     public DataBaseMonitor init() throws SQLException {
         if (conn == null) {
             String jdbcURL = System.getenv("TDENGINE_JDBC_URL");
+            if (jdbcURL == null || jdbcURL == ""){
+                jdbcURL = "jdbc:TAOS://localhost:6030?user=root&password=taosdata";
+            }
             conn = DriverManager.getConnection(jdbcURL);
             stmt = conn.createStatement();
         }

docs/examples/java/src/main/java/com/taos/example/highvolume/SQLWriter.java

@@ -69,6 +69,9 @@ public SQLWriter(int maxBatchSize) {
      */
     private static Connection getConnection() throws SQLException {
         String jdbcURL = System.getenv("TDENGINE_JDBC_URL");
+        if (jdbcURL == null || jdbcURL == ""){
+            jdbcURL = "jdbc:TAOS://localhost:6030?user=root&password=taosdata";
+        }
         return DriverManager.getConnection(jdbcURL);
     }
 

docs/examples/java/src/test/java/com/taos/test/TestAll.java

@@ -17,6 +17,37 @@ public void dropDB(String dbName) throws SQLException {
                 stmt.execute("drop database if exists " + dbName);
             }
         }
+        waitTransaction();
+    }
+
+    public void dropTopic(String topicName) throws SQLException {
+        String jdbcUrl = "jdbc:TAOS://localhost:6030?user=root&password=taosdata";
+        try (Connection conn = DriverManager.getConnection(jdbcUrl)) {
+            try (Statement stmt = conn.createStatement()) {
+                stmt.execute("drop topic if exists " + topicName);
+            }
+        }
+        waitTransaction();
+    }
+
+    public void waitTransaction() throws SQLException {
+
+        String jdbcUrl = "jdbc:TAOS://localhost:6030?user=root&password=taosdata";
+        try (Connection conn = DriverManager.getConnection(jdbcUrl)) {
+            try (Statement stmt = conn.createStatement()) {
+                for (int i = 0; i < 10; i++) {
+                    stmt.execute("show transactions");
+                    try (ResultSet resultSet = stmt.getResultSet()) {
+                        if (resultSet.next()) {
+                            int count = resultSet.getInt(1);
+                            if (count == 0) {
+                                break;
+                            }
+                        }
+                    }
+                }
+            }
+        }
     }
 
     public void insertData() throws SQLException {
@@ -104,14 +135,20 @@ public void testConsumer() throws Exception {
         SubscribeDemo.main(args);
     }
 
-//    @Test
-//    public void testSubscribeJni() throws SQLException, InterruptedException {
-//        dropDB("power");
-//        ConsumerLoopFull.main(args);
-//    }
-//    @Test
-//    public void testSubscribeWs() throws SQLException, InterruptedException {
-//        dropDB("power");
-//        WsConsumerLoopFull.main(args);
-//    }
+    @Test
+    public void testSubscribeJni() throws SQLException, InterruptedException {
+        dropTopic("topic_meters");
+        dropDB("power");
+        ConsumerLoopFull.main(args);
+        dropTopic("topic_meters");
+        dropDB("power");
+    }
+    @Test
+    public void testSubscribeWs() throws SQLException, InterruptedException {
+        dropTopic("topic_meters");
+        dropDB("power");
+        WsConsumerLoopFull.main(args);
+        dropTopic("topic_meters");
+        dropDB("power");
+    }
 }

docs/zh/04-get-started/01-docker.md

@@ -17,7 +17,7 @@ docker pull tdengine/tdengine:latest
 或者指定版本的容器镜像：
 
 ```shell
-docker pull tdengine/tdengine:3.0.1.4
+docker pull tdengine/tdengine:3.3.3.0
 ```
 
 然后只需执行下面的命令：
@@ -121,4 +121,4 @@ SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters WHERE groupId = 1
 SELECT _wstart, AVG(current), MAX(voltage), MIN(phase) FROM test.d1001 INTERVAL(10s);
 ```
 
-在上面的查询中，使用系统提供的伪列_wstart 来给出每个窗口的开始时间。
+在上面的查询中，使用系统提供的伪列 _wstart 来给出每个窗口的开始时间。

docs/zh/04-get-started/03-package.md

@@ -317,4 +317,4 @@ SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters WHERE groupId = 1
 SELECT _wstart, AVG(current), MAX(voltage), MIN(phase) FROM test.d1001 INTERVAL(10s);
 ```
 
-在上面的查询中，使用系统提供的伪列_wstart 来给出每个窗口的开始时间。
+在上面的查询中，使用系统提供的伪列 _wstart 来给出每个窗口的开始时间。

docs/zh/04-get-started/_07-use.md

@@ -54,4 +54,4 @@ SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters WHERE groupId = 1
 SELECT _wstart, AVG(current), MAX(voltage), MIN(phase) FROM test.d1001 INTERVAL(10s);
 ```
 
-在上面的查询中，使用系统提供的伪列_wstart 来给出每个窗口的开始时间。
+在上面的查询中，使用系统提供的伪列 _wstart 来给出每个窗口的开始时间。