add integrate test

.gitlab-ci.yml

@@ -9,6 +9,7 @@ stages:
   - check
   - test
   - build
+  - integrate_tests
   - e2e-test
 
 code_check:
@@ -26,6 +27,15 @@ build_branch:
   script:
     - pwd
 
+
+integrate_tests:
+  stage: integrate_tests
+  script:
+    - echo "start integrate tests"
+    - nohup bin/gaea -config etc/gaea.ini &
+    - sleep 10
+    - make integrate_test
+
 e2e_test:
   stage: e2e-test
   script:

Makefile

@@ -33,5 +33,11 @@ test:
 	tail -1 .coverage.func
 	go tool cover -html=.coverage.out -o .coverage.html
 
+integrate_test:
+	go test -coverprofile=.integrate_coverage.out ./... -run ^TestIntegration$
+	go tool cover -func=.integrate_coverage.out -o .integrate_coverage.func
+	tail -1 .coverage.func
+	go tool cover -html=.integrate_coverage.out -o .integrate_coverage.html
+
 build_with_coverage:
 	go test -c cmd/gaea/main.go cmd/gaea/main_test.go -coverpkg ./... -covermode=count -o bin/gaea

cmd/gaea/main_test.go

@@ -11,12 +11,164 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
 package main
 
 import (
+	"database/sql"
+	_ "github.com/go-sql-driver/mysql"
+	"github.com/stretchr/testify/assert"
+	"io/ioutil"
+	"strings"
 	"testing"
+	"time"
 )
 
+const MYSQL_DRIVER_NAME = "mysql"
+
 func Test_Main(t *testing.T) {
 	main()
 }
+
+func getConnection(proxyUrl, mysqlUrl string) (*sql.DB, *sql.DB, error) {
+	proxyDb, err := sql.Open(MYSQL_DRIVER_NAME, proxyUrl)
+	if err != nil {
+		return nil, nil, err
+	}
+	proxyDb.SetConnMaxLifetime(time.Minute * 3)
+	proxyDb.SetMaxOpenConns(10)
+	proxyDb.SetMaxIdleConns(10)
+
+	mysqlDb, err := sql.Open(MYSQL_DRIVER_NAME, mysqlUrl)
+	if err != nil {
+		return nil, nil, err
+	}
+	mysqlDb.SetConnMaxLifetime(time.Minute * 3)
+	mysqlDb.SetMaxOpenConns(10)
+	mysqlDb.SetMaxIdleConns(10)
+	return proxyDb, mysqlDb, nil
+}
+
+func rawToString(raw sql.RawBytes) string {
+	if raw == nil {
+		return "NULL"
+	}
+
+	return string(raw)
+}
+
+func checkResult(proxyResult, mysqlresult *sql.Rows, t *testing.T) error {
+	columns, _ := proxyResult.Columns()
+	columnSize := len(columns)
+
+	proxyValues := make([]sql.RawBytes, len(columns))
+	proxyArgs := make([]interface{}, columnSize)
+	for i := range proxyValues {
+		proxyArgs[i] = &proxyValues[i]
+	}
+
+	mysqlValues := make([]sql.RawBytes, len(columns))
+	mysqlArgs := make([]interface{}, columnSize)
+	for i := range mysqlValues {
+		mysqlArgs[i] = &mysqlValues[i]
+	}
+
+	for {
+		b1 := proxyResult.Next()
+		if !b1 {
+			if mysqlresult.Next() {
+				assert.Fail(t, "row number is not equal")
+			}
+			break
+		}
+
+		b2 := mysqlresult.Next()
+		if !b2 {
+			assert.Fail(t, "row number is not equal")
+			break
+		}
+
+		if err := proxyResult.Scan(proxyArgs...); err != nil {
+			return err
+		}
+
+		if err := mysqlresult.Scan(mysqlArgs...); err != nil {
+			return err
+		}
+
+		for idx := range proxyValues {
+			assert.Equal(t, rawToString(proxyValues[idx]), rawToString(mysqlValues[idx]))
+		}
+	}
+
+	return nil
+}
+
+// Test the proxy
+func TestIntegration(t *testing.T) {
+	// the following code can be refator to a function
+	// maybe we should encode the username and password
+	proxyUrl := "dbtest:cUBm0Lvf468DkoS9eVuFWDDGczMvFkNw@tcp(127.0.0.1:13306)/sbtest1"
+	mysqlUrl := "dbtest:cUBm0Lvf468DkoS9eVuFWDDGczMvFkNw@tcp(10.38.164.125:3308)/sbtest1"
+	proxyDb, mysqlDb, err := getConnection(proxyUrl, mysqlUrl)
+	if err != nil {
+		panic(err)
+	}
+
+	defer func() {
+		proxyDb.Close()
+		proxyDb.Close()
+	}()
+
+	//
+	sqlFiles, err := ioutil.ReadDir("../../sql_cases")
+	if err != nil {
+		assert.Fail(t, err.Error())
+	}
+
+	for _, fs := range sqlFiles {
+		bys, err := ioutil.ReadFile("../../sql_cases/" + fs.Name())
+		if err != nil {
+			assert.Fail(t, err.Error())
+		}
+
+		for _, sqlString := range strings.Split(string(bys), "\n") {
+			trimSql := strings.TrimSpace(sqlString)
+			if strings.HasPrefix(trimSql, "//") || trimSql == "" {
+				continue
+			}
+
+			if err = retryer(t, proxyDb, mysqlDb, sqlString, doCheck); err != nil {
+				assert.Fail(t, err.Error())
+			}
+		}
+	}
+}
+
+func retryer(t *testing.T, proxyDb, mysqlDb *sql.DB, sqlString string,
+	fn func(t *testing.T, proxyDb, mysqlDb *sql.DB, sqlString string) error) error {
+	var retryTimes = 3
+	var err error
+	for i := 0; i < retryTimes; i++ {
+		if err = fn(t, proxyDb, mysqlDb, sqlString); err == nil {
+			return nil
+		}
+	}
+
+	return err
+}
+
+func doCheck(t *testing.T, proxyDb, mysqlDb *sql.DB, sqlString string) error {
+	var r1, r2 *sql.Rows
+	var err error
+
+	//To make the test more, we should add retry methetism
+	if r1, err = proxyDb.Query(sqlString); err != nil {
+		return err
+	}
+	if r2, err = mysqlDb.Query(sqlString); err != nil {
+		return err
+	}
+
+	return checkResult(r1, r2, t)
+}

etc/gaea.ini

@@ -1,11 +1,13 @@
 ; config type, etcd/file/etcdv3, you can test gaea with file type, you shoud use etcd/etcdv3 in production
 ; 请指定设定方式为 file 或 etcd 或 etcdv3
-config_type=etcd
+;config_type=file
+config_type=etcdv3
 ;file config path, 具体配置放到file_config_path的namespace目录下，该下级目录为固定目录
-file_config_path=./etc/file
+;file_config_path=./etc/file
+;file_config_path=/etc/
 
 ;coordinator addr
-coordinator_addr=http://127.0.0.1:2379
+coordinator_addr=http://10.38.160.113:2379
 ;etcd user config
 username=root
 password=root

go.mod

@@ -50,6 +50,7 @@ require (
 	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/ini.v1 v1.42.0
 	gopkg.in/yaml.v2 v2.4.0 // indirect
+	github.com/go-sql-driver/mysql v1.6.0
 )
 
 replace github.com/dgrijalva/jwt-go => github.com/golang-jwt/jwt v3.2.2-0.20210713063142-860640e8862d+incompatible

go.sum

@@ -41,6 +41,8 @@ github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+
 github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
 github.com/go-playground/validator/v10 v10.8.0 h1:1kAa0fCrnpv+QYdkdcRzrRM7AyYs5o8+jZdJCz9xj6k=
 github.com/go-playground/validator/v10 v10.8.0/go.mod h1:9JhgTzTaE31GZDpH/HSvHiRJrJ3iKAgqqH0Bl/Ocjdk=
+github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
+github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c=

sql_cases/test1.sql

@@ -0,0 +1,2 @@
+select * from sbtest1.t1 where id > 1 order by id desc limit 10;
+select * from sbtest1.t2 where id < 0;

sql_cases/test2.sql

@@ -0,0 +1 @@
+select * from sbtest1.t1 where id > 4 and id <=8 order by name desc;