add wiki

README.md

@@ -1,11 +1,38 @@
 # Kratos
 
-Kratos是[bilibili](https://www.bilibili.com)开源的一套Go微服务框架，包含大量微服务相关框架及工具。  
-如：discovery(服务注册发现)、blademaster(HTTP框架)、warden(gRPC封装)、log、breaker、dapper(trace)、cache&db sdk、kratos tool(代码生成等工具)等等。  
+Kratos是[bilibili](https://www.bilibili.com)开源的一套Go微服务框架，包含大量微服务相关框架及工具。主要包括以下组件：
 
-我们致力于提供完整的微服务研发体验，大仓整合相关框架及工具后，微服务治理相关部分可对整体业务开发周期无感，从而更加聚焦于业务交付。对每位开发者而言，整套Kratos框架也是不错的学习仓库，可以了解和参考到[bilibili](https://www.bilibili.com)在微服务方面的技术积累和经验。
+* [http框架blademaster(bm)](doc/wiki-cn/blademaster.md)：基于[gin](https://github.com/gin-gonic/gin)二次开发，具有快速、灵活的特点，可以方便的开发中间件处理通用或特殊逻辑，基础库默认实现了log&trace等。
+* [gRPC框架warden](doc/wiki-cn/warden.md)：基于官方gRPC封装，默认使用[discovery](https://github.com/bilibili/discovery)进行服务注册发现，及wrr和p2c(默认)负载均衡。
+* [dapper trace](doc/wiki-cn/dapper.md)：基于opentracing，全链路集成了trace，我们还提供dapper实现，请参看：[dapper敬请期待]()。
+* [log](doc/wiki-cn/logger.md)：基于[zap](https://github.com/uber-go/zap)的field方式实现的高性能log库，集成了我们提供的[log-agent敬请期待]()日志收集方案。
+* [database](doc/wiki-cn/database.md)：集成MySQL&HBase&TiDB的SDK，其中TiDB使用服务发现方案。
+* [cache](doc/wiki-cn/cache.md)：集成memcache&redis的SDK，注意无redis-cluster实现，推荐使用代理模式[overlord](https://github.com/bilibili/overlord)。
+* [kratos tool](doc/wiki-cn/kratos-tool.md)：kratos相关工具量，包括项目快速生成、pb文件代码生成、swagger文档生成等。
+
+我们致力于提供完整的微服务研发体验，整合相关框架及工具后，微服务治理相关部分可对整体业务开发周期无感，从而更加聚焦于业务交付。对每位开发者而言，整套Kratos框架也是不错的学习仓库，可以了解和参考到[bilibili](https://www.bilibili.com)在微服务方面的技术积累和经验。
+
+# 快速开始
+
+```shell
+go get -u github.com/bilibili/kratos/tool/kratos
+kratos init
+```
+
+`kratos init`会快速生成基于kratos库的脚手架代码，如生成[kratos-demo](https://github.com/bilibili/kratos-demo)
+
+```shell
+cd kratos-demo/cmd
+go build
+./cmd -conf ../configs
+```
+
+打开浏览器访问：[http://localhost:8000/kratos-demo/start](http://localhost:8000/kratos-demo/start)，你会看到输出了`Golang 大法好 ！！！`
+
+# Document
+
+[简体中文](doc/wiki-cn/summary.md)
 
 -------------
 
 *Please report bugs, concerns, suggestions by issues, or join QQ-group 716486124 to discuss problems around source code.*
-

doc/wiki-cn/summary.md

@@ -0,0 +1,22 @@
+# Summary
+
+* [介绍](README.md)
+  * [快速开始](quickstart.md)
+  * [案例](https://github.com/bilibili/kratos-demo)
+* [http blademaster](blademaster.md)
+  * [middleware](blademaster-mid.md) 
+  * [protobuf生成](blademaster-pb.md) 
+* [grpc warden](warden.md)
+  * [middleware](warden-mid.md) 
+  * [protobuf生成](warden-pb.md) 
+* [dapper trace](dapper.md)
+* [log](logger.md)
+  * [log-agent](log-agent.md)
+* [database](database.md)
+  * [mysql](database-mysql.md)
+  * [hbase](database-hbase.md)
+  * [tidb](database-tidb.md)
+* [cache](cache.md)
+  * [memcache](cache-mc.md)
+  * [redis](cache-redis.md)
+* [kratos tool](kratos-tool.md)

go.mod

@@ -1,11 +1,14 @@
 module github.com/bilibili/kratos
 
+go 1.12
+
 require (
 	github.com/BurntSushi/toml v0.3.1
 	github.com/aristanetworks/goarista v0.0.0-20190409234242-46f4bc7b73ef // indirect
 	github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d // indirect
 	github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
 	github.com/cznic/strutil v0.0.0-20181122101858-275e90344537 // indirect
+	github.com/dgryski/go-farm v0.0.0-20190323231341-8198c7b169ec
 	github.com/fatih/color v1.7.0
 	github.com/fsnotify/fsnotify v1.4.7
 	github.com/go-playground/locales v0.12.1 // indirect
@@ -15,19 +18,20 @@ require (
 	github.com/golang/protobuf v1.2.0
 	github.com/kr/pty v1.1.4
 	github.com/leodido/go-urn v1.1.0 // indirect
+	github.com/montanaflynn/stats v0.5.0
 	github.com/pkg/errors v0.8.1
 	github.com/prometheus/client_golang v0.9.2
 	github.com/remyoudompheng/bigfft v0.0.0-20190321074620-2f0d2b0e0001 // indirect
 	github.com/samuel/go-zookeeper v0.0.0-20180130194729-c4fab1ac1bec // indirect
 	github.com/sirupsen/logrus v1.4.1 // indirect
+	github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a
 	github.com/stretchr/testify v1.3.0
 	github.com/tsuna/gohbase v0.0.0-20190201102810-d3184c1526df
 	github.com/urfave/cli v1.20.0
+	golang.org/x/net v0.0.0-20190311183353-d8887717615a
 	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect
 	google.golang.org/grpc v1.18.0
 	gopkg.in/AlecAivazis/survey.v1 v1.8.2
 	gopkg.in/go-playground/assert.v1 v1.2.1 // indirect
 	gopkg.in/go-playground/validator.v9 v9.26.0
 )
-
-go 1.12

pkg/database/tidb/discovery.go

@@ -16,15 +16,13 @@ var _schema = "tidb://"
 
 func (db *DB) nodeList() (nodes []string) {
 	var (
-		insInfo *naming.InstancesInfo
-		insMap  map[string][]*naming.Instance
-		ins     []*naming.Instance
-		ok      bool
+		insMap map[string][]*naming.Instance
+		ins    []*naming.Instance
+		ok     bool
 	)
-	if insInfo, ok = db.dis.Fetch(context.Background()); !ok {
+	if insMap, ok = db.dis.Fetch(context.Background()); !ok {
 		return
 	}
-	insMap = insInfo.Instances
 	if ins, ok = insMap[env.Zone]; !ok || len(ins) == 0 {
 		return
 	}

pkg/naming/discovery/discovery.go

@@ -59,6 +59,12 @@ type Config struct {
 	Host  string
 }
 
+type appData struct {
+	ZoneInstances map[string][]*naming.Instance `json:"zone_instances"`
+	LastTs        int64                         `json:"latest_timestamp"`
+	Err           string                        `json:"err"`
+}
+
 // Discovery is discovery client.
 type Discovery struct {
 	once       sync.Once
@@ -175,15 +181,15 @@ func (d *Discovery) selfproc(resolver naming.Resolver, event <-chan struct{}) {
 		if !ok {
 			return
 		}
-		instances, ok := resolver.Fetch(context.Background())
+		zones, ok := resolver.Fetch(context.Background())
 		if ok {
-			d.newSelf(instances)
+			d.newSelf(zones)
 		}
 	}
 }
 
-func (d *Discovery) newSelf(instances *naming.InstancesInfo) {
-	ins, ok := instances.Instances[d.conf.Zone]
+func (d *Discovery) newSelf(zones map[string][]*naming.Instance) {
+	ins, ok := zones[d.conf.Zone]
 	if !ok {
 		return
 	}
@@ -270,12 +276,12 @@ func (r *Resolver) Watch() <-chan struct{} {
 }
 
 // Fetch fetch resolver instance.
-func (r *Resolver) Fetch(c context.Context) (ins *naming.InstancesInfo, ok bool) {
+func (r *Resolver) Fetch(c context.Context) (ins map[string][]*naming.Instance, ok bool) {
 	r.d.mutex.RLock()
 	app, ok := r.d.apps[r.id]
 	r.d.mutex.RUnlock()
 	if ok {
-		ins, ok = app.zoneIns.Load().(*naming.InstancesInfo)
+		ins, ok = app.zoneIns.Load().(map[string][]*naming.Instance)
 		return
 	}
 	return
@@ -527,7 +533,6 @@ func (d *Discovery) serverproc() {
 			return
 		default:
 		}
-
 		apps, err := d.polls(ctx, d.pickNode())
 		if err != nil {
 			d.switchNode()
@@ -572,7 +577,7 @@ func (d *Discovery) nodes() (nodes []string) {
 	return
 }
 
-func (d *Discovery) polls(ctx context.Context, host string) (apps map[string]naming.InstancesInfo, err error) {
+func (d *Discovery) polls(ctx context.Context, host string) (apps map[string]appData, err error) {
 	var (
 		lastTs  []int64
 		appid   []string
@@ -597,8 +602,9 @@ func (d *Discovery) polls(ctx context.Context, host string) (apps map[string]nam
 	}
 	uri := fmt.Sprintf(_pollURL, host)
 	res := new(struct {
-		Code int                             `json:"code"`
-		Data map[string]naming.InstancesInfo `json:"data"`
+		Code    int                `json:"code"`
+		Message string             `json:"message"`
+		Data    map[string]appData `json:"data"`
 	})
 	params := url.Values{}
 	params.Set("env", conf.Env)
@@ -611,8 +617,17 @@ func (d *Discovery) polls(ctx context.Context, host string) (apps map[string]nam
 	}
 	if ec := ecode.Int(res.Code); !ec.Equal(ecode.OK) {
 		if !ec.Equal(ecode.NotModified) {
-			log.Error("discovery: client.Get(%s) get error code(%d)", uri+"?"+params.Encode(), res.Code)
+			log.Error("discovery: client.Get(%s) get error code(%d) message(%s)", uri+"?"+params.Encode(), res.Code, res.Message)
 			err = ec
+			if ec.Equal(ecode.NothingFound) {
+				for appID, value := range res.Data {
+					if value.Err != "" {
+						errInfo := fmt.Sprintf("discovery: app(%s) on ENV(%s) %s!\n", appID, conf.Env, value.Err)
+						log.Error(errInfo)
+						fmt.Fprintf(os.Stderr, errInfo)
+					}
+				}
+			}
 		}
 		return
 	}
@@ -630,12 +645,12 @@ func (d *Discovery) polls(ctx context.Context, host string) (apps map[string]nam
 	return
 }
 
-func (d *Discovery) broadcast(apps map[string]naming.InstancesInfo) {
+func (d *Discovery) broadcast(apps map[string]appData) {
 	for id, v := range apps {
 		var count int
-		for zone, ins := range v.Instances {
+		for zone, ins := range v.ZoneInstances {
 			if len(ins) == 0 {
-				delete(v.Instances, zone)
+				delete(v.ZoneInstances, zone)
 			}
 			count += len(ins)
 		}
@@ -647,7 +662,7 @@ func (d *Discovery) broadcast(apps map[string]naming.InstancesInfo) {
 		d.mutex.RUnlock()
 		if ok {
 			app.lastTs = v.LastTs
-			app.zoneIns.Store(v)
+			app.zoneIns.Store(v.ZoneInstances)
 			d.mutex.RLock()
 			for rs := range app.resolver {
 				select {

pkg/naming/naming.go

@@ -6,19 +6,17 @@ import (
 
 // metadata common key
 const (
-	MetaColor   = "color"
-	MetaWeight  = "weight"
-	MetaCluster = "cluster"
 	MetaZone    = "zone"
+	MetaCluster = "cluster"
+	MetaWeight  = "weight"
+	MetaColor   = "color"
 )
 
 // Instance represents a server the client connects to.
 type Instance struct {
-	// Region bj/sh/gz
-	Region string `json:"region"`
 	// Zone is IDC.
 	Zone string `json:"zone"`
-	// Env prod/pre、uat/fat1
+	// Env prod/pre/uat/fat1
 	Env string `json:"env"`
 	// AppID is mapping servicetree appid.
 	AppID string `json:"appid"`
@@ -34,25 +32,13 @@ type Instance struct {
 	// Metadata is the information associated with Addr, which may be used
 	// to make load balancing decision.
 	Metadata map[string]string `json:"metadata"`
-	Status   int64
-}
-
-// InstancesInfo instance info.
-type InstancesInfo struct {
-	Instances map[string][]*Instance `json:"zone_instances"`
-	LastTs    int64                  `json:"latest_timestamp"`
-	Scheduler []*Scheduler           `json:"scheduler"`
-}
-
-// Scheduler  scheduler info in multi cluster.
-type Scheduler struct {
-	Src string           `json:"src"`
-	Dst map[string]int64 `json:"dst"`
+	// Status status
+	Status int64
 }
 
 // Resolver resolve naming service
 type Resolver interface {
-	Fetch(context.Context) (*InstancesInfo, bool)
+	Fetch(context.Context) (map[string][]*Instance, bool)
 	Watch() <-chan struct{}
 	Close() error
 }

pkg/net/http/blademaster/README.md

@@ -0,0 +1,5 @@
+#### net/http/blademaster
+
+##### 项目简介
+
+http 框架，带来如飞一般的体验。

pkg/net/http/blademaster/client.go

@@ -137,7 +137,7 @@ func (client *Client) SetConfig(c *ClientConfig) {
 // TODO(zhoujiahui): param realIP should be removed later.
 func (client *Client) NewRequest(method, uri, realIP string, params url.Values) (req *xhttp.Request, err error) {
 	if method == xhttp.MethodGet {
-		req, err = xhttp.NewRequest(xhttp.MethodGet, uri+params.Encode(), nil)
+		req, err = xhttp.NewRequest(xhttp.MethodGet, fmt.Sprintf("%s?%s", uri, params.Encode()), nil)
 	} else {
 		req, err = xhttp.NewRequest(xhttp.MethodPost, uri, strings.NewReader(params.Encode()))
 	}

pkg/net/netutil/breaker/README.md

@@ -0,0 +1,21 @@
+#### breaker
+
+##### 项目简介
+1. 提供熔断器功能，供各种client（如rpc、http、msyql）等进行熔断
+2. 提供Go方法供业务在breaker熔断前后进行回调处理
+
+##### 配置说明
+> 1. NewGroup(name string,c *Config)当c==nil时则采用默认配置
+> 2. 可通过breaker.Init(c *Config)替换默认配置
+> 3. 可通过group.Reload(c *Config)进行配置更新
+> 4. 默认配置如下所示：
+    _conf = &Config{
+            Window:  xtime.Duration(3 * time.Second),
+            Sleep:   xtime.Duration(100 * time.Millisecond),
+            Bucket:  10,
+            Ratio:   0.5,
+            Request: 100,
+    }
+
+##### 测试
+1. 执行当前目录下所有测试文件，测试所有功能