Nodes now register and pull rom services in dashboard

.gitignore

@@ -20,6 +20,8 @@ tyk.conf.bak
 tyk.folder.conf
 tyk.db.conf
 tyk.compose.conf
+ty.clonedash.conf
+tyk.dashboard.conf
 tyk.live.conf
 tyk.compose.rpc2.conf
 run.sh

CHANGELOG.md

@@ -12,6 +12,29 @@
 
 - Purger has been removed from core completely (replaced by Pump)
 - More tracing and more logging has been put in place, especially around API operations for easy tracing
+- Hot reloads (redis based), will not reset the muxer if no specs are returned (mproves stability if a data source goes down or fails)
+- REMOVED: MongoDB poller
+- REMOVED: MongoDB Policy poller
+- ADDED: Service-based API Definition loader (dashboard)
+- ADDED: Service-based Policy loader (dashboard)
+- To configure for API Defs: 
+
+	"use_db_app_configs": true,
+    "db_app_conf_options": {
+        "connection_string": "http://dashboard_host:port",
+        "node_id": "<your-node-id>",
+        "node_is_segmented": false,
+        "tags": []
+    },
+
+- To configure for Policies: 
+
+	"policies": {
+    	"policy_source": "service",
+        "policy_connection_string": "http://dashboard_host:port"
+    },
+
+- Tyk nodes require a valid nodeID that is supported by the dashboard in order to register. Node ID's cannot be cloned, the cloned ID will load, but will bounce other node connections.
 
 # 1.9.1.1
 

api_definition_manager.go

@@ -8,7 +8,6 @@ import (
 	"github.com/lonelycode/tykcommon"
 	"github.com/rubyist/circuitbreaker"
 	"gopkg.in/mgo.v2"
-	"gopkg.in/mgo.v2/bson"
 	"io/ioutil"
 	"net/http"
 	"net/url"
@@ -128,6 +127,9 @@ type APIDefinitionLoader struct {
 	dbSession *mgo.Session
 }
 
+// Nonce to use when interacting with the dashboard service
+var ServiceNonce string
+
 // Connect connects to the storage engine - can be null
 func (a *APIDefinitionLoader) Connect() {
 	var err error
@@ -229,43 +231,158 @@ func (a *APIDefinitionLoader) MakeSpec(thisAppConfig tykcommon.APIDefinition) AP
 	return newAppSpec
 }
 
-// LoadDefinitionsFromMongo will connect and download ApiDefintions from a Mongo DB instance.
-func (a *APIDefinitionLoader) LoadDefinitionsFromMongo() *[]*APISpec {
+func (a *APIDefinitionLoader) readBody(response *http.Response) ([]byte, error) {
+	defer response.Body.Close()
+	contents, err := ioutil.ReadAll(response.Body)
+
+	if err != nil {
+		return []byte(""), err
+	}
+
+	return contents, nil
+
+}
+
+func RegisterNodeWithDashboard(endpoint string, nodeID string) error {
+	// Get the definitions
+	log.Debug("Calling: ", endpoint)
+	newRequest, err := http.NewRequest("PUT", endpoint, nil)
+	if err != nil {
+		log.Error("Failed to create request: ", err)
+	}
+
+	newRequest.Header.Add("authorization", nodeID)
+
+	c := &http.Client{}
+	response, reqErr := c.Do(newRequest)
+
+	if reqErr != nil {
+		log.Error("Request failed: ", reqErr)
+		return reqErr
+	}
+
+	defer response.Body.Close()
+	retBody, err := ioutil.ReadAll(response.Body)
+
+	if err != nil {
+		return err
+	}
+
+	// Extract tagged APIs#
+	type NodeResponseOK struct {
+		Status  string
+		Message interface{}
+		Nonce   string
+	}
+
+	thisVal := NodeResponseOK{}
+	decErr := json.Unmarshal(retBody, &thisVal)
+	if decErr != nil {
+		log.Error("Failed to decode body: ", decErr)
+		return decErr
+	}
+
+	// Set the nonce
+	ServiceNonce = thisVal.Nonce
+
+	return nil
+}
+
+// LoadDefinitionsFromDashboardService will connect and download ApiDefintions from a Tyk Dashboard instance.
+func (a *APIDefinitionLoader) LoadDefinitionsFromDashboardService(endpoint string, nodeID string) *[]*APISpec {
 	var APISpecs = []*APISpec{}
 
-	a.Connect()
-	apiCollection := a.dbSession.DB("").C("tyk_apis")
+	// Get the definitions
+	log.Debug("Calling: ", endpoint)
+	newRequest, err := http.NewRequest("GET", endpoint, nil)
+	if err != nil {
+		log.Error("Failed to create request: ", err)
+	}
+
+	newRequest.Header.Add("authorization", nodeID)
+	newRequest.Header.Add("x-tyk-nonce", ServiceNonce)
+
+	c := &http.Client{}
+	response, reqErr := c.Do(newRequest)
 
-	search := bson.M{
-		"active": true,
+	if reqErr != nil {
+		log.Error("Request failed: ", reqErr)
+		return &APISpecs
 	}
 
-	// enable segments
-	if config.DBAppConfOptions.NodeIsSegmented {
-		log.Info("Segmented node, loading: ", config.DBAppConfOptions.Tags)
-		search["tags"] = bson.M{"$in": config.DBAppConfOptions.Tags}
+	retBody, bErr := a.readBody(response)
+	if bErr != nil {
+		log.Error("Failed to read body: ", bErr)
+		return &APISpecs
 	}
 
-	var APIDefinitions = []tykcommon.APIDefinition{}
-	var StringDefs = make([]bson.M, 0)
-	mongoErr := apiCollection.Find(search).Sort("-sort_by").All(&APIDefinitions)
+	// Extract tagged APIs#
 
-	if mongoErr != nil {
-		log.Error("Could not find any application configs!: ", mongoErr)
+	type ResponseStruct struct {
+		ApiDefinition tykcommon.APIDefinition `bson:"api_definition" json:"api_definition"`
+	}
+	type NodeResponseOK struct {
+		Status  string
+		Message []ResponseStruct
+		Nonce   string
+	}
+
+	thisList := NodeResponseOK{}
+
+	decErr := json.Unmarshal(retBody, &thisList)
+	if decErr != nil {
+		log.Error("Failed to decode body: ", decErr)
 		return &APISpecs
 	}
 
-	apiCollection.Find(search).All(&StringDefs)
+	thisRawList := make(map[string]interface{})
+	rawdecErr := json.Unmarshal(retBody, &thisRawList)
+	if rawdecErr != nil {
+		log.Error("Failed to decode body (raw): ", rawdecErr)
+		return &APISpecs
+	}
 
-	for i, thisAppConfig := range APIDefinitions {
-		thisAppConfig.DecodeFromDB()
-		thisAppConfig.RawData = StringDefs[i] // Lets keep a copy for plugable modules
+	// Extract tagged entries only
+	APIDefinitions := make([]tykcommon.APIDefinition, 0)
+
+	if config.DBAppConfOptions.NodeIsSegmented {
+		APIDefinitions = make([]tykcommon.APIDefinition, 0)
+		tagList := make(map[string]bool)
+		toLoad := make(map[string]tykcommon.APIDefinition)
+
+		for _, mt := range config.DBAppConfOptions.Tags {
+			tagList[mt] = true
+		}
+
+		for index, apiEntry := range thisList.Message {
+			for _, t := range apiEntry.ApiDefinition.Tags {
+				_, ok := tagList[t]
+				if ok {
+					apiEntry.ApiDefinition.RawData = thisRawList["Message"].([]interface{})[index].(map[string]interface{})["api_definition"].(map[string]interface{})
+					toLoad[apiEntry.ApiDefinition.APIID] = apiEntry.ApiDefinition
+				}
+
+			}
+		}
+
+		for _, apiDef := range toLoad {
+			APIDefinitions = append(APIDefinitions, apiDef)
+		}
+	} else {
+		for index, apiEntry := range thisList.Message {
+			apiEntry.ApiDefinition.RawData = thisRawList["Message"].([]interface{})[index].(map[string]interface{})["api_definition"].(map[string]interface{})
+			APIDefinitions = append(APIDefinitions, apiEntry.ApiDefinition)
+		}
+	}
 
+	// Process
+	for _, thisAppConfig := range APIDefinitions {
 		newAppSpec := a.MakeSpec(thisAppConfig)
 		APISpecs = append(APISpecs, &newAppSpec)
 	}
 
-	a.Disconnect()
+	// Set the nonce
+	ServiceNonce = thisList.Nonce
 
 	return &APISpecs
 }

auth_manager.go

@@ -138,7 +138,7 @@ func (b DefaultSessionManager) GetSessionDetail(keyName string) (SessionState, b
 			"prefix":      "auth-mgr",
 			"inbound-key": keyName,
 			"err":         err,
-		}).Info("Could not get session detail, key not found")
+		}).Debug("Could not get session detail, key not found")
 		return thisSession, false
 	}
 

config.go

@@ -17,13 +17,16 @@ type Config struct {
 	TykJSPath      string `json:"tyk_js_path"`
 	MiddlewarePath string `json:"middleware_path"`
 	Policies       struct {
-		PolicySource     string `json:"policy_source"`
-		PolicyRecordName string `json:"policy_record_name"`
+		PolicySource           string `json:"policy_source"`
+		PolicyConnectionString string `json:"policy_connection_string"`
+		PolicyRecordName       string `json:"policy_record_name"`
 	} `json:"policies"`
 	UseDBAppConfigs  bool `json:"use_db_app_configs"`
 	DBAppConfOptions struct {
-		NodeIsSegmented bool     `json:"node_is_segmented"`
-		Tags            []string `json:"tags"`
+		ConnectionString string   `json:"connection_string"`
+		NodeID           string   `json:"node_id"`
+		NodeIsSegmented  bool     `json:"node_is_segmented"`
+		Tags             []string `json:"tags"`
 	} `json:"db_app_conf_options"`
 	AppPath string `json:"app_path"`
 	Storage struct {