Skip to content

Commit

Permalink
Merge pull request TykTechnologies#1592 from TykTechnologies/config-g…
Browse files Browse the repository at this point in the history 
…lobal-races-fix

acces to config.Global is protected with atomic.Value
  • Loading branch information
@dencoded
dencoded authored May 4, 2018
2 parents abb1b35 + cfa69c4 commit aed2b9a
Show file tree
Hide file tree
Showing 65 changed files with 760 additions and 598 deletions.
28 changes: 13 additions & 15 deletions analytics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ type GeoData struct {
const analyticsKeyName = "tyk-system-analytics"

func (a *AnalyticsRecord) GetGeo(ipStr string) {
if !config.Global.AnalyticsConfig.EnableGeoIP {
if !config.Global().AnalyticsConfig.EnableGeoIP {
return
}
// Not great, tightly coupled
Expand Down Expand Up @@ -108,7 +108,7 @@ func initNormalisationPatterns() (pats config.NormaliseURLPatterns) {
pats.UUIDs = regexp.MustCompile(`[0-9a-fA-F]{8}(-)?[0-9a-fA-F]{4}(-)?[0-9a-fA-F]{4}(-)?[0-9a-fA-F]{4}(-)?[0-9a-fA-F]{12}`)
pats.IDs = regexp.MustCompile(`\/(\d+)`)

for _, pattern := range config.Global.AnalyticsConfig.NormaliseUrls.Custom {
for _, pattern := range config.Global().AnalyticsConfig.NormaliseUrls.Custom {
if patRe, err := regexp.Compile(pattern); err != nil {
log.Error("failed to compile custom pattern: ", err)
} else {
Expand All @@ -119,13 +119,13 @@ func initNormalisationPatterns() (pats config.NormaliseURLPatterns) {
}

func (a *AnalyticsRecord) NormalisePath() {
if config.Global.AnalyticsConfig.NormaliseUrls.NormaliseUUIDs {
a.Path = config.Global.AnalyticsConfig.NormaliseUrls.CompiledPatternSet.UUIDs.ReplaceAllString(a.Path, "{uuid}")
if config.Global().AnalyticsConfig.NormaliseUrls.NormaliseUUIDs {
a.Path = config.Global().AnalyticsConfig.NormaliseUrls.CompiledPatternSet.UUIDs.ReplaceAllString(a.Path, "{uuid}")
}
if config.Global.AnalyticsConfig.NormaliseUrls.NormaliseNumbers {
a.Path = config.Global.AnalyticsConfig.NormaliseUrls.CompiledPatternSet.IDs.ReplaceAllString(a.Path, "/{id}")
if config.Global().AnalyticsConfig.NormaliseUrls.NormaliseNumbers {
a.Path = config.Global().AnalyticsConfig.NormaliseUrls.CompiledPatternSet.IDs.ReplaceAllString(a.Path, "/{id}")
}
for _, r := range config.Global.AnalyticsConfig.NormaliseUrls.CompiledPatternSet.Custom {
for _, r := range config.Global().AnalyticsConfig.NormaliseUrls.CompiledPatternSet.Custom {
a.Path = r.ReplaceAllString(a.Path, "{var}")
}
}
Expand Down Expand Up @@ -154,8 +154,8 @@ type RedisAnalyticsHandler struct {

func (r *RedisAnalyticsHandler) Init() {
var err error
if config.Global.AnalyticsConfig.EnableGeoIP {
db, err := maxminddb.Open(config.Global.AnalyticsConfig.GeoIPDBLocation)
if config.Global().AnalyticsConfig.EnableGeoIP {
db, err := maxminddb.Open(config.Global().AnalyticsConfig.GeoIPDBLocation)
if err != nil {
log.Error("Failed to init GeoIP Database: ", err)
} else {
Expand All @@ -165,7 +165,7 @@ func (r *RedisAnalyticsHandler) Init() {

analytics.Store.Connect()

ps := config.Global.AnalyticsConfig.PoolSize
ps := config.Global().AnalyticsConfig.PoolSize
if ps == 0 {
ps = 50
}
Expand All @@ -182,17 +182,15 @@ func (r *RedisAnalyticsHandler) RecordHit(record AnalyticsRecord) error {
// If we are obfuscating API Keys, store the hashed representation (config check handled in hashing function)
record.APIKey = storage.HashKey(record.APIKey)

configMu.Lock()
if config.Global.SlaveOptions.UseRPC {
if config.Global().SlaveOptions.UseRPC {
// Extend tag list to include this data so wecan segment by node if necessary
record.Tags = append(record.Tags, "tyk-hybrid-rpc")
}

if config.Global.DBAppConfOptions.NodeIsSegmented {
if config.Global().DBAppConfOptions.NodeIsSegmented {
// Extend tag list to include this data so wecan segment by node if necessary
record.Tags = append(record.Tags, config.Global.DBAppConfOptions.Tags...)
record.Tags = append(record.Tags, config.Global().DBAppConfOptions.Tags...)
}
configMu.Unlock()

// Lets add some metadata
if record.APIKey != "" {
Expand Down
16 changes: 10 additions & 6 deletions analytics_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,13 @@ func TestGeoIPLookup(t *testing.T) {
}

func TestURLReplacer(t *testing.T) {
config.Global.AnalyticsConfig.NormaliseUrls.Enabled = true
config.Global.AnalyticsConfig.NormaliseUrls.NormaliseUUIDs = true
config.Global.AnalyticsConfig.NormaliseUrls.NormaliseNumbers = true
config.Global.AnalyticsConfig.NormaliseUrls.Custom = []string{"ihatethisstring"}
defer resetTestConfig()
globalConf := config.Global()
globalConf.AnalyticsConfig.NormaliseUrls.Enabled = true
globalConf.AnalyticsConfig.NormaliseUrls.NormaliseUUIDs = true
globalConf.AnalyticsConfig.NormaliseUrls.NormaliseNumbers = true
globalConf.AnalyticsConfig.NormaliseUrls.Custom = []string{"ihatethisstring"}
config.SetGlobal(globalConf)

recordUUID1 := AnalyticsRecord{Path: "/15873a748894492162c402d67e92283b/search"}
recordUUID2 := AnalyticsRecord{Path: "/CA761232-ED42-11CE-BACD-00AA0057B223/search"}
Expand All @@ -39,7 +42,8 @@ func TestURLReplacer(t *testing.T) {
recordID1 := AnalyticsRecord{Path: "/widgets/123456/getParams"}
recordCust := AnalyticsRecord{Path: "/widgets/123456/getParams/ihatethisstring"}

config.Global.AnalyticsConfig.NormaliseUrls.CompiledPatternSet = initNormalisationPatterns()
globalConf.AnalyticsConfig.NormaliseUrls.CompiledPatternSet = initNormalisationPatterns()
config.SetGlobal(globalConf)

recordUUID1.NormalisePath()
recordUUID2.NormalisePath()
Expand All @@ -51,7 +55,7 @@ func TestURLReplacer(t *testing.T) {
if recordUUID1.Path != "/{uuid}/search" {
t.Error("Path not altered, is:")
t.Error(recordUUID1.Path)
t.Error(config.Global.AnalyticsConfig.NormaliseUrls)
t.Error(config.Global().AnalyticsConfig.NormaliseUrls)
}

if recordUUID2.Path != "/{uuid}/search" {
Expand Down
24 changes: 12 additions & 12 deletions api.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ func doAddOrUpdate(keyName string, newSession *user.SessionState, dontReset bool
}
} else {
// nothing defined, add key to ALL
if !config.Global.AllowMasterKeys {
if !config.Global().AllowMasterKeys {
log.Error("Master keys disallowed in configuration, key not added.")
return errors.New("Master keys not allowed")
}
Expand Down Expand Up @@ -287,15 +287,15 @@ func handleAddOrUpdate(keyName string, r *http.Request) (interface{}, int) {
}

// add key hash for newly created key
if config.Global.HashKeys && r.Method == http.MethodPost {
if config.Global().HashKeys && r.Method == http.MethodPost {
response.KeyHash = storage.HashKey(keyName)
}

return response, 200
}

func handleGetDetail(sessionKey, apiID string, byHash bool) (interface{}, int) {
if byHash && !config.Global.HashKeys {
if byHash && !config.Global().HashKeys {
return apiError("Key requested by hash but key hashing is not enabled"), 400
}

Expand Down Expand Up @@ -464,7 +464,7 @@ func handleGetAPI(apiID string) (interface{}, int) {
}

func handleAddOrUpdateApi(apiID string, r *http.Request) (interface{}, int) {
if config.Global.UseDBAppConfigs {
if config.Global().UseDBAppConfigs {
log.Error("Rejected new API Definition due to UseDBAppConfigs = true")
return apiError("Due to enabled use_db_app_configs, please use the Dashboard API"), 500
}
Expand All @@ -481,7 +481,7 @@ func handleAddOrUpdateApi(apiID string, r *http.Request) (interface{}, int) {
}

// Create a filename
defFilePath := filepath.Join(config.Global.AppPath, newDef.APIID+".json")
defFilePath := filepath.Join(config.Global().AppPath, newDef.APIID+".json")

// If it exists, delete it
if _, err := os.Stat(defFilePath); err == nil {
Expand Down Expand Up @@ -517,7 +517,7 @@ func handleAddOrUpdateApi(apiID string, r *http.Request) (interface{}, int) {

func handleDeleteAPI(apiID string) (interface{}, int) {
// Generate a filename
defFilePath := filepath.Join(config.Global.AppPath, apiID+".json")
defFilePath := filepath.Join(config.Global().AppPath, apiID+".json")

// If it exists, delete it
if _, err := os.Stat(defFilePath); err != nil {
Expand Down Expand Up @@ -591,9 +591,9 @@ func keyHandler(w http.ResponseWriter, r *http.Request) {
obj, code = handleGetDetail(keyName, apiID, isHashed)
} else {
// Return list of keys
if config.Global.HashKeys {
if config.Global().HashKeys {
// get all keys is disabled by default
if !config.Global.EnableHashedKeysListing {
if !config.Global().EnableHashedKeysListing {
doJSONWrite(
w,
http.StatusNotFound,
Expand Down Expand Up @@ -731,7 +731,7 @@ func handleOrgAddOrUpdate(keyName string, r *http.Request) (interface{}, int) {

if spec == nil {
log.Warning("Couldn't find org session store in active API list")
if config.Global.SupressDefaultOrgStore {
if config.Global().SupressDefaultOrgStore {
return apiError("No such organisation found in Active API list"), 404
}
sessionManager = &DefaultOrgStore
Expand Down Expand Up @@ -934,7 +934,7 @@ func createKeyHandler(w http.ResponseWriter, r *http.Request) {
}
}
} else {
if config.Global.AllowMasterKeys {
if config.Global().AllowMasterKeys {
// nothing defined, add key to ALL
log.WithFields(logrus.Fields{
"prefix": "api",
Expand Down Expand Up @@ -988,7 +988,7 @@ func createKeyHandler(w http.ResponseWriter, r *http.Request) {
}

// add key hash to reply
if config.Global.HashKeys {
if config.Global().HashKeys {
obj.KeyHash = storage.HashKey(newKey)
}

Expand Down Expand Up @@ -1407,7 +1407,7 @@ func getOauthClients(apiID string) (interface{}, int) {
}

func healthCheckhandler(w http.ResponseWriter, r *http.Request) {
if !config.Global.HealthCheck.EnableHealthChecks {
if !config.Global().HealthCheck.EnableHealthChecks {
doJSONWrite(w, 400, apiError("Health checks are not enabled for this node"))
return
}
Expand Down
23 changes: 13 additions & 10 deletions api_definition.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,7 @@ type APISpec struct {
ServiceRefreshInProgress bool
HTTPTransport http.RoundTripper
HTTPTransportCreated time.Time
GlobalConfig config.Config
}

// APIDefinitionLoader will load an Api definition from a storage
Expand All @@ -176,8 +177,10 @@ func (a APIDefinitionLoader) MakeSpec(def *apidef.APIDefinition) *APISpec {
spec.SessionManager = &DefaultSessionManager{}
spec.OrgSessionManager = &DefaultSessionManager{}

spec.GlobalConfig = config.Global()

// Create and init the virtual Machine
if config.Global.EnableJSVM {
if config.Global().EnableJSVM {
spec.JSVM.Init(spec)
}

Expand Down Expand Up @@ -269,11 +272,11 @@ func (a APIDefinitionLoader) FromDashboardService(endpoint, secret string) []*AP
// Extract tagged entries only
apiDefs := make([]*apidef.APIDefinition, 0)

if config.Global.DBAppConfOptions.NodeIsSegmented {
tagList := make(map[string]bool, len(config.Global.DBAppConfOptions.Tags))
if config.Global().DBAppConfOptions.NodeIsSegmented {
tagList := make(map[string]bool, len(config.Global().DBAppConfOptions.Tags))
toLoad := make(map[string]*apidef.APIDefinition)

for _, mt := range config.Global.DBAppConfOptions.Tags {
for _, mt := range config.Global().DBAppConfOptions.Tags {
tagList[mt] = true
}

Expand Down Expand Up @@ -314,16 +317,16 @@ func (a APIDefinitionLoader) FromRPC(orgId string) []*APISpec {
return LoadDefinitionsFromRPCBackup()
}

store := RPCStorageHandler{UserKey: config.Global.SlaveOptions.APIKey, Address: config.Global.SlaveOptions.ConnectionString}
store := RPCStorageHandler{UserKey: config.Global().SlaveOptions.APIKey, Address: config.Global().SlaveOptions.ConnectionString}
if !store.Connect() {
return nil
}

// enable segments
var tags []string
if config.Global.DBAppConfOptions.NodeIsSegmented {
log.Info("Segmented node, loading: ", config.Global.DBAppConfOptions.Tags)
tags = config.Global.DBAppConfOptions.Tags
if config.Global().DBAppConfOptions.NodeIsSegmented {
log.Info("Segmented node, loading: ", config.Global().DBAppConfOptions.Tags)
tags = config.Global().DBAppConfOptions.Tags
}

apiCollection := store.GetApiDefinitions(orgId, tags)
Expand All @@ -349,7 +352,7 @@ func (a APIDefinitionLoader) processRPCDefinitions(apiCollection string) []*APIS
for _, def := range apiDefs {
def.DecodeFromDB()

if config.Global.SlaveOptions.BindToSlugsInsteadOfListenPaths {
if config.Global().SlaveOptions.BindToSlugsInsteadOfListenPaths {
newListenPath := "/" + def.Slug //+ "/"
log.Warning("Binding to ",
newListenPath,
Expand Down Expand Up @@ -684,7 +687,7 @@ func (a APIDefinitionLoader) compileURLRewritesPathSpec(paths []apidef.URLRewrit
}

func (a APIDefinitionLoader) compileVirtualPathspathSpec(paths []apidef.VirtualMeta, stat URLStatus, apiSpec *APISpec) []URLSpec {
if !config.Global.EnableJSVM {
if !config.Global().EnableJSVM {
return nil
}

Expand Down
16 changes: 7 additions & 9 deletions api_definition_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -234,15 +234,13 @@ func TestSyncAPISpecsDashboardSuccess(t *testing.T) {
apisByID = make(map[string]*APISpec)
apisMu.Unlock()

config.Global.UseDBAppConfigs = true
config.Global.AllowInsecureConfigs = true
config.Global.DBAppConfOptions.ConnectionString = ts.URL

defer func() {
config.Global.UseDBAppConfigs = false
config.Global.AllowInsecureConfigs = false
config.Global.DBAppConfOptions.ConnectionString = ""
}()
globalConf := config.Global()
globalConf.UseDBAppConfigs = true
globalConf.AllowInsecureConfigs = true
globalConf.DBAppConfOptions.ConnectionString = ts.URL
config.SetGlobal(globalConf)

defer resetTestConfig()

var wg sync.WaitGroup
wg.Add(1)
Expand Down
15 changes: 6 additions & 9 deletions api_healthcheck.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ type DefaultHealthChecker struct {
var healthWarn sync.Once

func (h *DefaultHealthChecker) Init(storeType storage.Handler) {
if config.Global.HealthCheck.EnableHealthChecks {
if config.Global().HealthCheck.EnableHealthChecks {
log.Debug("Health Checker initialised.")
healthWarn.Do(func() {
log.Warning("The Health Checker is deprecated and we do no longer recommend its use.")
Expand All @@ -60,12 +60,9 @@ func (h *DefaultHealthChecker) CreateKeyName(subKey HealthPrefix) string {

// reportHealthValue is a shortcut we can use throughout the app to push a health check value
func reportHealthValue(spec *APISpec, counter HealthPrefix, value string) {
configMu.Lock()
if !config.Global.HealthCheck.EnableHealthChecks {
configMu.Unlock()
if !spec.GlobalConfig.HealthCheck.EnableHealthChecks {
return
}
configMu.Unlock()

spec.Health.StoreCounterVal(counter, value)
}
Expand All @@ -81,16 +78,16 @@ func (h *DefaultHealthChecker) StoreCounterVal(counterType HealthPrefix, value s
value = now_string + "." + value
log.Debug("Set value to: ", value)
}
go h.storage.SetRollingWindow(searchStr, config.Global.HealthCheck.HealthCheckValueTimeout, value, false)
go h.storage.SetRollingWindow(searchStr, config.Global().HealthCheck.HealthCheckValueTimeout, value, false)
}

func (h *DefaultHealthChecker) getAvgCount(prefix HealthPrefix) float64 {
searchStr := h.CreateKeyName(prefix)
log.Debug("Searching for: ", searchStr)

count, _ := h.storage.SetRollingWindow(searchStr, config.Global.HealthCheck.HealthCheckValueTimeout, "-1", false)
count, _ := h.storage.SetRollingWindow(searchStr, config.Global().HealthCheck.HealthCheckValueTimeout, "-1", false)
log.Debug("Count is: ", count)
divisor := float64(config.Global.HealthCheck.HealthCheckValueTimeout)
divisor := float64(config.Global().HealthCheck.HealthCheckValueTimeout)
if divisor == 0 {
log.Warning("The Health Check sample timeout is set to 0, samples will never be deleted!!!")
divisor = 60.0
Expand Down Expand Up @@ -118,7 +115,7 @@ func (h *DefaultHealthChecker) ApiHealthValues() (HealthCheckValues, error) {
// Get the micro latency graph, an average upstream latency
searchStr := h.APIID + "." + string(RequestLog)
log.Debug("Searching KV for: ", searchStr)
_, vals := h.storage.SetRollingWindow(searchStr, config.Global.HealthCheck.HealthCheckValueTimeout, "-1", false)
_, vals := h.storage.SetRollingWindow(searchStr, config.Global().HealthCheck.HealthCheckValueTimeout, "-1", false)
log.Debug("Found: ", vals)
if len(vals) == 0 {
return values, nil
Expand Down
Loading

0 comments on commit aed2b9a

Please sign in to comment.