Add global http timeout for manager (kedacore#1548)

* Add global http timeout for manager This was already done in adapter but was missed in manager. Since manager calls scaler.IsActive() with 0 timeout, it is erring out with "context deadline exceeded" errors when connection latency is high (like prometheus scaler trying to contact a prometheus server running in different region) Signed-off-by: Suresh Kumar Ponnusamy <[email protected]> * Update changelog Signed-off-by: Suresh Kumar Ponnusamy <[email protected]>
cuong-nd · Jan 29, 2021 · 8d70252 · 8d70252
1 parent f119845
commit 8d70252
Show file tree

Hide file tree

Showing 4 changed files with 32 additions and 15 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -23,7 +23,7 @@
 
 ### Improvements
 
-- TODO ([#XXX](https://github.com/kedacore/keda/issues/XXX))
+- Add `KEDA_HTTP_DEFAULT_TIMEOUT` support in operator ([#1548](https://github.com/kedacore/keda/issues/1548))
 
 ### Breaking Changes
 

diff --git a/controllers/scaledjob_controller.go b/controllers/scaledjob_controller.go
@@ -28,13 +28,13 @@ type ScaledJobReconciler struct {
 	client.Client
 	Log               logr.Logger
 	Scheme            *runtime.Scheme
+	GlobalHTTPTimeout time.Duration
 	scaleHandler      scaling.ScaleHandler
-	globalHTTPTimeout time.Duration
 }
 
 // SetupWithManager initializes the ScaledJobReconciler instance and starts a new controller managed by the passed Manager instance.
 func (r *ScaledJobReconciler) SetupWithManager(mgr ctrl.Manager) error {
-	r.scaleHandler = scaling.NewScaleHandler(mgr.GetClient(), nil, mgr.GetScheme(), r.globalHTTPTimeout)
+	r.scaleHandler = scaling.NewScaleHandler(mgr.GetClient(), nil, mgr.GetScheme(), r.GlobalHTTPTimeout)
 
 	return ctrl.NewControllerManagedBy(mgr).
 		// Ignore updates to ScaledJob Status (in this case metadata.Generation does not change)

diff --git a/controllers/scaledobject_controller.go b/controllers/scaledobject_controller.go
@@ -42,17 +42,16 @@ import (
 
 // ScaledObjectReconciler reconciles a ScaledObject object
 type ScaledObjectReconciler struct {
-	Log    logr.Logger
-	Client client.Client
-	Scheme *runtime.Scheme
+	Log               logr.Logger
+	Client            client.Client
+	Scheme            *runtime.Scheme
+	GlobalHTTPTimeout time.Duration
 
 	scaleClient              *scale.ScalesGetter
 	restMapper               meta.RESTMapper
 	scaledObjectsGenerations *sync.Map
 	scaleHandler             scaling.ScaleHandler
 	kubeVersion              kedautil.K8sVersion
-
-	globalHTTPTimeout time.Duration
 }
 
 // A cache mapping "resource.group" to true or false if we know if this resource is scalable.
@@ -92,7 +91,7 @@ func (r *ScaledObjectReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	// Init the rest of ScaledObjectReconciler
 	r.restMapper = mgr.GetRESTMapper()
 	r.scaledObjectsGenerations = &sync.Map{}
-	r.scaleHandler = scaling.NewScaleHandler(mgr.GetClient(), r.scaleClient, mgr.GetScheme(), r.globalHTTPTimeout)
+	r.scaleHandler = scaling.NewScaleHandler(mgr.GetClient(), r.scaleClient, mgr.GetScheme(), r.GlobalHTTPTimeout)
 
 	// Start controller
 	return ctrl.NewControllerManagedBy(mgr).

diff --git a/main.go b/main.go
@@ -21,6 +21,8 @@ import (
 	"fmt"
 	"os"
 	"runtime"
+	"strconv"
+	"time"
 
 	apimachineryruntime "k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -108,18 +110,34 @@ func main() {
 		os.Exit(1)
 	}
 
+	globalHTTPTimeoutStr := os.Getenv("KEDA_HTTP_DEFAULT_TIMEOUT")
+	if globalHTTPTimeoutStr == "" {
+		// default to 3 seconds if they don't pass the env var
+		globalHTTPTimeoutStr = "3000"
+	}
+
+	globalHTTPTimeoutMS, err := strconv.Atoi(globalHTTPTimeoutStr)
+	if err != nil {
+		setupLog.Error(err, "Invalid KEDA_HTTP_DEFAULT_TIMEOUT")
+		return
+	}
+
+	globalHTTPTimeout := time.Duration(globalHTTPTimeoutMS) * time.Millisecond
+
 	if err = (&controllers.ScaledObjectReconciler{
-		Client: mgr.GetClient(),
-		Log:    ctrl.Log.WithName("controllers").WithName("ScaledObject"),
-		Scheme: mgr.GetScheme(),
+		Client:            mgr.GetClient(),
+		Log:               ctrl.Log.WithName("controllers").WithName("ScaledObject"),
+		Scheme:            mgr.GetScheme(),
+		GlobalHTTPTimeout: globalHTTPTimeout,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "ScaledObject")
 		os.Exit(1)
 	}
 	if err = (&controllers.ScaledJobReconciler{
-		Client: mgr.GetClient(),
-		Log:    ctrl.Log.WithName("controllers").WithName("ScaledJob"),
-		Scheme: mgr.GetScheme(),
+		Client:            mgr.GetClient(),
+		Log:               ctrl.Log.WithName("controllers").WithName("ScaledJob"),
+		Scheme:            mgr.GetScheme(),
+		GlobalHTTPTimeout: globalHTTPTimeout,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "ScaledJob")
 		os.Exit(1)