Automatically load default plugins if unconfigured (Netflix#265)

consoleme/__main__.py

@@ -17,7 +17,7 @@
 
 logging.basicConfig(level=logging.DEBUG, format=config.get("logging.format"))
 logging.getLogger("urllib3.connectionpool").setLevel(logging.CRITICAL)
-stats = get_plugin_by_name(config.get("plugins.metrics"))()
+stats = get_plugin_by_name(config.get("plugins.metrics", "default_metrics"))()
 log = config.get_logger()
 
 

consoleme/celery/celery_tasks.py

@@ -105,12 +105,18 @@ def on_configure(self) -> None:
 
 log = config.get_logger()
 red = async_to_sync(RedisHandler().redis)()
-aws = get_plugin_by_name(config.get("plugins.aws"))
-auth = get_plugin_by_name(config.get("plugins.auth"))()
-group_mapping = get_plugin_by_name(config.get("plugins.group_mapping"))()
-internal_celery_tasks = get_plugin_by_name(config.get("plugins.internal_celery_tasks"))
-stats = get_plugin_by_name(config.get("plugins.metrics"))()
-internal_policies = get_plugin_by_name(config.get("plugins.internal_policies"))()
+aws = get_plugin_by_name(config.get("plugins.aws", "default_aws"))
+auth = get_plugin_by_name(config.get("plugins.auth", "default_auth"))()
+group_mapping = get_plugin_by_name(
+ config.get("plugins.group_mapping", "default_group_mapping")
+)()
+internal_celery_tasks = get_plugin_by_name(
+ config.get("plugins.internal_celery_tasks", "default_celery_tasks")
+)
+stats = get_plugin_by_name(config.get("plugins.metrics", "default_metrics"))()
+internal_policies = get_plugin_by_name(
+ config.get("plugins.internal_policies", "default_policies")
+)()
 REDIS_IAM_COUNT = 1000
 
 

consoleme/exceptions/exceptions.py

@@ -5,7 +5,7 @@
 from consoleme.lib.plugins import get_plugin_by_name
 
 log = config.get_logger("consoleme")
-stats = get_plugin_by_name(config.get("plugins.metrics"))()
+stats = get_plugin_by_name(config.get("plugins.metrics", "default_metrics"))()
 
 
 class BaseException(Exception):

consoleme/handlers/base.py

@@ -34,9 +34,11 @@
 from consoleme.lib.tracing import ConsoleMeTracer
 
 log = config.get_logger()
-stats = get_plugin_by_name(config.get("plugins.metrics"))()
-auth = get_plugin_by_name(config.get("plugins.auth"))()
-group_mapping = get_plugin_by_name(config.get("plugins.group_mapping"))()
+stats = get_plugin_by_name(config.get("plugins.metrics", "default_metrics"))()
+auth = get_plugin_by_name(config.get("plugins.auth", "default_auth"))()
+group_mapping = get_plugin_by_name(
+ config.get("plugins.group_mapping", "default_group_mapping")
+)()
 
 
 class BaseJSONHandler(tornado.web.RequestHandler):

consoleme/handlers/v1/credentials.py

@@ -12,13 +12,15 @@
 from consoleme.lib.duo import duo_mfa_user
 from consoleme.lib.plugins import get_plugin_by_name
 
-stats = get_plugin_by_name(config.get("plugins.metrics"))()
+stats = get_plugin_by_name(config.get("plugins.metrics", "default_metrics"))()
 log = config.get_logger()
 crypto = Crypto()
-aws = get_plugin_by_name(config.get("plugins.aws"))()
+aws = get_plugin_by_name(config.get("plugins.aws", "default_aws"))()
 internal_config = config.config_plugin
-auth = get_plugin_by_name(config.get("plugins.auth"))()
-group_mapping = get_plugin_by_name(config.get("plugins.group_mapping"))()
+auth = get_plugin_by_name(config.get("plugins.auth", "default_auth"))()
+group_mapping = get_plugin_by_name(
+ config.get("plugins.group_mapping", "default_group_mapping")
+)()
 
 
 class CredentialsSchema(Schema):

consoleme/handlers/v1/headers.py

@@ -4,7 +4,7 @@
 from consoleme.handlers.base import BaseHandler, BaseMtlsHandler
 from consoleme.lib.plugins import get_plugin_by_name
 
-stats = get_plugin_by_name(config.get("plugins.metrics"))()
+stats = get_plugin_by_name(config.get("plugins.metrics", "default_metrics"))()
 log = config.get_logger()
 
 

consoleme/handlers/v1/policies.py

@@ -12,9 +12,9 @@
 from consoleme.lib.redis import redis_get, redis_hgetall
 
 log = config.get_logger()
-stats = get_plugin_by_name(config.get("plugins.metrics"))()
-aws = get_plugin_by_name(config.get("plugins.aws"))()
-auth = get_plugin_by_name(config.get("plugins.auth"))()
+stats = get_plugin_by_name(config.get("plugins.metrics", "default_metrics"))()
+aws = get_plugin_by_name(config.get("plugins.aws", "default_aws"))()
+auth = get_plugin_by_name(config.get("plugins.auth", "default_auth"))()
 
 
 class AutocompleteHandler(BaseAPIV1Handler):

consoleme/handlers/v1/roles.py

@@ -5,10 +5,10 @@
 from consoleme.lib.crypto import Crypto
 from consoleme.lib.plugins import get_plugin_by_name
 
-stats = get_plugin_by_name(config.get("plugins.metrics"))()
+stats = get_plugin_by_name(config.get("plugins.metrics", "default_metrics"))()
 log = config.get_logger()
 crypto = Crypto()
-auth = get_plugin_by_name(config.get("plugins.auth"))()
+auth = get_plugin_by_name(config.get("plugins.auth", "default_auth"))()
 
 
 class GetRolesHandler(BaseMtlsHandler):

consoleme/handlers/v1/saml.py

@@ -13,10 +13,10 @@
 if config.get("auth.get_user_by_saml"):
  from onelogin.saml2.utils import OneLogin_Saml2_Utils
 
-stats = get_plugin_by_name(config.get("plugins.metrics"))()
+stats = get_plugin_by_name(config.get("plugins.metrics", "default_metrics"))()
 log = config.get_logger()
 crypto = Crypto()
-auth = get_plugin_by_name(config.get("plugins.auth"))()
+auth = get_plugin_by_name(config.get("plugins.auth", "default_auth"))()
 
 
 class SamlHandler(BaseHandler):

consoleme/handlers/v2/dynamic_config.py

@@ -15,8 +15,8 @@
 
 ddb = UserDynamoHandler()
 log = config.get_logger()
-stats = get_plugin_by_name(config.get("plugins.metrics"))()
-aws = get_plugin_by_name(config.get("plugins.aws"))()
+stats = get_plugin_by_name(config.get("plugins.metrics", "default_metrics"))()
+aws = get_plugin_by_name(config.get("plugins.aws", "default_aws"))()
 
 
 class DynamicConfigApiHandler(BaseHandler):

consoleme/handlers/v2/errors.py

@@ -2,7 +2,7 @@
 from consoleme.handlers.base import BaseAPIV2Handler
 from consoleme.lib.plugins import get_plugin_by_name
 
-stats = get_plugin_by_name(config.get("plugins.metrics"))()
+stats = get_plugin_by_name(config.get("plugins.metrics", "default_metrics"))()
 
 log = config.get_logger()
 

consoleme/handlers/v2/generate_changes.py

@@ -11,7 +11,7 @@
 from consoleme.models import ChangeGeneratorModelArray
 
 log = config.get_logger()
-stats = get_plugin_by_name(config.get("plugins.metrics"))()
+stats = get_plugin_by_name(config.get("plugins.metrics", "default_metrics"))()
 
 
 class GenerateChangesHandler(BaseAPIV2Handler):

consoleme/handlers/v2/generate_policy.py

@@ -2,7 +2,7 @@
 from consoleme.handlers.base import BaseAPIV2Handler
 from consoleme.lib.plugins import get_plugin_by_name
 
-stats = get_plugin_by_name(config.get("plugins.metrics"))()
+stats = get_plugin_by_name(config.get("plugins.metrics", "default_metrics"))()
 log = config.get_logger()
 
 BASE_INLINE_POLICY = {"Statement": [{"Action": [], "Effect": "Allow", "Resource": []}]}

consoleme/handlers/v2/index.py

@@ -7,9 +7,9 @@
 from consoleme.lib.plugins import get_plugin_by_name
 
 log = config.get_logger()
-aws = get_plugin_by_name(config.get("plugins.aws"))()
-auth = get_plugin_by_name(config.get("plugins.auth"))()
-stats = get_plugin_by_name(config.get("plugins.metrics"))()
+aws = get_plugin_by_name(config.get("plugins.aws", "default_aws"))()
+auth = get_plugin_by_name(config.get("plugins.auth", "default_auth"))()
+stats = get_plugin_by_name(config.get("plugins.metrics", "default_metrics"))()
 
 
 # TODO, move followings to util file

consoleme/handlers/v2/policies.py

@@ -10,7 +10,7 @@
 from consoleme.lib.policies import get_url_for_resource
 from consoleme.lib.timeout import Timeout
 
-stats = get_plugin_by_name(config.get("plugins.metrics"))()
+stats = get_plugin_by_name(config.get("plugins.metrics", "default_metrics"))()
 log = config.get_logger()
 
 

consoleme/handlers/v2/requests.py

@@ -47,9 +47,9 @@
  RequestStatus,
 )
 
-stats = get_plugin_by_name(config.get("plugins.metrics"))()
+stats = get_plugin_by_name(config.get("plugins.metrics", "default_metrics"))()
 log = config.get_logger()
-aws = get_plugin_by_name(config.get("plugins.aws"))()
+aws = get_plugin_by_name(config.get("plugins.aws", "default_aws"))()
 
 
 class RequestHandler(BaseAPIV2Handler):

consoleme/handlers/v2/resources.py

@@ -12,11 +12,15 @@
 from consoleme.lib.plugins import get_plugin_by_name
 
 log = config.get_logger()
-stats = get_plugin_by_name(config.get("plugins.metrics"))()
-aws = get_plugin_by_name(config.get("plugins.aws"))()
-group_mapping = get_plugin_by_name(config.get("plugins.group_mapping"))()
-auth = get_plugin_by_name(config.get("plugins.auth"))()
-internal_policies = get_plugin_by_name(config.get("plugins.internal_policies"))()
+stats = get_plugin_by_name(config.get("plugins.metrics", "default_metrics"))()
+aws = get_plugin_by_name(config.get("plugins.aws", "default_aws"))()
+group_mapping = get_plugin_by_name(
+ config.get("plugins.group_mapping", "default_group_mapping")
+)()
+auth = get_plugin_by_name(config.get("plugins.auth", "default_auth"))()
+internal_policies = get_plugin_by_name(
+ config.get("plugins.internal_policies", "default_policies")
+)()
 
 
 class ResourceDetailHandler(BaseHandler):

consoleme/handlers/v2/roles.py

@@ -16,13 +16,17 @@
 from consoleme.lib.v2.roles import get_role_details
 from consoleme.models import CloneRoleRequestModel, RoleCreationRequestModel
 
-stats = get_plugin_by_name(config.get("plugins.metrics"))()
+stats = get_plugin_by_name(config.get("plugins.metrics", "default_metrics"))()
 log = config.get_logger()
 crypto = Crypto()
-auth = get_plugin_by_name(config.get("plugins.auth"))()
-aws = get_plugin_by_name(config.get("plugins.aws"))()
-group_mapping = get_plugin_by_name(config.get("plugins.group_mapping"))()
-internal_policies = get_plugin_by_name(config.get("plugins.internal_policies"))()
+auth = get_plugin_by_name(config.get("plugins.auth", "default_auth"))()
+aws = get_plugin_by_name(config.get("plugins.aws", "default_aws"))()
+group_mapping = get_plugin_by_name(
+ config.get("plugins.group_mapping", "default_group_mapping")
+)()
+internal_policies = get_plugin_by_name(
+ config.get("plugins.internal_policies", "default_policies")
+)()
 
 
 class RoleConsoleLoginHandler(BaseAPIV2Handler):

consoleme/handlers/v2/user_profile.py

@@ -9,7 +9,7 @@
 from consoleme.lib.generic import get_random_security_logo, is_in_group
 from consoleme.lib.plugins import get_plugin_by_name
 
-stats = get_plugin_by_name(config.get("plugins.metrics"))()
+stats = get_plugin_by_name(config.get("plugins.metrics", "default_metrics"))()
 log = config.get_logger()
 
 

consoleme/lib/account_indexers/__init__.py

@@ -15,8 +15,8 @@
 from consoleme.models import CloudAccountModelArray
 
 log = config.get_logger(__name__)
-auth = get_plugin_by_name(config.get("plugins.auth"))()
-stats = get_plugin_by_name(config.get("plugins.metrics"))()
+auth = get_plugin_by_name(config.get("plugins.auth", "default_auth"))()
+stats = get_plugin_by_name(config.get("plugins.metrics", "default_metrics"))()
 
 
 async def cache_cloud_accounts() -> CloudAccountModelArray:

consoleme/lib/account_indexers/swag.py

@@ -10,7 +10,7 @@
 from consoleme.models import CloudAccountModel, CloudAccountModelArray
 
 log = config.get_logger()
-stats = get_plugin_by_name(config.get("plugins.metrics"))()
+stats = get_plugin_by_name(config.get("plugins.metrics", "default_metrics"))()
 
 
 async def retrieve_accounts_from_swag() -> CloudAccountModelArray:

consoleme/lib/auth.py

@@ -14,7 +14,7 @@
 from consoleme.lib.plugins import get_plugin_by_name
 
 crypto = Crypto()
-stats = get_plugin_by_name(config.get("plugins.metrics"))()
+stats = get_plugin_by_name(config.get("plugins.metrics", "default_metrics"))()
 log = config.get_logger()
 
 

consoleme/lib/aws.py

@@ -32,8 +32,8 @@
 ALL_IAM_MANAGED_POLICIES_LAST_UPDATE: int = 0
 
 log = config.get_logger(__name__)
-auth = get_plugin_by_name(config.get("plugins.auth"))()
-stats = get_plugin_by_name(config.get("plugins.metrics"))()
+auth = get_plugin_by_name(config.get("plugins.auth", "default_auth"))()
+stats = get_plugin_by_name(config.get("plugins.metrics", "default_metrics"))()
 
 
 @rate_limited()

consoleme/lib/aws_config/aws_config.py

@@ -9,7 +9,7 @@
 from consoleme.lib.plugins import get_plugin_by_name
 
 log = config.get_logger()
-stats = get_plugin_by_name(config.get("plugins.metrics"))()
+stats = get_plugin_by_name(config.get("plugins.metrics", "default_metrics"))()
 
 
 def query(

consoleme/lib/cache.py

@@ -15,7 +15,7 @@
 from consoleme.lib.s3_helpers import get_object, put_object
 
 red = RedisHandler().redis_sync()
-stats = get_plugin_by_name(config.get("plugins.metrics"))()
+stats = get_plugin_by_name(config.get("plugins.metrics", "default_metrics"))()
 
 
 async def store_json_results_in_redis_and_s3(

consoleme/lib/change_request.py

@@ -25,7 +25,9 @@
  Status,
 )
 
-group_mapping = get_plugin_by_name(config.get("plugins.group_mapping"))()
+group_mapping = get_plugin_by_name(
+ config.get("plugins.group_mapping", "default_group_mapping")
+)()
 ALL_ACCOUNTS = None
 
 

consoleme/lib/cloud_credential_authorization_mapping/internal_plugin.py

@@ -14,7 +14,9 @@ async def generate_credential_authorization_mapping(
  self, authorization_mapping: Dict[user_or_group, RoleAuthorizations]
  ) -> Dict[user_or_group, RoleAuthorizations]:
  """This will list accounts that meet the account attribute search criteria."""
- group_mapping = get_plugin_by_name(config.get("plugins.group_mapping"))()
+ group_mapping = get_plugin_by_name(
+ config.get("plugins.group_mapping", "default_group_mapping")
+ )()
 
  # Generate mapping from internal plugin
  authorization_mapping = (

consoleme/lib/crypto.py

@@ -6,7 +6,7 @@
 from consoleme.lib.plugins import get_plugin_by_name
 
 log = config.get_logger("consoleme")
-stats = get_plugin_by_name(config.get("plugins.metrics"))()
+stats = get_plugin_by_name(config.get("plugins.metrics", "default_metrics"))()
 
 
 class Crypto:

consoleme/lib/duo.py

@@ -8,7 +8,7 @@
 from consoleme.config import config
 from consoleme.lib.plugins import get_plugin_by_name
 
-stats = get_plugin_by_name(config.get("plugins.metrics"))()
+stats = get_plugin_by_name(config.get("plugins.metrics", "default_metrics"))()
 log = config.get_logger()
 
 

consoleme/lib/dynamo.py

@@ -43,7 +43,7 @@
  ["pending", "approved", "rejected", "cancelled", "expired", "removed"],
 )
 
-stats = get_plugin_by_name(config.get("plugins.metrics"))()
+stats = get_plugin_by_name(config.get("plugins.metrics", "default_metrics"))()
 log = config.get_logger("consoleme")
 crypto = Crypto()
 red = RedisHandler().redis_sync()

consoleme/lib/google.py

@@ -29,10 +29,10 @@
 from consoleme.lib.groups import does_group_require_bg_check
 from consoleme.lib.plugins import get_plugin_by_name
 
-stats = get_plugin_by_name(config.get("plugins.metrics"))()
+stats = get_plugin_by_name(config.get("plugins.metrics", "default_metrics"))()
 
 log = config.get_logger()
-auth = get_plugin_by_name(config.get("plugins.auth"))()
+auth = get_plugin_by_name(config.get("plugins.auth", "default_auth"))()
 
 
 async def add_user_to_group_task(

consoleme/lib/handler_utils.py

@@ -6,8 +6,8 @@
 from consoleme.lib.account_indexers import get_account_id_to_name_mapping
 from consoleme.lib.plugins import get_plugin_by_name
 
-aws = get_plugin_by_name(config.get("plugins.aws"))()
-stats = get_plugin_by_name(config.get("plugins.metrics"))()
+aws = get_plugin_by_name(config.get("plugins.aws", "default_aws"))()
+stats = get_plugin_by_name(config.get("plugins.metrics", "default_metrics"))()
 log = config.get_logger()
 
 # ALL ACCOUNTS is a dictionary of account ID to a list of account names (including aliases)