Refactored boto client module and update boto clients

scar/providers/aws/__init__.py

@@ -42,7 +42,7 @@ class GenericClient():
  def __init__(self, aws_properties: Dict):
  self.aws = aws_properties
 
- def _get_client_args(self):
+ def _get_client_args(self) -> Dict:
  return {'client' : {'region_name' : self.aws.region},
  'session' : {'profile_name' : self.aws.boto_profile}}
 

scar/providers/aws/clients/__init__.py

@@ -0,0 +1,57 @@
+# Copyright (C) GRyCAP - I3M - UPV
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Module with the generic classes and methods used in the boto client modules."""
+
+from typing import Dict
+import boto3
+import botocore
+from scar.utils import lazy_property
+
+
+class BotoClient():
+ """Generic class in charge of creating a boto client based on the
+ 'boto_client_name' property of the class that inherits it."""
+
+ _READ_TIMEOUT = 360
+ _BOTO_CLIENT_NAME = ''
+
+ def __init__(self, client: Dict = None, session: Dict = None):
+ """
+ Default client args:
+ 'client' : {'region_name' : self.aws.region}
+ Default session args:
+ 'session' : {'profile_name' : self.aws.boto_profile}
+ """
+ self.client_args = {}
+ if client:
+ self.client_args = client
+ self.session_args = {}
+ if session:
+ self.session_args = session
+
+ @lazy_property
+ def client(self):
+ """Returns a boto client based on the 'boto_client_name' property,
+ the region specified on the client args and
+ the profile specified on the session args."""
+ # 'default' profile if nothing set
+ session = boto3.Session(**self.session_args)
+ self.client_args['config'] = botocore.config.Config(read_timeout=self._READ_TIMEOUT)
+ return session.client(self._BOTO_CLIENT_NAME, **self.client_args)
+
+ def get_access_key(self) -> str:
+ """Returns the access key belonging to the boto_profile used."""
+ session = boto3.Session(**self.session_args)
+ credentials = session.get_credentials()
+ return credentials.access_key

scar/providers/aws/clients/apigateway.py

@@ -12,89 +12,90 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import time
 from botocore.exceptions import ClientError
-from scar.providers.aws.clients.boto import BotoClient
+from scar.providers.aws.clients import BotoClient
 import scar.exceptions as excp
 import scar.logger as logger
-import time
+
 
 class APIGatewayClient(BotoClient):
  '''A low-level client representing Amazon API Gateway.
  https://boto3.readthedocs.io/en/latest/reference/services/apigateway.html'''
- 
+
  # Parameter used by the parent to create the appropriate boto3 client
- boto_client_name = 'apigateway'
- endpoint_configuration = {'types': ['REGIONAL']}
- API_DESCRIPTION="API created automatically with SCAR"
- MAX_NUMBER_OF_RETRIES = 5
- WAIT_BETWEEN_RETIRES = 5
- 
- @excp.exception(logger) 
- def create_rest_api(self, name, count=MAX_NUMBER_OF_RETRIES):
- """ 
+ _BOTO_CLIENT_NAME = 'apigateway'
+ _ENDPOINT_CONFIGURATION = {'types': ['REGIONAL']}
+ _API_DESCRIPTION = "API created automatically with SCAR"
+ _MAX_NUMBER_OF_RETRIES = 5
+ _WAIT_BETWEEN_RETIRES = 5
+
+ @excp.exception(logger)
+ def create_rest_api(self, name, count=_MAX_NUMBER_OF_RETRIES):
+ """
  Creates a new RestApi resource.
  https://boto3.readthedocs.io/en/latest/reference/services/apigateway.html#APIGateway.Client.create_rest_api
- 
+
  :param str name: The name of the RestApi.
  :param int count: (Optional) The maximum number of retries to create the API
  """
- try: 
+ try:
  return self.client.create_rest_api(name=name,
- description=self.API_DESCRIPTION,
- endpointConfiguration=self.endpoint_configuration)
+ description=self._API_DESCRIPTION,
+ endpointConfiguration=self._ENDPOINT_CONFIGURATION)
  except ClientError as ce:
- if (ce.response['Error']['Code'] == 'TooManyRequestsException') and (self.MAX_NUMBER_OF_RETRIES > 0):
- time.sleep(self.WAIT_BETWEEN_RETIRES)
- return self.create_rest_api(name, count-1)
+ if (ce.response['Error']['Code'] == 'TooManyRequestsException') and (self._MAX_NUMBER_OF_RETRIES > 0):
+ time.sleep(self._WAIT_BETWEEN_RETIRES)
+ return self.create_rest_api(name, count - 1)
  except:
  raise excp.ApiCreationError(api_name=name)
- 
- @excp.exception(logger) 
+
+ @excp.exception(logger)
  def get_resources(self, api_id):
  ''' Lists information about a collection of Resource resources.
  https://boto3.readthedocs.io/en/latest/reference/services/apigateway.html#APIGateway.Client.get_resources
  '''
- return self.client.get_resources(restApiId=api_id) 
- 
- @excp.exception(logger) 
+ return self.client.get_resources(restApiId=api_id)
+
+ @excp.exception(logger)
  def create_resource(self, api_id, parent_id, path_part):
  ''' Creates a new RestApi resource.
  https://boto3.readthedocs.io/en/latest/reference/services/apigateway.html#APIGateway.Client.create_rest_api
  '''
  return self.client.create_resource(restApiId=api_id, parentId=parent_id, pathPart=path_part)
- 
- @excp.exception(logger) 
+
+ @excp.exception(logger)
  def create_method(self, **kwargs):
  ''' Add a method to an existing Resource resource.
  https://boto3.readthedocs.io/en/latest/reference/services/apigateway.html#APIGateway.Client.put_method
  '''
  return self.client.put_method(**kwargs)
- 
- @excp.exception(logger) 
+
+ @excp.exception(logger)
  def set_integration(self, **kwargs):
  ''' Sets up a method's integration.
  https://boto3.readthedocs.io/en/latest/reference/services/apigateway.html#APIGateway.Client.put_integration
  Also https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html
  '''
  return self.client.put_integration(**kwargs)
- 
- @excp.exception(logger) 
+
+ @excp.exception(logger)
  def create_deployment(self, api_id, stage_name):
  ''' Creates a Deployment resource, which makes a specified RestApi callable over the internet.
  https://boto3.readthedocs.io/en/latest/reference/services/apigateway.html#APIGateway.Client.create_deployment
  '''
  return self.client.create_deployment(restApiId=api_id, stageName=stage_name)
- 
- @excp.exception(logger) 
- def delete_rest_api(self, api_id, count=MAX_NUMBER_OF_RETRIES):
+
+ @excp.exception(logger)
+ def delete_rest_api(self, api_id, count=_MAX_NUMBER_OF_RETRIES):
  ''' Deletes the specified API.
  https://boto3.readthedocs.io/en/latest/reference/services/apigateway.html#APIGateway.Client.delete_rest_api
  '''
  try:
  return self.client.delete_rest_api(restApiId=api_id)
  except ClientError as ce:
- if (ce.response['Error']['Code'] == 'TooManyRequestsException') and (self.MAX_NUMBER_OF_RETRIES > 0):
- time.sleep(self.WAIT_BETWEEN_RETIRES)
- return self.delete_rest_api(api_id, count-1)
+ if (ce.response['Error']['Code'] == 'TooManyRequestsException') and (self._MAX_NUMBER_OF_RETRIES > 0):
+ time.sleep(self._WAIT_BETWEEN_RETIRES)
+ return self.delete_rest_api(api_id, count - 1)
  else:
- raise 
+ raise

scar/providers/aws/clients/batchfunction.py

@@ -12,24 +12,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from scar.providers.aws.clients.boto import BotoClient
+from scar.providers.aws.clients import BotoClient
 import scar.logger as logger
 
+
 class BatchClient(BotoClient):
  '''A low-level client representing aws batchClient.
- http://boto3.readthedocs.io/en/latest/reference/services/batch.html''' 
- 
+ http://boto3.readthedocs.io/en/latest/reference/services/batch.html'''
+
  # Parameter used by the parent to create the appropriate boto3 client
- boto_client_name = 'batch'
- 
+ _BOTO_CLIENT_NAME = 'batch'
+
  def create_compute_environment(self, **kwargs):
  '''
  Creates a new compute environment.
  http://boto3.readthedocs.io/en/latest/reference/services/batch.html#Batch.Client.create_compute_environment
  '''
  logger.debug("Creating compute environment.")
  return self.client.create_compute_environment(**kwargs)
- 
+
  def create_job_queue(self, **kwargs):
  '''
  Creates a new job queue.
@@ -53,22 +54,22 @@ def describe_job_queues(self, **kwargs):
  '''
  logger.debug("Describing job queue.")
  return self.client.describe_job_queues(**kwargs)
- 
+
  def describe_job_definitions(self, **kwargs):
  '''
  Describes a list of job definitions.
  https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/batch.html#Batch.Client.describe_job_definitions
  '''
  logger.debug("Describing job definition.")
  return self.client.describe_job_definitions(**kwargs)
- 
+
  def deregister_job_definition(self, **kwargs):
  '''
  Deregisters an AWS Batch job definition.
  https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/batch.html#Batch.Client.deregister_job_definition
  '''
  logger.debug("Deleting job definition.")
- return self.client.deregister_job_definition(**kwargs) 
+ return self.client.deregister_job_definition(**kwargs)
 
  def update_job_queue(self, **kwargs):
  '''
@@ -77,7 +78,7 @@ def update_job_queue(self, **kwargs):
  '''
  logger.debug("Updating job queue.")
  return self.client.update_job_queue(**kwargs)
- 
+
  def delete_job_queue(self, **kwargs):
  '''
  Delete a job queue.
@@ -86,23 +87,22 @@ def delete_job_queue(self, **kwargs):
  logger.debug("Deleting job queue.")
  return self.client.delete_job_queue(**kwargs)
 
-
  def update_compute_environment(self, **kwargs):
  '''
  update a compute environment.
  https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/batch.html#Batch.Client.update_compute_environment
  '''
  logger.debug("Updating compute environment.")
  return self.client.update_compute_environment(**kwargs)
- 
+
  def delete_compute_environment(self, **kwargs):
  '''
  Delete a compute environmet.
  https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/batch.html#Batch.Client.delete_compute_environment
  '''
  logger.debug("Deleting compute environment.")
  return self.client.delete_compute_environment(**kwargs)
- 
+
  def describe_jobs(self, **kwargs):
  '''
  Describe a job.
@@ -111,5 +111,3 @@ def describe_jobs(self, **kwargs):
  logger.debug("Describing a job.")
  return self.client.describe_jobs(**kwargs)
 
-
-