Refactor utils

scar/parser/cfgfile.py

@@ -14,10 +14,11 @@
 
 import json
 import os
+import shutil
 import scar.exceptions as excp
 import scar.logger as logger
-import scar.utils as utils
-import shutil
+from scar.utils import FileUtils
+
 
 default_cfg = {
  "scar" : {
@@ -61,13 +62,13 @@
 
 
 class ConfigFileParser(object):
- 
+
  config_file_name = "scar.cfg"
  backup_config_file_name = "scar.cfg_old"
  config_folder_name = ".scar"
- config_file_folder = utils.join_paths(os.path.expanduser("~"), config_folder_name)
- config_file_path = utils.join_paths(config_file_folder, config_file_name)
- backup_file_path = utils.join_paths(config_file_folder, backup_config_file_name)
+ config_file_folder = FileUtils.join_paths(os.path.expanduser("~"), config_folder_name)
+ config_file_path = FileUtils.join_paths(config_file_folder, config_file_name)
+ backup_file_path = FileUtils.join_paths(config_file_folder, backup_config_file_name)
 
  @excp.exception(logger)
  def __init__(self):
@@ -96,6 +97,9 @@ def get_properties(self):
  def get_supervisor_version(self):
  return self.cfg_data['scar']['supervisor_version']
 
+ def get_udocker_zip_url(self):
+ return self.cfg_data['scar']['udocker_info']['zip_url']
+
  def _add_missing_attributes(self):
  logger.info("Updating old scar config file '{0}'.\n".format(self.config_file_path))
  shutil.copy(self.config_file_path, self.backup_file_path)

scar/parser/cli.py

@@ -13,11 +13,12 @@
 # limitations under the License.
 
 import argparse
+import sys
 import scar.exceptions as excp
 import scar.logger as logger
-import scar.utils as utils
+from scar.utils import DataTypesUtils
 import scar.version as version
-import sys
+
 
 class CommandParser(object):
 
@@ -184,7 +185,7 @@ def parse_arguments(self):
  raise excp.MissingCommandError()
  scar_args = self.parse_scar_args(cmd_args)
  aws_args = self.parse_aws_args(cmd_args)
- return utils.merge_dicts(scar_args, aws_args)
+ return DataTypesUtils.merge_dicts(scar_args, aws_args)
  except AttributeError as ae:
  logger.error("Incorrect arguments: use scar -h to see the options available",
  "Error parsing arguments: {}".format(ae))
@@ -204,43 +205,43 @@ def parse_aws_args(self, cmd_args):
  self.set_args(aws_args, 'cloudwatch', self.parse_cloudwatchlogs_args(cmd_args))
  self.set_args(aws_args, 's3', self.parse_s3_args(cmd_args))
  self.set_args(aws_args, 'api_gateway', self.parse_api_gateway_args(cmd_args))
- aws_args.update(utils.parse_arg_list(other_args, cmd_args))
- return {'aws' : aws_args }
+ aws_args.update(DataTypesUtils.parse_arg_list(other_args, cmd_args))
+ return {'aws' : aws_args}
 
  def parse_scar_args(self, cmd_args):
  scar_args = ['func', 'conf_file', 'json',
  'verbose', 'path', 'all',
  'preheat', 'execution_mode',
  'output_file', 'supervisor_version']
- return {'scar' : utils.parse_arg_list(scar_args, cmd_args)}
+ return {'scar' : DataTypesUtils.parse_arg_list(scar_args, cmd_args)}
 
  def parse_lambda_args(self, cmd_args):
  lambda_args = ['name', 'asynchronous', 'init_script', 'run_script', 'c_args', 'memory', 'time',
  'timeout_threshold', 'log_level', 'image', 'image_file', 'description', 
  'lambda_role', 'extra_payload', ('environment', 'environment_variables'),
  'layers', 'lambda_environment', 'list_layers']
- return utils.parse_arg_list(lambda_args, cmd_args)
+ return DataTypesUtils.parse_arg_list(lambda_args, cmd_args)
 
  def parse_batch_args(self, cmd_args):
  batch_args = [('batch_vcpus', 'vcpus'), ('batch_memory', 'memory'), 'enable_gpu']
- return utils.parse_arg_list(batch_args, cmd_args)
+ return DataTypesUtils.parse_arg_list(batch_args, cmd_args)
 
  def parse_iam_args(self, cmd_args):
  iam_args = [('iam_role', 'role')]
- return utils.parse_arg_list(iam_args, cmd_args) 
+ return DataTypesUtils.parse_arg_list(iam_args, cmd_args) 
 
  def parse_cloudwatchlogs_args(self, cmd_args):
  cw_log_args = ['log_stream_name', 'request_id']
- return utils.parse_arg_list(cw_log_args, cmd_args)
+ return DataTypesUtils.parse_arg_list(cw_log_args, cmd_args)
 
  def parse_api_gateway_args(self, cmd_args):
  api_gtw_args = [('api_gateway_name', 'name'), 'parameters', 'data_binary', 'json_data']
- return utils.parse_arg_list(api_gtw_args, cmd_args) 
+ return DataTypesUtils.parse_arg_list(api_gtw_args, cmd_args) 
 
  def parse_s3_args(self, cmd_args):
  s3_args = ['deployment_bucket', 
  'input_bucket', 
  'output_bucket', 
  ('bucket', 'input_bucket'), 
  ]
- return utils.parse_arg_list(s3_args, cmd_args) 
+ return DataTypesUtils.parse_arg_list(s3_args, cmd_args) 

scar/parser/yaml.py

@@ -11,13 +11,15 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from scar.exceptions import YamlFileNotFoundError
+
 import os
-import scar.utils as utils
 import yaml
+from scar.exceptions import YamlFileNotFoundError
+from scar.utils import DataTypesUtils
+
 
 class YamlParser(object):
- 
+
  def __init__(self, scar_args):
  file_path = scar_args['conf_file']
  if os.path.isfile(file_path):
@@ -27,7 +29,7 @@ def __init__(self, scar_args):
  raise YamlFileNotFoundError(file_path=file_path)
 
  def parse_arguments(self):
- functions = [] 
+ functions = []
  for function in self.yaml_data['functions']:
  functions.append(self.parse_aws_function(function, self.yaml_data['functions'][function]))
  return functions[0]
@@ -39,17 +41,17 @@ def parse_aws_function(self, function_name, function_data):
  aws_args['lambda'] = self.parse_lambda_args(function_data)
  aws_args['lambda']['name'] = function_name
  aws_services = ['iam', 'cloudwatch', 's3', 'api_gateway', 'batch']
- aws_args.update(utils.parse_arg_list(aws_services, function_data))
+ aws_args.update(DataTypesUtils.parse_arg_list(aws_services, function_data))
  other_args = [('profile','boto_profile'),'region','execution_mode']
- aws_args.update(utils.parse_arg_list(other_args, function_data))
- scar_args.update(utils.parse_arg_list(['supervisor_version'], function_data))
+ aws_args.update(DataTypesUtils.parse_arg_list(other_args, function_data))
+ scar_args.update(DataTypesUtils.parse_arg_list(['supervisor_version'], function_data))
  scar = {'scar' : scar_args if scar_args else {}}
  aws = {'aws' : aws_args if aws_args else {}}
- return utils.merge_dicts(scar, aws)
+ return DataTypesUtils.merge_dicts(scar, aws)
 
  def parse_lambda_args(self, cmd_args):
  lambda_args = ['asynchronous', 'init_script', 'run_script', 'c_args', 'memory', 'time',
  'timeout_threshold', 'log_level', 'image', 'image_file', 'description', 
  'lambda_role', 'extra_payload', ('environment', 'environment_variables'),
  'layers', 'lambda_environment']
- return utils.parse_arg_list(lambda_args, cmd_args)
+ return DataTypesUtils.parse_arg_list(lambda_args, cmd_args)

scar/providers/aws/__init__.py

@@ -14,7 +14,7 @@
 """Module with the generic classes and methods used in other aws classes"""
 
 from typing import Dict
-import scar.utils as utils
+from scar.utils import DataTypesUtils
 from scar.providers.aws.clients.apigateway import APIGatewayClient
 from scar.providers.aws.clients.batchfunction import BatchClient
 from scar.providers.aws.clients.cloudwatchlogs import CloudWatchLogsClient
@@ -46,7 +46,7 @@ def _get_client_args(self) -> Dict:
  return {'client' : {'region_name' : self.aws.region},
  'session' : {'profile_name' : self.aws.boto_profile}}
 
- @utils.lazy_property
+ @DataTypesUtils.lazy_property
  def client(self):
  """Returns the required boto client based on the implementing class name."""
  client_name = self.__class__.__name__.upper()

scar/providers/aws/clients/__init__.py

@@ -16,7 +16,7 @@
 from typing import Dict
 import boto3
 import botocore
-from scar.utils import lazy_property
+from scar.utils import DataTypesUtils
 
 
 class BotoClient():
@@ -40,7 +40,7 @@ def __init__(self, client: Dict = None, session: Dict = None):
  if session:
  self.session_args = session
 
- @lazy_property
+ @DataTypesUtils.lazy_property
  def client(self):
  """Returns a boto client based on the 'boto_client_name' property,
  the region specified on the client args and

scar/providers/aws/clients/iam.py

@@ -19,7 +19,7 @@
 from scar.providers.aws.clients import BotoClient
 from scar.exceptions import exception, GetUserInfoError
 import scar.logger as logger
-import scar.utils as utils
+from scar.utils import StrUtils
 
 
 class IAMClient(BotoClient):
@@ -40,7 +40,7 @@ def get_user_info(self) -> Dict:
  if cerr.response['Error']['Code'] == 'AccessDenied':
  # If the user doesn't have access rights to IAMClient
  # we can find the user name in the error response
- user_name = utils.find_expression(str(cerr), self._USER_NAME_REGEX)
+ user_name = StrUtils.find_expression(str(cerr), self._USER_NAME_REGEX)
  return {'UserName' : user_name,
  'User' : {'UserName' : user_name,
  'UserId' : ''}}

scar/providers/aws/clients/lambdafunction.py

@@ -18,7 +18,7 @@
 from scar.providers.aws.clients import BotoClient
 import scar.exceptions as excp
 import scar.logger as logger
-import scar.utils as utils
+from scar.utils import StrUtils
 
 
 class LambdaClient(BotoClient):
@@ -74,7 +74,7 @@ def invoke_function(self, **kwargs: Dict) -> Dict:
  def add_invocation_permission(self, **kwargs: Dict) -> Dict:
  """Adds a permission to the resource policy associated
  with the specified AWS Lambda function."""
- kwargs['StatementId'] = utils.get_random_uuid4_str()
+ kwargs['StatementId'] = StrUtils.get_random_uuid4_str()
  kwargs['Action'] = "lambda:InvokeFunction"
  return self.client.add_permission(**kwargs)