Refactor: Removed old modules

- Removed modules referring to command dictionary from common.py
- Cleaned up modules in docker.py
- Fixed return on set_command_attrs
- Removed dictionary modules in command_lib/command_lib.py

Signed-off-by: Nisha K <[email protected]>

command_lib/command_lib.py

@@ -10,7 +10,6 @@
 
 from utils.container import docker_command
 from utils.container import execute
-from utils.general import parse_command
 from utils.constants import container
 from classes.command import Command
 from report import errors
@@ -152,98 +151,9 @@ def set_command_attrs(command_obj):
  if ignore_word in command_obj.words:
  command_obj.set_ignore()
  break
-
-
-def get_packages_per_run(docker_run_command):
- '''Given a Docker RUN instruction retrieve a dictionary of recognized
- and unrecognized commands
- the dictionary should look like this:
- instruction: <dockerfile instruction>
- recognized: { <command name>:
- {installed: [list of packages installed]
- removed: [list of packaged removed]},...}
- unrecognized: [list shell commands that were not recognized]'''
- docker_inst = docker_run_command[0] + ' ' + docker_run_command[1]
- pkg_dict = {'instruction': docker_inst,
- 'recognized': {},
- 'unrecognized': []}
- shell_commands = get_shell_commands(docker_run_command[1])
- for command in shell_commands:
- installed_dict = {'installed': [], 'removed': []}
- command_obj = parse_command(command)
- # see if command is in the snippet library
- name = command_obj['name']
- sub = command_obj['subcommand']
- if name in command_lib['snippets'].keys():
- is_package_op = False
- if sub == command_lib['snippets'][name]['install']:
- is_package_op = True
- installed_dict['installed'] = command_obj['arguments']
- if sub == command_lib['snippets'][name]['remove']:
- is_package_op = True
- installed_dict['removed'] = command_obj['arguments']
- # add only if there are some packages installed or removed
- if is_package_op:
- pkg_dict['recognized'].update({name: installed_dict})
- else:
- pkg_dict['unrecognized'].append(command)
- return pkg_dict
-
-
-def get_package_listing(docker_instructions):
- '''Given the docker instructions in a dockerfile, get a dictionary of
- packages that are in the command library of retrievable sources
- If it does not exist in the library then record them under
- unrecognized commands
- the dict looks like this:
- recognized:{ <command name>:
- {installed: [list of packages installed],
- removed: [list of packages removed]}}
- unrecognized: [list of shell commands that were not recognized]
- '''
- pkg_dict = {'recognized': {}, 'unrecognized': []}
- shell_commands = []
- for instr in docker_instructions:
- if instr[0] == 'RUN':
- shell_commands.extend(get_shell_commands(instr[1]))
- for command in shell_commands:
- installed_dict = {'installed': [], 'removed': []}
- command_obj = parse_command(command)
- # see if command is in the snippet library
- name = command_obj['name']
- sub = command_obj['subcommand']
- if name in command_lib['snippets'].keys():
- is_package_op = False
- if sub == command_lib['snippets'][name]['install']:
- is_package_op = True
- installed_dict['installed'] = command_obj['arguments']
- if sub == command_lib['snippets'][name]['remove']:
- is_package_op = True
- installed_dict['removed'] = command_obj['arguments']
- # add only if there are some packages installed or removed
- if is_package_op:
- pkg_dict['recognized'].update({name: installed_dict})
- else:
- pkg_dict['unrecognized'].append(command)
- return pkg_dict
-
-
-def remove_uninstalled(pkg_dict):
- '''Given a dictionary containing the package listing for a set of
- docker commands, return an updated dictionary with only the packages that
- are installed
- The resulting dictionary should look like this:
- recognized:{ {<command name>: [list of packages installed]},...}
- unrecognized: [list of shell commands that were not recognized]
- '''
- for command in pkg_dict['recognized'].keys():
- installed_list = pkg_dict['recognized'][command]['installed']
- remove_list = pkg_dict['recognized'][command]['removed']
- for remove in remove_list:
- if remove in installed_list:
- installed_list.remove(remove)
- pkg_dict['recognized'].update({command: installed_list})
- return pkg_dict
+ return True
+ else:
+ return False
 
 
 def invoke_in_container(snippet_list, shell, package='', override=''):

common.py

@@ -4,15 +4,13 @@
 '''
 
 import logging
-import subprocess
 
 from classes.package import Package
 from classes.notice import Notice
 from command_lib import command_lib as cmdlib
 from report import info
 from report import errors
 from utils import cache as cache
-from utils import metadata as meta
 from utils import constants as const
 from utils.container import check_container
 '''
@@ -164,120 +162,17 @@ def fill_package_metadata(pkg_obj, pkg_listing, shell):
  pkg_obj.add_notice(no_url_listing_notice)
 
 
-def get_package_obj(command_name, package_name, shell):
- '''Given the command name, and the package name, retrieve the package
- information, create an oject and return the package object'''
- listing = cmdlib.get_command_listing(command_name)
- if listing:
- # get the unique or default information
- pkg_info = cmdlib.check_for_unique_package(
- listing['packages'], package_name)
- if pkg_info:
- pkg = Package(package_name)
- fill_package_metadata(pkg, pkg_info, shell)
- return pkg, ''
+def get_package_dependencies(package_listing, package_name, shell):
+ '''The package listing is the result of looking up the command name in the
+ command library. Given this listing, the package name and the shell
+ return a list of package dependency names'''
+ deps_listing, deps_msg = cmdlib.check_library_key(package_listing, 'deps')
+ if deps_listing:
+ deps_list, invoke_msg = cmdlib.get_pkg_attr_list(
+ shell, deps_listing, package_name=package_name)
+ if deps_list:
+ return list(set(deps_list)), ''
  else:
- return None, errors.no_command_listing.format(
- command_name=command_name)
-
- else:
- return None, errors.no_listing_for_snippet_key
-
-def get_package_dependencies(command_name, package_name, shell):
- '''Given the command name, the package name and the shell,
- find the list of dependencies'''
- deps = []
- # look up snippet library
- pkg_list = cmds.command_lib['snippets'][command_name]['packages']
- pkg_dict = check_for_unique_package(pkg_list, package_name)
- if pkg_dict and 'deps' in pkg_dict.keys():
- deps.extend(cmds.get_pkg_attr_list(shell, pkg_dict['deps'],
- package_name=package_name))
- return list(set(deps))
-
-
-def get_confirmed_packages(docker_run_inst, shell, prev_pkg_names):
- '''For a dockerfile run instruction which is a tuple of type:
- ('RUN', command)
- 1. Get the packages that were installed
- This is in the form of a dictionary that looks like this:
- instruction: <dockerfile instruction>
- recognized: {command_name: [list of installed packages], ...}
- unrecognized: [list of commands in the docker RUN instruction]
- 2. Get the dependencies for each of the packages that were installed
- 3. Remove dependencies already installed in the previous list of
- package names
- Update the dictionary to move the recognized to confirmed with a
- unique list of packages. The resulting dictionary looks like this:
- instruction: <dockerfile instruction>
- recognized: {command_name: [], ...}
- confirmed: {command_name: [],...}
- unrecognized: [list of commands in the docker RUN instruction]'''
- # get the instruction, the recognized packages and unrecognized commands
- run_dict = cmds.remove_uninstalled(
- cmds.get_packages_per_run(docker_run_inst))
- run_dict.update({'confirmed': {}})
- # get package dependencies
- for cmd in run_dict['recognized'].keys():
- cmd_dict = {cmd: []}
- all_pkgs = []
- remove_pkgs = []
- for pkg in run_dict['recognized'][cmd]:
- deps = get_package_dependencies(cmd, pkg, shell)
- for p in prev_pkg_names:
- if p in deps:
- deps.remove(p)
- all_pkgs.append(pkg)
- all_pkgs.extend(deps)
- remove_pkgs.append(pkg)
- cmd_dict[cmd].extend(list(set(all_pkgs)))
- run_dict['confirmed'].update(cmd_dict)
- for rem in remove_pkgs:
- run_dict['recognized'][cmd].remove(rem)
- return run_dict
-
-
-
-
-def get_packages_from_snippets(command_dict, shell):
- '''Command dictionary looks like this:
- { command: [list of packages], command: [list of packages]...}
- This is the result of parsing through a Dockerfile RUN command.
- Return a list of packages objects that are installed from the Dockerfile
- RUN command'''
- package_list = []
- for cmd in command_dict.keys():
- for pkg in command_dict[cmd]:
- pkg_obj = get_package_obj(cmd, pkg, shell)
- package_list.append(pkg_obj)
- return package_list
-
-
-def get_layer_history(image_tag_string):
- '''For an available image, get a list of tuples containing the dockerfile
- instruction that created the layer and the diff id of that layer'''
- history_list = []
- # save the image first
- if not cmds.extract_image_metadata(image_tag_string):
- # there was some error in extracting the metadata so we cannot
- # find the context for the base image
- print(cannot_extract_base_image)
- raise
+ return [], invoke_msg
  else:
- # get the list of non-empty history
- config = meta.get_image_config()
- history = meta.get_nonempty_history(config)
- diff_ids = meta.get_diff_ids(config)
- # create a list of tuples
- for index in range(0, len(history)):
- history_list.append((history[index], diff_ids[index]))
- return history_list
-
-
-def collate_package_names(pkg_name_list, layer_obj):
- '''Given a list of package names or an empty list. Append packages from
- the layer object and return the new list.
- Use this to keep track of packages introduced in the layers before
- so the subsequent layers do not have the same packages.'''
- for pkg in layer_obj.get_package_names():
- pkg_name_list.append(pkg)
+ return [], deps_msg

docker.py

@@ -4,16 +4,12 @@
 '''
 import logging
 import subprocess
-import sys
 
 from classes.docker_image import DockerImage
 from classes.notice import Notice
-from classes.layer import Layer
-from classes.package import Package
 from utils import dockerfile as df
 from utils import container as cont
 from utils import constants as const
-from command_lib import command_lib as cmdlib
 from report import errors
 '''
 Docker specific functions - used when trying to retrieve packages when
@@ -104,12 +100,6 @@ def check_base_image(image, tag):
  success = cont.pull_image(image_tag_string)
  return success
 
-# REMOVE
-def get_image_tag_string(image_tag_tuple):
- '''Given a tuple of the image and tag, return a string containing
- the image and tag'''
- return image_tag_tuple[0] + df.tag_separator + image_tag_tuple[1]
-
 
 def get_dockerfile_image_tag():
  '''Return the image and tag used to build an image from the dockerfile'''
@@ -133,25 +123,3 @@ def is_build():
  else:
  success = True
  return success, msg
-
-# LEFTOFF
-def get_dockerfile_packages():
- '''Given the dockerfile commands get packages that may have been
- installed. Use this when there is no image or if it cannot be
- built
- 1. For each RUN directive get the command used and the packages
- installed with it
- 2. All of the packages that are recognized would be unconfirmed
- because there is no container to run the snippets against
- All the unrecognized commands will be returned as is
- Since there will be nothing more to do - recognized is just a list
- of packages that may have been installed in the dockerfile'''
- pkg_dict = cmds.remove_uninstalled(cmds.get_package_listing(
- docker_commands))
- recognized_list = []
- for command in pkg_dict['recognized'].keys():
- recognized_list.extend(pkg_dict['recognized'][command])
- pkg_dict.update({'recognized': recognized_list})
- return pkg_dict
-
-