Refactor: Added full report generation

Tern can now print a full report for cached
layers. Tern cannot print a summary report yet.

- Added string printing modules to content.py for
full reporting
- Removed deprecated Image documentation
- Added methods to Origins class to enable reporting
- Modified docker.py and common.py to use the modified
Origins class
- Added reporting file name to constants.py
- Fixed some formatting in formats.py
- Added report writing to report.py
- Had reporting also look in the cache for the full image

Signed-off-by: Nisha K <[email protected]>

classes/image.py

@@ -16,8 +16,6 @@ class Image(object):
  methods:
  load_image: this method is to be implemented in the derived classes
  get_layer_diff_ids: returns a list of layer diff ids only
- get_image_option: returns whether the image object was instantiated
- using the repotag or id
  '''
  def __init__(self, id=None):
  '''Either initialize using id'''

classes/origins.py

@@ -18,6 +18,8 @@ class Origins(object):
  If there is no NoticeOrigin object with the given string,
  create a NoticeOrigin object and add it to the list of
  origins
+ add_notice_origin: add an empty NoticeOrigin object
+ is_empty: check if there are any notices
  '''
  def __init__(self):
  self.__origins = []
@@ -39,3 +41,15 @@ def add_notice_to_origins(self, orig_string, notice):
  notice_orij = NoticeOrigin(orig_string)
  notice_orij.add_notice(notice)
  self.__origins.append(notice_orij)
+
+ def add_notice_origin(self, orig_string):
+ self.__origins.append(NoticeOrigin(orig_string))
+
+ def is_empty(self):
+ empty = True
+ if len(self.__origins) != 0:
+ for orij in self.__origins:
+ if len(orij.notices) != 0:
+ empty = False
+ break
+ return empty

common.py

@@ -37,20 +37,27 @@ def load_from_cache(image):
  doesn't exist continue. If not all the layers have packages, return False
  else return True'''
  is_full = True
+ # check if we can use repotag
+ origin_str = ''
+ if image.repotag:
+ origin_str = image.repotag
+ else:
+ origin_str = 'Image ID - ' + image.id[:10]
  for layer in image.layers:
  if not layer.packages:
  # create an origin for this layer
- origin_str = image.get_image_option() + layer.diff_id
+ origin_str = origin_str + ': ' + layer.diff_id[:10]
+ print(origin_str)
  # there are no packages in this layer
  # try to get it from the cache
  raw_pkg_list = cache.get_packages(layer.diff_id)
  if not raw_pkg_list:
  is_full = False
  else:
  message = formats.loading_from_cache.format(
- layer_id=layer.diff_id)
+ layer_id=layer.diff_id[:10])
  # add notice to the origin
- image.origins.add_notice_to_origins(
+ layer.origins.add_notice_to_origins(
  origin_str, Notice(message, 'info'))
  for pkg_dict in raw_pkg_list:
  pkg = Package(pkg_dict['name'])

docker.py

@@ -67,6 +67,7 @@ def get_dockerfile_base():
  repotag = base_image_tag[0] + df.tag_separator + base_image_tag[1]
  from_line = 'FROM ' + repotag
  base_image = DockerImage(repotag)
+ base_image.origins.add_notice_origin(dockerfile_lines)
  base_image.name = base_image_tag[0]
  # check if there is a tag
  if not base_image_tag[1]:
@@ -149,7 +150,7 @@ def add_packages_from_history(image_obj, shell):
  # for Docker the created_by comes from the instruction in the
  # dockerfile
  # each layer is an origin
- origin_str = layer.diff_id + ': ' + instruction
+ origin_str = layer.diff_id[:10] + ': ' + instruction
  run_command_line = instruction.split(' ', 1)[1]
  cmd_list, msg = common.filter_install_commands(run_command_line)
  if msg:

report/content.py

@@ -7,7 +7,7 @@
 from command_lib.command_lib import get_command_listing
 from command_lib.command_lib import FormatAwk
 from command_lib.command_lib import check_for_unique_package
-import formats
+from report import formats
 
 '''
 Functions to generate content for the report
@@ -34,7 +34,7 @@ def print_invoke_list(info_dict, info):
  return report
 
 
-def print_base_image_info(base_image_tag):
+def print_base_invoke(base_image_tag):
  '''Given the base image and tag in a tuple return a string containing
  the command_lib/base.yml'''
  info = get_base_listing(base_image_tag)
@@ -47,7 +47,7 @@ def print_base_image_info(base_image_tag):
  return report
 
 
-def print_package_info(command_name, package_name):
+def print_package_invoke(command_name, package_name):
  '''Given the command name to look up in the snippet library and the
  package name, return a string with the list of commands that will be
  invoked in the container'''
@@ -65,3 +65,48 @@ def print_package_info(command_name, package_name):
  report = report + print_invoke_list(pkg_dict, 'deps').format_map(
  FormatAwk(package=package_name))
  return report
+
+
+def print_package(pkg_obj, prefix):
+ '''Given a Package object, print out information with a prefix'''
+ notes = formats.package_demarkation
+ notes = notes + prefix + formats.package_name.format(
+ package_name=pkg_obj.name)
+ notes = notes + prefix + formats.package_version.format(
+ package_version=pkg_obj.version)
+ notes = notes + prefix + formats.package_url.format(
+ package_url=pkg_obj.src_url)
+ notes = notes + prefix + formats.package_license.format(
+ package_license=pkg_obj.license)
+ notes = notes + '\n'
+ return notes
+
+
+def print_notices(notice_origin, origin_pfx, notice_pfx):
+ '''Given a NoticeOrigin object with a prefix (like a series of tabs)
+ for the origin and the notice messages, return the notes'''
+ notes = origin_pfx + notice_origin.origin_str + ':\n'
+ for notice in notice_origin.notices:
+ notes = notes + notice_pfx + notice.level + ': ' + \
+ notice.message + '\n'
+ return notes
+
+
+def print_full_report(image):
+ '''Given an image, go through the Origins object and collect all the
+ notices for the image, layers and packages'''
+ notes = ''
+ for image_origin in image.origins.origins:
+ notes = notes + print_notices(image_origin, '', '\t')
+ for layer in image.layers:
+ if layer.import_image:
+ notes = notes + print_full_report(layer.import_image)
+ else:
+ for layer_origin in layer.origins.origins:
+ notes = notes + print_notices(layer_origin, '\t', '\t\t')
+ for package in layer.packages:
+ notes = notes + print_package(package, '\t\t')
+ for package_origin in package.origins.origins:
+ notes = notes + print_notices(
+ package_origin, '\t\t', '\t\t\t')
+ return notes

report/formats.py

@@ -19,27 +19,29 @@
 
 # general formatting
 # report disclaimer
-disclaimer = f'''This report was generated using the Tern Project\n\n'''
+disclaimer = '''This report was generated using the Tern Project\n\n'''
 # cache
-retrieve_from_cache = f'''Retrieving packages from cache for layer ''' \
- f'''{layer_id}:\n\n'''
+retrieve_from_cache = '''Retrieving packages from cache for layer ''' \
+ '''{layer_id}:\n\n'''
 # command library
-base_listing = f'''Direct listing in command_lib/base.yml:\n\n'''
-snippet_listing = f'''Direct listing in command_lib/snippets.yml:\n\n'''
-invoke_for_base = f'''Using invoke listing in command_lib/base.yml:\n\n'''
-invoke_for_snippets = f'''Using invoke listing in command_lib/snippets.yml''' \
- f''':\n\n'''
+base_listing = '''Direct listing in command_lib/base.yml:\n\n'''
+snippet_listing = '''Direct listing in command_lib/snippets.yml:\n\n'''
+invoke_for_base = '''Using invoke listing in command_lib/base.yml:\n\n'''
+invoke_for_snippets = '''Using invoke listing in command_lib/snippets.yml''' \
+ ''':\n\n'''
 invoke_in_container = '''\tin container:\n'''
 invoke_on_host = '''\ton host:\n'''
 # package information
-package_info = f'''Package: {package_name}\nVersion: {package_version}\n''' \
- f'''Project URL: {package_url}\nLicense: {package_license}\n\n'''
+package_name = '''Package: {package_name}\n'''
+package_version = '''Version: {package_version}\n'''
+package_url = '''Project URL: {package_url}\n'''
+package_license = '''License: {package_license}\n\n'''
 # notes
-package_notes = f'''Errors: {package_info_retrieval_errors}\n''' \
- f'''Improvements: {package_info_reporting_improvements}\n'''
+package_notes = '''Errors: {package_info_retrieval_errors}\n''' \
+ '''Improvements: {package_info_reporting_improvements}\n'''
 # demarkation
-package_demarkation = f'''------------------------------------------------''' \
- f'''\n\n'''
+package_demarkation = '''------------------------------------------------''' \
+ '''\n\n'''
 
 # informational
 loading_from_cache = '''Loading packages from cache for layer {layer_id}:'''
@@ -57,5 +59,9 @@
 
 # dockerfile report sections
 dockerfile_header = '''Report from Dockerfile\n'''
-dockerfile_base = f'''Base Image: {base_image_instructions}\n'''
-dockerfile_line = f'''Instruction Line: {dockerfile_instruction}\n'''
+dockerfile_base = '''Base Image: {base_image_instructions}\n'''
+dockerfile_line = '''Instruction Line: {dockerfile_instruction}\n'''
+
+# format for notices
+notice_format = '''{origin}:\n\t{info}\n\twarnings:{warnings}''' \
+ '''\n\terrors:{errors}\n\thints:{hints}\n'''

report/report.py

@@ -6,6 +6,7 @@
 import logging
 import subprocess
 
+from report import content
 from utils import container
 from utils import constants
 from utils import cache
@@ -121,7 +122,7 @@ def execute_dockerfile(args):
  base_image = load_base_image()
  logger.debug('Base image loaded...')
  # check if the base image added any notices
- if len(base_image.origins.origins) == 0:
+ if base_image.origins.is_empty():
  # load any packages from cache
  logger.debug('Looking up cache for base image layers...')
  if not common.load_from_cache(base_image):
@@ -148,21 +149,27 @@ def execute_dockerfile(args):
  if build:
  # attempt to get built image metadata
  full_image = load_full_image()
- if len(full_image.origins.origins) == 0:
+ if full_image.origins.is_empty():
  # link layer to imported base image
  full_image.set_image_import(base_image)
- # find packages per layer
- container.start_container(full_image.repotag)
- logger.debug('Retrieving metadata using scripts from '
- 'snippets.yml')
- docker.add_packages_from_history(full_image, shell)
- container.remove_container()
- # record missing layers in the cache
- common.save_to_cache(full_image)
- cache.save()
+ if not common.load_from_cache(full_image):
+ # find packages per layer
+ container.start_container(full_image.repotag)
+ logger.debug('Retrieving metadata using scripts from '
+ 'snippets.yml')
+ docker.add_packages_from_history(full_image, shell)
+ container.remove_container()
+ # record missing layers in the cache
+ common.save_to_cache(full_image)
+ cache.save()
  else:
  # we cannot extract the built image's metadata
  dockerfile_parse = True
+ report = content.print_full_report(full_image)
+ logger.debug('Cleaning up...')
+ container.remove_image(full_image.repotag)
+ logger.debug('Writing report...')
+ write_report(report)
  else:
  # we cannot build the image
  common.record_image_layers(base_image)
@@ -172,7 +179,11 @@ def execute_dockerfile(args):
  dockerfile_parse = True
  # check if the dockerfile needs to be parsed
  if dockerfile_parse:
+ logger.debug('Parsing Dockerfile to generate report...')
  stub_image = get_dockerfile_packages()
+ report = content.print_full_report(base_image)
+ report = report + content.print_full_report(stub_image)
+ write_report(report)
  logger.debug('Cleaning up...')
  container.remove_image(full_image.repotag)
  cache.save()

utils/constants.py

@@ -25,3 +25,5 @@
 cache_file = 'cache.yml'
 # default shell
 shell = '/bin/sh'
+# report file
+report_file = 'report.txt'