Added caching functions

- Removed from the README the TODO to use Docker API
The API does not support a user not in the docker group
and so assumes the docker commands be run without sudo
- Added support to convert the layer and package classes to
python dictionaries
- Added the cache util

Signed-off-by: Nisha K <[email protected]>

README.md

@@ -26,15 +26,18 @@ $ python tests/<test file>.py
 
 #### Phase 2:
 
-1. Replace the docker commands with docker API
-2. Knowledge base: Each layer hash should come with a list of known packages that are installed in that layer
-3. Allow for exceptions or additions for the command library
-4. In the reporting do not ignore packages that may be installed in the docker image - see bug 4
-5. Harden for testing within VMware's docker community
+1. Knowledge base: Each layer hash should come with a list of known packages that are installed in that layer
+2. Allow for exceptions or additions for the command library
+3. In the reporting do not ignore packages that may be installed in the docker image - see bug 4
+4. Harden for testing within VMware's docker community
 
 ### Bugs:
 1. Script assumes user is not in the docker group
 2. When a command fails within a container that package should be moved over to 'unrecognized'
 3. For reporting purposes - parse ENV
 4. Report should have 3 sections: confirmed, unconfirmed, unrecognized
 5. docker-command should raise exceptions that can be caught in demo for exiting
+
+### Upstream potential:
+1. Docker API assumes user is in docker group and hence can run docker commands without sudo
+2. Docker has no ability to step through docker history

cache.yml

@@ -1,10 +1 @@
-# The layer database akin to Docker's layerdb
-# This stores the sha of the image layer and the list of packages that came with it
-# This is a temporary cache implementation - perhaps a nosql database implementation later
-#layers:
-# - sha: '123abc'
-# packages:
-# - name:
-# version:
-# src_url:
-# license: 
+{}

classes/layer.py

@@ -5,7 +5,8 @@ class Layer(object):
  packages: list of objects of type Package (package.py)
  methods:
  add: adds a package to the layer
- remove: removes a package from the layer'''
+ remove: removes a package from the layer
+ to_dict: returns a dict representation of the instance'''
  def __init__(self, sha):
  self.__sha = sha
  self.__packages = []
@@ -32,3 +33,11 @@ def remove(self, package_name):
  if success:
  self.__packages.remove(self.__packages[rem_index])
  return success
+
+ def to_dict(self):
+ layer_dict = {}
+ pkg_list = []
+ for pkg in self.__packages:
+ pkg_list.append(pkg.to_dict())
+ layer_dict.update({self.sha: pkg_list})
+ return layer_dict

classes/package.py

@@ -4,7 +4,9 @@ class Package(object):
  name: package name
  version: package version
  license: package license
- src_url: package source url'''
+ src_url: package source url
+ methods:
+ to_dict: returns a dict representation of the instance'''
  def __init__(self, name):
  self.__name = name
  self.__version = 0.0
@@ -38,3 +40,11 @@ def src_url(self):
  @src_url.setter
  def src_url(self, src_url):
  self.__src_url = src_url
+
+ def to_dict(self):
+ pkg_dict = {}
+ pkg_dict.update({'name': self.name})
+ pkg_dict.update({'version': self.version})
+ pkg_dict.update({'license': self.license})
+ pkg_dict.update({'src_url': self.src_url})
+ return pkg_dict

tests/test_class_layer.py

@@ -30,5 +30,15 @@ def testRemovePackage(self):
  self.assertTrue(self.layer.remove('y'))
  self.assertFalse(self.layer.remove('y'))
 
+ def testToDict(self):
+ p1 = Package('x')
+ self.layer.add(p1)
+ a_dict = self.layer.to_dict()
+ print(a_dict)
+ self.assertTrue(a_dict['123abc'])
+ self.assertEqual(len(a_dict['123abc']), 1)
+ self.assertEqual(a_dict['123abc'][0]['name'], 'x')
+
+
 if __name__ == '__main__':
  unittest.main()

tests/test_class_package.py

@@ -35,6 +35,16 @@ def testGetters(self):
  self.assertEqual(self.package.license, 'Apache 2.0')
  self.assertEqual(self.package.src_url, 'github.com')
 
+ def testToDict(self):
+ self.package.version = 1.0
+ self.package.license = 'Apache 2.0'
+ self.package.src_url = 'github.com'
+ a_dict = self.package.to_dict()
+ self.assertEqual(a_dict['name'], 'x')
+ self.assertEqual(a_dict['version'], 1.0)
+ self.assertEqual(a_dict['license'], 'Apache 2.0')
+ self.assertEqual(a_dict['src_url'], 'github.com')
+
 
 if __name__ == '__main__':
  unittest.main()

utils/cache.py

@@ -2,35 +2,61 @@
 import yaml
 '''
 Docker image and layer related modules
+NOTE: the cache contains base image information
+currently there is no way to step through docker history
+So the assumption is that the base image is a flat image (which it is not)
+For now we will run commands within the whole base image based on the
+Dockerfile, but ideally we need to step through the base image history and
+find the actual base image
 '''
 
-# known layer database
-layer_db_file = 'layer_db.yml'
-layer_db = {}
-with open(os.path.abspath(layer_db_file)) as f:
- layer_db = yaml.safe_load(f)
-
-
-def check_trusted(image, tag='latest'):
- '''Check to see if the given image is trusted
- Return a string: 'image:tag'
- Else raise a NameError
- WARNING: if a Dockerfile does not specify a tag, Docker will pull down
- the tag marked 'latest'. The list of trusted images should be kept up to
- date on the tag 'latest' is pointing to. We assume that the images
- being produced from the Dockerfile pulling from the 'latest' tag are
- taken care of by the maintainer of the images'''
- ret_string = ''
- # list of trusted images
- images = trusted.keys()
- if image in images:
- # list of trusted tags
- if tag == 'latest':
- ret_string = image + ':' + trusted[image]['latest']
- elif tag in trusted[image]['tags']:
- ret_string = image + ':' + tag
- else:
- raise NameError('Not a trusted Docker tag for image: ' + image)
+# known base image database
+cache_file = 'cache.yml'
+cache = {}
+
+
+def load():
+ '''Load the cache'''
+ with open(os.path.abspath(cache_file)) as f:
+ global cache
+ cache = yaml.load(f)
+ if cache is not None:
+ global cache
+ cache = {}
+
+
+def get_packages(sha):
+ '''Given an image sha retrieve cache record. If none return an empty list'''
+ if sha in cache.keys():
+ return cache[sha]
  else:
- raise NameError('Not a trusted Docker image: ' + image)
- return ret_string
+ return []
+
+
+def add_layer(layer_obj):
+ '''Given a layer object, add it to the cache'''
+ cache.update(layer_obj.to_dict())
+
+
+def save():
+ '''Given a layer object record it into the cache'''
+ with open(os.path.abspath(cache_file), 'w') as f:
+ yaml.dump(cache, f, default_flow_style=False)
+
+
+def remove_layer(sha):
+ '''Remove from cache the layer represented by the sha'''
+ success = False
+ if sha in cache.keys():
+ del cache[sha]
+ success = True
+ return success
+
+
+def clear():
+ '''Empty the cache - don't use unless you really have to_dict'''
+ global cache
+ cache = {}
+ with open(os.path.abspath(cache_file), 'w') as f:
+ global cache
+ yaml.dump(cache, f, default_flow_style=False)