Issue #39: Updated all Dagda external dependencies

dagda/analysis/runtime/docker_events_monitor.py

@@ -45,10 +45,10 @@ def run(self):
                 for event in self.docker_driver.docker_events():
                     e = json.loads(event.decode('UTF-8').replace("\n", ""))
                     if 'Actor' in e and 'Attributes' in e['Actor']:
-                        for key in e['Actor']['Attributes']:
+                        iter = list(e['Actor']['Attributes'])
+                        for key in iter:
                             if '.' in key:
                                 del e['Actor']['Attributes'][key]
-                                break
                     # Bulk insert
                     self.mongodb_driver.bulk_insert_docker_daemon_events([e])
             except requests.packages.urllib3.exceptions.ReadTimeoutError:

dagda/analysis/static/dependencies/dep_info_extractor.py

@@ -25,13 +25,14 @@
 
 # Gets programming languages dependencies from docker image
 def get_dependencies_from_docker_image(docker_driver, image_name, temp_dir):
+    # Init
+    filtered_image_name = image_name.replace(' ', '_').replace('/', '_').replace(':', '_')
     # Docker pull for ensuring the 3grander/4depcheck image
     docker_driver.docker_pull('3grander/4depcheck', '0.1.0')
     # Start container
     container_id = docker_driver.create_container('3grander/4depcheck:0.1.0',
-                                                  'python3 /opt/app/4depcheck.py ' +
-                                                    image_name.replace(' ', '_').replace('/', '_') + ' ' +
-                                                    temp_dir,
+                                                  'python3 /opt/app/4depcheck.py ' + filtered_image_name + ' ' +
+                                                   temp_dir,
                                                   [
                                                          temp_dir,
                                                          tempfile.gettempdir() + '/4depcheck'
@@ -47,7 +48,7 @@ def get_dependencies_from_docker_image(docker_driver, image_name, temp_dir):
     # Wait for 3grander/4depcheck
     docker_driver.docker_logs(container_id, True, False, True)
     # Get dependencies info
-    dependencies_info = json.loads(read_4depcheck_output_file(image_name))
+    dependencies_info = json.loads(read_4depcheck_output_file(filtered_image_name))
     # Stop container
     docker_driver.docker_stop(container_id)
     # Clean up
@@ -69,7 +70,6 @@ def get_filtered_dependencies_info(dependencies, temp_dir):
 
 # Reads the 4depcheck output file
 def read_4depcheck_output_file(image_name):
-    image_name = image_name.replace(' ', '_').replace('/', '_')
     filename = tempfile.gettempdir() + '/4depcheck/' + image_name + '.json'
 
     # Check file

dagda/analysis/static/util/utils.py

@@ -32,13 +32,14 @@ def extract_filesystem_bundle(docker_driver, container_id=None, image_name=None)
     temporary_dir = tempfile.mkdtemp()
     # Get and save filesystem bundle
     if container_id is not None:
-        data = docker_driver.get_docker_client().export(container=container_id).data
+        image = docker_driver.get_docker_client().export(container=container_id)
         name = container_id
     else:
-        data = docker_driver.get_docker_client().get_image(image=image_name).data
+        image = docker_driver.get_docker_client().get_image(image=image_name)
         name = image_name.replace('/', '_').replace(':', '_')
     with open(temporary_dir + "/" + name + ".tar", "wb") as file:
-        file.write(data)
+        for chunk in image:
+            file.write(chunk)
     # Untar filesystem bundle
     tarfile = TarFile(temporary_dir + "/" + name + ".tar")
     tarfile.extractall(temporary_dir)

requirements.txt

@@ -1,9 +1,9 @@
-pymongo==3.3.1
-requests==2.11.1
-python-dateutil==2.6.0
-joblib==0.10.3
-docker==2.2.0
-Flask==0.11.1
-flask-cors==3.0.2
-PyYAML==3.12
+pymongo==3.7.1
+requests==2.19.1
+python-dateutil==2.7.3
+joblib==0.12.5
+docker==3.5.0
+Flask==1.0.2
+flask-cors==3.0.6
+PyYAML==3.13
 defusedxml==0.5.0

tests/analysis/static/util/test_utils.py

@@ -76,24 +76,10 @@ def get_docker_client(self):
         return self
 
     def get_image(self, image):
-        return generate_dict(base64.b64decode(mock_image_filesystem_bundle_base64))
+        return [base64.b64decode(mock_image_filesystem_bundle_base64)]
 
     def export(self, container):
-        return generate_dict(base64.b64decode(mock_container_filesystem_bundle_base64))
-
-
-# -- Util methods
-
-def generate_dict(data):
-    return AttrDict([('data', data)])
-
-
-# -- Util classes
-
-class AttrDict(dict):
-    def __init__(self, *args, **kwargs):
-        super(AttrDict, self).__init__(*args, **kwargs)
-        self.__dict__ = self
+        return [base64.b64decode(mock_container_filesystem_bundle_base64)]
 
 
 # -- Mock Constants