Merge branch 'master' into caldera-slack-c2-final

.github/workflows/security.yml

@@ -9,8 +9,6 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - python-version: 3.6
-            toxenv: safety
           - python-version: 3.7
             toxenv: safety
           - python-version: 3.8

.github/workflows/testing.yml

@@ -9,8 +9,6 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - python-version: 3.6
-            toxenv: py36,style,coverage-ci
           - python-version: 3.7
             toxenv: py37,style,coverage-ci
           - python-version: 3.8

app/api/v2/__init__.py

@@ -21,6 +21,9 @@ def make_app(services):
     from .handlers.planner_api import PlannerApi
     PlannerApi(services).add_routes(app)
 
+    from .handlers.ability_api import AbilityApi
+    AbilityApi(services).add_routes(app)
+
     from .handlers.plugins_api import PluginApi
     PluginApi(services).add_routes(app)
 

app/api/v2/handlers/ability_api.py

@@ -0,0 +1,64 @@
+import aiohttp_apispec
+from aiohttp import web
+
+from app.api.v2.handlers.base_object_api import BaseObjectApi
+from app.api.v2.managers.ability_api_manager import AbilityApiManager
+from app.api.v2.schemas.base_schemas import BaseGetAllQuerySchema, BaseGetOneQuerySchema
+from app.objects.c_ability import Ability, AbilitySchema
+
+
+class AbilityApi(BaseObjectApi):
+    def __init__(self, services):
+        super().__init__(description='ability', obj_class=Ability, schema=AbilitySchema, ram_key='abilities',
+                         id_property='ability_id', auth_svc=services['auth_svc'])
+        self._api_manager = AbilityApiManager(data_svc=services['data_svc'], file_svc=services['file_svc'])
+
+    def add_routes(self, app: web.Application):
+        router = app.router
+        router.add_get('/abilities', self.get_abilities)
+        router.add_get('/abilities/{ability_id}', self.get_ability_by_id)
+        router.add_post('/abilities', self.create_ability)
+        router.add_put('/abilities/{ability_id}', self.create_or_update_ability)
+        router.add_patch('/abilities/{ability_id}', self.update_ability)
+        router.add_delete('/abilities/{ability_id}', self.delete_ability)
+
+    @aiohttp_apispec.docs(tags=['abilities'])
+    @aiohttp_apispec.querystring_schema(BaseGetAllQuerySchema)
+    @aiohttp_apispec.response_schema(AbilitySchema(many=True, partial=True))
+    async def get_abilities(self, request: web.Request):
+        abilities = await self.get_all_objects(request)
+        return web.json_response(abilities)
+
+    @aiohttp_apispec.docs(tags=['abilities'])
+    @aiohttp_apispec.querystring_schema(BaseGetOneQuerySchema)
+    @aiohttp_apispec.response_schema(AbilitySchema(partial=True))
+    async def get_ability_by_id(self, request: web.Request):
+        ability = await self.get_object(request)
+        return web.json_response(ability)
+
+    @aiohttp_apispec.docs(tags=['abilities'])
+    @aiohttp_apispec.request_schema(AbilitySchema)
+    @aiohttp_apispec.response_schema(AbilitySchema)
+    async def create_ability(self, request: web.Request):
+        ability = await self.create_on_disk_object(request)
+        return web.json_response(ability.display)
+
+    @aiohttp_apispec.docs(tags=['abilities'])
+    @aiohttp_apispec.request_schema(AbilitySchema(partial=True))
+    @aiohttp_apispec.response_schema(AbilitySchema)
+    async def create_or_update_ability(self, request: web.Request):
+        ability = await self.create_or_update_on_disk_object(request)
+        return web.json_response(ability.display)
+
+    @aiohttp_apispec.docs(tags=['abilities'])
+    @aiohttp_apispec.request_schema(AbilitySchema(partial=True))
+    @aiohttp_apispec.response_schema(AbilitySchema)
+    async def update_ability(self, request: web.Request):
+        ability = await self.update_on_disk_object(request)
+        return web.json_response(ability.display)
+
+    @aiohttp_apispec.docs(tags=['abilities'])
+    @aiohttp_apispec.response_schema(AbilitySchema)
+    async def delete_ability(self, request: web.Request):
+        await self.delete_on_disk_object(request)
+        return web.HTTPNoContent()

app/api/v2/managers/ability_api_manager.py

@@ -0,0 +1,93 @@
+import re
+import uuid
+import os
+import yaml
+
+from typing import Any
+
+from app.api.v2.managers.base_api_manager import BaseApiManager
+from app.api.v2.responses import JsonHttpBadRequest
+from app.objects.c_ability import AbilitySchema
+from app.utility.base_world import BaseWorld
+
+
+class AbilityApiManager(BaseApiManager):
+    def __init__(self, data_svc, file_svc):
+        super().__init__(data_svc=data_svc, file_svc=file_svc)
+
+    async def create_on_disk_object(self, data: dict, access: dict, ram_key: str, id_property: str, obj_class: type):
+        self._validate_ability_data(create=True, data=data)
+        obj_id = data.get('id')
+        file_path = self._create_ability_filepath(data.get('tactic'), obj_id)
+        allowed = self._get_allowed_from_access(access)
+        await self._save_and_reload_object(file_path, data, obj_class, allowed)
+        return next(self.find_objects(ram_key, {id_property: obj_id}))
+
+    async def replace_on_disk_object(self, obj: Any, data: dict, ram_key: str, id_property: str):
+        self._validate_ability_data(create=True, data=data)
+        obj_id = getattr(obj, id_property)
+        file_path = await self._get_existing_object_file_path(obj_id, ram_key)
+        if data.get('tactic') not in file_path:
+            await self.remove_object_from_disk_by_id(obj_id, ram_key)
+            file_path = self._create_ability_filepath(data.get('tactic'), obj_id)
+        await self._save_and_reload_object(file_path, data, type(obj), obj.access)
+        return next(self.find_objects(ram_key, {id_property: obj_id}))
+
+    async def update_on_disk_object(self, obj: Any, data: dict, ram_key: str, id_property: str, obj_class: type):
+        obj_id = getattr(obj, id_property)
+        file_path = await self._get_existing_object_file_path(obj_id, ram_key)
+        existing_obj_data = AbilitySchema().dump(obj)
+        existing_obj_data.update(data)
+        self._validate_ability_data(create=False, data=existing_obj_data)
+        if existing_obj_data.get('tactic') not in file_path:
+            await self.remove_object_from_disk_by_id(obj_id, ram_key)
+            file_path = self._create_ability_filepath(data.get('tactic'), obj_id)
+        await self._save_and_reload_object(file_path, existing_obj_data, obj_class, obj.access)
+        return next(self.find_objects(ram_key, {id_property: obj_id}))
+
+    '''Helpers'''
+    def _validate_ability_data(self, create: bool, data: dict):
+        # Correct ability_id key for ability file saving.
+        data['id'] = data.pop('ability_id', '')
+
+        # If a new ability is being created, ensure required fields present.
+        if create:
+            # Set ability ID if undefined
+            if not data['id']:
+                data['id'] = str(uuid.uuid4())
+            if not data.get('name'):
+                raise JsonHttpBadRequest(f'Cannot create ability {data["id"]} due to missing name')
+            if 'tactic' not in data:
+                raise JsonHttpBadRequest(f'Cannot create ability {data["id"]} due to missing tactic')
+            if not data.get('executors'):
+                raise JsonHttpBadRequest(f'Cannot create ability {data["id"]}: at least one executor required')
+        # Validate ID, used for file creation
+        validator = re.compile(r'^[a-zA-Z0-9-_]+$')
+        if 'id' in data and not validator.match(data['id']):
+            raise JsonHttpBadRequest(f'Invalid ability ID {data["id"]}. IDs can only contain '
+                                     'alphanumeric characters, hyphens, and underscores.')
+
+        # Validate tactic, used for directory creation, lower case if present
+        if 'tactic' in data:
+            if not validator.match(data['tactic']):
+                raise JsonHttpBadRequest(f'Invalid ability tactic {data["tactic"]}. Tactics can only contain '
+                                         'alphanumeric characters, hyphens, and underscores.')
+            data['tactic'] = data['tactic'].lower()
+
+        if 'executors' in data and not data.get('executors'):
+            raise JsonHttpBadRequest(f'Cannot create ability {data["id"]}: at least one executor required')
+
+        if 'name' in data and not data.get('name'):
+            raise JsonHttpBadRequest(f'Cannot create ability {data["id"]} due to missing name')
+
+    def _create_ability_filepath(self, tactic: str, obj_id: str):
+        tactic_dir = os.path.join('data', 'abilities', tactic)
+        if not os.path.exists(tactic_dir):
+            os.makedirs(tactic_dir)
+        return os.path.join(tactic_dir, '%s.yml' % obj_id)
+
+    async def _save_and_reload_object(self, file_path: str, data: dict, obj_type: type, access: BaseWorld.Access):
+        await self._file_svc.save_file(file_path, yaml.dump([data], encoding='utf-8', sort_keys=False),
+                                       '', encrypt=False)
+        await self._data_svc.remove('abilities', dict(ability_id=data['id']))
+        await self._data_svc.load_ability_file(file_path, access)

app/objects/c_ability.py

@@ -19,7 +19,7 @@ class AbilitySchema(ma.Schema):
     technique_id = ma.fields.String(missing=None)
     name = ma.fields.String(missing=None)
     description = ma.fields.String(missing=None)
-    executors = ma.fields.List(ma.fields.Nested(ExecutorSchema), missing=None)
+    executors = ma.fields.List(ma.fields.Nested(ExecutorSchema))
     requirements = ma.fields.List(ma.fields.Nested(RequirementSchema), missing=None)
     privilege = ma.fields.String(missing=None)
     repeatable = ma.fields.Bool(missing=None)
@@ -28,11 +28,17 @@ class AbilitySchema(ma.Schema):
     access = ma.fields.Nested(AccessSchema, missing=None)
     singleton = ma.fields.Bool(missing=None)
 
+    @ma.pre_load
+    def fix_id(self, data, **_):
+        if 'id' in data:
+            data['ability_id'] = data.pop('id')
+        return data
+
     @ma.post_load
-    def build_ability(self, data, **_):
+    def build_ability(self, data, **kwargs):
         if 'technique' in data:
             data['technique_name'] = data.pop('technique')
-        return Ability(**data)
+        return None if kwargs.get('partial') is True else Ability(**data)
 
 
 class Ability(FirstClassObjectInterface, BaseObject):
@@ -93,6 +99,10 @@ def store(self, ram):
         existing.update('description', self.description)
         existing.update('_executor_map', self._executor_map)
         existing.update('privilege', self.privilege)
+        existing.update('repeatable', self.repeatable)
+        existing.update('buckets', self.buckets)
+        existing.update('tags', self.tags)
+        existing.update('singleton', self.singleton)
         return existing
 
     async def which_plugin(self):

app/service/data_svc.py

@@ -15,7 +15,7 @@
 from app.objects.c_planner import Planner
 from app.objects.c_plugin import Plugin
 from app.objects.c_source import Source
-from app.objects.secondclass.c_executor import Executor
+from app.objects.secondclass.c_executor import Executor, ExecutorSchema
 from app.objects.secondclass.c_goal import Goal
 from app.objects.secondclass.c_parser import Parser
 from app.objects.secondclass.c_requirement import Requirement
@@ -151,14 +151,15 @@ async def load_ability_file(self, filename, access):
                 name = ab.pop('name', '')
                 description = ab.pop('description', '')
                 tactic = ab.pop('tactic', None)
-                technique_id = ab.get('technique', dict()).get('attack_id')
-                technique_name = ab.pop('technique', dict()).get('name')
+                executors = await self.convert_v0_ability_executor(ab)
+                technique_id = self.convert_v0_ability_technique_id(ab)
+                technique_name = self.convert_v0_ability_technique_name(ab)
                 privilege = ab.pop('privilege', None)
                 repeatable = ab.pop('repeatable', False)
                 singleton = ab.pop('singleton', False)
                 requirements = await self._load_ability_requirements(ab.pop('requirements', []))
                 buckets = ab.pop('buckets', [tactic])
-                executors = await self.load_executors_from_platform_dict(ab.pop('platforms', dict()))
+                ab.pop('access', None)
 
                 if tactic and tactic not in filename:
                     self.log.error('Ability=%s has wrong tactic' % id)
@@ -169,6 +170,24 @@ async def load_ability_file(self, filename, access):
                                            repeatable=repeatable, buckets=buckets, access=access, singleton=singleton,
                                            **ab)
 
+    async def convert_v0_ability_executor(self, ability_data: dict):
+        """Checks if ability file follows v0 executor format, otherwise assumes v1 ability formatting."""
+        if 'platforms' in ability_data:
+            return await self.load_executors_from_platform_dict(ability_data.pop('platforms', dict()))
+        return await self.load_executors_from_list(ability_data.pop('executors', []))
+
+    def convert_v0_ability_technique_name(self, ability_data: dict):
+        """Checks if ability file follows v0 technique_name format, otherwise assumes v1 ability formatting."""
+        if 'technique' in ability_data:
+            return ability_data.pop('technique', dict()).get('name')
+        return ability_data.pop('technique_name')
+
+    def convert_v0_ability_technique_id(self, ability_data: dict):
+        """Checks if ability file follows v0 technique_id format, otherwise assumes v1 ability formatting."""
+        if 'technique' in ability_data:
+            return ability_data.get('technique', dict()).get('attack_id')
+        return ability_data.pop('technique_id')
+
     async def load_executors_from_platform_dict(self, platforms):
         executors = []
         for platform_names, platform_executors in platforms.items():
@@ -201,6 +220,9 @@ async def load_executors_from_platform_dict(self, platforms):
                                                   parsers=parsers, cleanup=cleanup, variations=variations))
         return executors
 
+    async def load_executors_from_list(self, executors: list):
+        return [ExecutorSchema().load(entry) for entry in executors]
+
     async def load_adversary_file(self, filename, access):
         warnings.warn("Function deprecated and will be removed in a future update. Use load_yaml_file", DeprecationWarning)
         await self.load_yaml_file(Adversary, filename, access)

app/service/file_svc.py

@@ -49,7 +49,7 @@ async def get_file(self, headers):
             if packer in self.packers:
                 file_path, contents = await self.get_payload_packer(packer).pack(file_path, contents)
             else:
-                self.log.warning('packer <%s> not available for payload <%s>, returning unpacked' % (packer, payload))
+                self.log.warning('packer not available for payload, returning unpacked')
         if headers.get('xor_key'):
             xor_key = headers['xor_key']
             contents = xor_bytes(contents, xor_key.encode())

app/service/rest_svc.py

@@ -617,7 +617,7 @@ async def _strip_parsers_from_ability(self, ability):
         Return ability (minus parsers) and parsers as seperate dict
         """
         parsers = {}
-        for platform, executors in ability['platforms'].items():
+        for platform, executors in ability.get('platforms', {}).items():
             parsers[platform] = {}
             for executor, d in executors.items():
                 if d.get('parsers', False):
@@ -630,7 +630,7 @@ async def _add_parsers_to_ability(self, ability, parsers):
         not an ability object but just the loaded dict from yaml
         ability file)
         """
-        for platform, executors in ability['platforms'].items():
+        for platform, executors in ability.get('platforms', {}).items():
             if parsers.get(platform, False):
                 for executor, _ in executors.items():
                     if parsers[platform].get(executor, False):