Merge branch 'master' into ftp-c2

.github/workflows/security.yml

@@ -9,8 +9,6 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - python-version: 3.6
-            toxenv: safety
           - python-version: 3.7
             toxenv: safety
           - python-version: 3.8

.github/workflows/testing.yml

@@ -9,8 +9,6 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - python-version: 3.6
-            toxenv: py36,style,coverage-ci
           - python-version: 3.7
             toxenv: py37,style,coverage-ci
           - python-version: 3.8

app/api/v2/__init__.py

@@ -21,6 +21,9 @@ def make_app(services):
     from .handlers.planner_api import PlannerApi
     PlannerApi(services).add_routes(app)
 
+    from .handlers.ability_api import AbilityApi
+    AbilityApi(services).add_routes(app)
+
     from .handlers.plugins_api import PluginApi
     PluginApi(services).add_routes(app)
 

app/api/v2/handlers/ability_api.py

@@ -0,0 +1,64 @@
+import aiohttp_apispec
+from aiohttp import web
+
+from app.api.v2.handlers.base_object_api import BaseObjectApi
+from app.api.v2.managers.ability_api_manager import AbilityApiManager
+from app.api.v2.schemas.base_schemas import BaseGetAllQuerySchema, BaseGetOneQuerySchema
+from app.objects.c_ability import Ability, AbilitySchema
+
+
+class AbilityApi(BaseObjectApi):
+    def __init__(self, services):
+        super().__init__(description='ability', obj_class=Ability, schema=AbilitySchema, ram_key='abilities',
+                         id_property='ability_id', auth_svc=services['auth_svc'])
+        self._api_manager = AbilityApiManager(data_svc=services['data_svc'], file_svc=services['file_svc'])
+
+    def add_routes(self, app: web.Application):
+        router = app.router
+        router.add_get('/abilities', self.get_abilities)
+        router.add_get('/abilities/{ability_id}', self.get_ability_by_id)
+        router.add_post('/abilities', self.create_ability)
+        router.add_put('/abilities/{ability_id}', self.create_or_update_ability)
+        router.add_patch('/abilities/{ability_id}', self.update_ability)
+        router.add_delete('/abilities/{ability_id}', self.delete_ability)
+
+    @aiohttp_apispec.docs(tags=['abilities'])
+    @aiohttp_apispec.querystring_schema(BaseGetAllQuerySchema)
+    @aiohttp_apispec.response_schema(AbilitySchema(many=True, partial=True))
+    async def get_abilities(self, request: web.Request):
+        abilities = await self.get_all_objects(request)
+        return web.json_response(abilities)
+
+    @aiohttp_apispec.docs(tags=['abilities'])
+    @aiohttp_apispec.querystring_schema(BaseGetOneQuerySchema)
+    @aiohttp_apispec.response_schema(AbilitySchema(partial=True))
+    async def get_ability_by_id(self, request: web.Request):
+        ability = await self.get_object(request)
+        return web.json_response(ability)
+
+    @aiohttp_apispec.docs(tags=['abilities'])
+    @aiohttp_apispec.request_schema(AbilitySchema)
+    @aiohttp_apispec.response_schema(AbilitySchema)
+    async def create_ability(self, request: web.Request):
+        ability = await self.create_on_disk_object(request)
+        return web.json_response(ability.display)
+
+    @aiohttp_apispec.docs(tags=['abilities'])
+    @aiohttp_apispec.request_schema(AbilitySchema(partial=True))
+    @aiohttp_apispec.response_schema(AbilitySchema)
+    async def create_or_update_ability(self, request: web.Request):
+        ability = await self.create_or_update_on_disk_object(request)
+        return web.json_response(ability.display)
+
+    @aiohttp_apispec.docs(tags=['abilities'])
+    @aiohttp_apispec.request_schema(AbilitySchema(partial=True))
+    @aiohttp_apispec.response_schema(AbilitySchema)
+    async def update_ability(self, request: web.Request):
+        ability = await self.update_on_disk_object(request)
+        return web.json_response(ability.display)
+
+    @aiohttp_apispec.docs(tags=['abilities'])
+    @aiohttp_apispec.response_schema(AbilitySchema)
+    async def delete_ability(self, request: web.Request):
+        await self.delete_on_disk_object(request)
+        return web.HTTPNoContent()

app/api/v2/handlers/adversary_api.py

@@ -15,12 +15,13 @@ def __init__(self, services):
 
     def add_routes(self, app: web.Application):
         router = app.router
+        adversaries_by_id_path = '/adversaries/{adversary_id}'
         router.add_get('/adversaries', self.get_adversaries)
-        router.add_get('/adversaries/{adversary_id}', self.get_adversary_by_id)
+        router.add_get(adversaries_by_id_path, self.get_adversary_by_id)
         router.add_post('/adversaries', self.create_adversary)
-        router.add_patch('/adversaries/{adversary_id}', self.update_adversary)
-        router.add_put('/adversaries/{adversary_id}', self.create_or_update_adversary)
-        router.add_delete('/adversaries/{adversary_id}', self.delete_adversary)
+        router.add_patch(adversaries_by_id_path, self.update_adversary)
+        router.add_put(adversaries_by_id_path, self.create_or_update_adversary)
+        router.add_delete(adversaries_by_id_path, self.delete_adversary)
 
     @aiohttp_apispec.docs(tags=['adversaries'])
     @aiohttp_apispec.querystring_schema(BaseGetAllQuerySchema)

app/api/v2/managers/ability_api_manager.py

@@ -0,0 +1,93 @@
+import re
+import uuid
+import os
+import yaml
+
+from typing import Any
+
+from app.api.v2.managers.base_api_manager import BaseApiManager
+from app.api.v2.responses import JsonHttpBadRequest
+from app.objects.c_ability import AbilitySchema
+from app.utility.base_world import BaseWorld
+
+
+class AbilityApiManager(BaseApiManager):
+    def __init__(self, data_svc, file_svc):
+        super().__init__(data_svc=data_svc, file_svc=file_svc)
+
+    async def create_on_disk_object(self, data: dict, access: dict, ram_key: str, id_property: str, obj_class: type):
+        self._validate_ability_data(create=True, data=data)
+        obj_id = data.get('id')
+        file_path = self._create_ability_filepath(data.get('tactic'), obj_id)
+        allowed = self._get_allowed_from_access(access)
+        await self._save_and_reload_object(file_path, data, obj_class, allowed)
+        return next(self.find_objects(ram_key, {id_property: obj_id}))
+
+    async def replace_on_disk_object(self, obj: Any, data: dict, ram_key: str, id_property: str):
+        self._validate_ability_data(create=True, data=data)
+        obj_id = getattr(obj, id_property)
+        file_path = await self._get_existing_object_file_path(obj_id, ram_key)
+        if data.get('tactic') not in file_path:
+            await self.remove_object_from_disk_by_id(obj_id, ram_key)
+            file_path = self._create_ability_filepath(data.get('tactic'), obj_id)
+        await self._save_and_reload_object(file_path, data, type(obj), obj.access)
+        return next(self.find_objects(ram_key, {id_property: obj_id}))
+
+    async def update_on_disk_object(self, obj: Any, data: dict, ram_key: str, id_property: str, obj_class: type):
+        obj_id = getattr(obj, id_property)
+        file_path = await self._get_existing_object_file_path(obj_id, ram_key)
+        existing_obj_data = AbilitySchema().dump(obj)
+        existing_obj_data.update(data)
+        self._validate_ability_data(create=False, data=existing_obj_data)
+        if existing_obj_data.get('tactic') not in file_path:
+            await self.remove_object_from_disk_by_id(obj_id, ram_key)
+            file_path = self._create_ability_filepath(data.get('tactic'), obj_id)
+        await self._save_and_reload_object(file_path, existing_obj_data, obj_class, obj.access)
+        return next(self.find_objects(ram_key, {id_property: obj_id}))
+
+    '''Helpers'''
+    def _validate_ability_data(self, create: bool, data: dict):
+        # Correct ability_id key for ability file saving.
+        data['id'] = data.pop('ability_id', '')
+
+        # If a new ability is being created, ensure required fields present.
+        if create:
+            # Set ability ID if undefined
+            if not data['id']:
+                data['id'] = str(uuid.uuid4())
+            if not data.get('name'):
+                raise JsonHttpBadRequest(f'Cannot create ability {data["id"]} due to missing name')
+            if 'tactic' not in data:
+                raise JsonHttpBadRequest(f'Cannot create ability {data["id"]} due to missing tactic')
+            if not data.get('executors'):
+                raise JsonHttpBadRequest(f'Cannot create ability {data["id"]}: at least one executor required')
+        # Validate ID, used for file creation
+        validator = re.compile(r'^[a-zA-Z0-9-_]+$')
+        if 'id' in data and not validator.match(data['id']):
+            raise JsonHttpBadRequest(f'Invalid ability ID {data["id"]}. IDs can only contain '
+                                     'alphanumeric characters, hyphens, and underscores.')
+
+        # Validate tactic, used for directory creation, lower case if present
+        if 'tactic' in data:
+            if not validator.match(data['tactic']):
+                raise JsonHttpBadRequest(f'Invalid ability tactic {data["tactic"]}. Tactics can only contain '
+                                         'alphanumeric characters, hyphens, and underscores.')
+            data['tactic'] = data['tactic'].lower()
+
+        if 'executors' in data and not data.get('executors'):
+            raise JsonHttpBadRequest(f'Cannot create ability {data["id"]}: at least one executor required')
+
+        if 'name' in data and not data.get('name'):
+            raise JsonHttpBadRequest(f'Cannot create ability {data["id"]} due to missing name')
+
+    def _create_ability_filepath(self, tactic: str, obj_id: str):
+        tactic_dir = os.path.join('data', 'abilities', tactic)
+        if not os.path.exists(tactic_dir):
+            os.makedirs(tactic_dir)
+        return os.path.join(tactic_dir, '%s.yml' % obj_id)
+
+    async def _save_and_reload_object(self, file_path: str, data: dict, obj_type: type, access: BaseWorld.Access):
+        await self._file_svc.save_file(file_path, yaml.dump([data], encoding='utf-8', sort_keys=False),
+                                       '', encrypt=False)
+        await self._data_svc.remove('abilities', dict(ability_id=data['id']))
+        await self._data_svc.load_ability_file(file_path, access)