From b05e8eb752b01fa62bb6b8d99e50d378a987a950 Mon Sep 17 00:00:00 2001 From: blee Date: Thu, 3 Jun 2021 16:21:45 -0400 Subject: [PATCH 01/58] create abilities file w/ routes and function stubs --- app/api/v2/__init__.py | 3 ++ app/api/v2/handlers/ability_api.py | 74 ++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 app/api/v2/handlers/ability_api.py diff --git a/app/api/v2/__init__.py b/app/api/v2/__init__.py index ff5e7ffa2..faefbc160 100644 --- a/app/api/v2/__init__.py +++ b/app/api/v2/__init__.py @@ -21,4 +21,7 @@ def make_app(services): from .handlers.planner_api import PlannerApi PlannerApi(services).add_routes(app) + from .handlers.ability_api import AbilityApi + AbilityApi(services).add_routes(app) + return app diff --git a/app/api/v2/handlers/ability_api.py b/app/api/v2/handlers/ability_api.py new file mode 100644 index 000000000..e29369d52 --- /dev/null +++ b/app/api/v2/handlers/ability_api.py @@ -0,0 +1,74 @@ +import aiohttp_apispec +from aiohttp import web + +from app.api.v2.handlers.base_api import BaseApi +from app.api.v2.managers.base_api_manager import BaseApiManager +from app.api.v2.responses import JsonHttpNotFound +from app.api.v2.schemas.base_schemas import BaseGetAllQuerySchema, BaseGetOneQuerySchema +from app.objects.c_ability import AbilitySchema + + +class AbilityApi(BaseApi): + def __init__(self, services): + super().__init__(auth_svc=services['auth_svc']) + self._api_manager = BaseApiManager(data_svc=services['data_svc']) + + def add_routes(self, app: web.Application): + router = app.router + router.add_get('/abilities', self.get_abilities) + router.add_get('/abilities/{ability_id}', self.get_ability_by_id) + router.add_post('/abilities', self.post_abilities) + router.add_put('/abilities/{ability_id}', self.put_ability_by_id) + router.add_patch('/abilities/{ability_id}', self.patch_ability_by_id) + router.add_delete('/abilities/{ability_id}', self.delete_ability_by_id) + + @aiohttp_apispec.docs(tags=['abilities']) + @aiohttp_apispec.querystring_schema(BaseGetAllQuerySchema) + @aiohttp_apispec.response_schema(AbilitySchema(many=True)) + async def get_abilities(self, request: web.Request): + sort = request['querystring'].get('sort', 'name') + include = request['querystring'].get('include') + exclude = request['querystring'].get('exclude') + + abilities = self._api_manager.get_objects_with_filters('abilities', sort=sort, include=include, exclude=exclude) + return web.json_response(abilities) + + @aiohttp_apispec.docs(tags=['abilities']) + @aiohttp_apispec.querystring_schema(BaseGetOneQuerySchema) + @aiohttp_apispec.response_schema(AbilitySchema) + async def get_ability_by_id(self, request: web.Request): + ability_id = request.match_info['ability_id'] + include = request['querystring'].get('include') + exclude = request['querystring'].get('exclude') + + search = dict(ability_id=ability_id) + + ability = self._api_manager.get_object_with_filters('abilities', search=search, include=include, exclude=exclude) + if not ability: + raise JsonHttpNotFound(f'Planner not found: {ability_id}') + + return web.json_response(ability) + + @aiohttp_apispec.docs(tags=['abilities']) + @aiohttp_apispec.querystring_schema(BaseGetAllQuerySchema) + @aiohttp_apispec.response_schema(AbilitySchema(many=True)) + async def post_abilities(self, request: web.Request): + pass + + @aiohttp_apispec.docs(tags=['abilities']) + @aiohttp_apispec.querystring_schema(BaseGetOneQuerySchema) + @aiohttp_apispec.response_schema(AbilitySchema) + async def put_ability_by_id(self, request: web.Request): + pass + + @aiohttp_apispec.docs(tags=['abilities']) + @aiohttp_apispec.querystring_schema(BaseGetOneQuerySchema) + @aiohttp_apispec.response_schema(AbilitySchema) + async def patch_ability_by_id(self, request: web.Request): + pass + + @aiohttp_apispec.docs(tags=['abilities']) + @aiohttp_apispec.querystring_schema(BaseGetOneQuerySchema) + @aiohttp_apispec.response_schema(AbilitySchema) + async def delete_ability_by_id(self, request: web.Request): + pass From 6fc2c97af7269aa4745f33aa7331b4400b2041af Mon Sep 17 00:00:00 2001 From: blee Date: Fri, 4 Jun 2021 14:10:24 -0400 Subject: [PATCH 02/58] Add implementation for delete and put --- app/api/v2/handlers/ability_api.py | 37 +++++++++++++++++++++---- app/api/v2/managers/base_api_manager.py | 10 +++++++ 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/app/api/v2/handlers/ability_api.py b/app/api/v2/handlers/ability_api.py index e29369d52..2a68878a4 100644 --- a/app/api/v2/handlers/ability_api.py +++ b/app/api/v2/handlers/ability_api.py @@ -17,7 +17,7 @@ def add_routes(self, app: web.Application): router = app.router router.add_get('/abilities', self.get_abilities) router.add_get('/abilities/{ability_id}', self.get_ability_by_id) - router.add_post('/abilities', self.post_abilities) + router.add_post('/abilities', self.create_abilities) router.add_put('/abilities/{ability_id}', self.put_ability_by_id) router.add_patch('/abilities/{ability_id}', self.patch_ability_by_id) router.add_delete('/abilities/{ability_id}', self.delete_ability_by_id) @@ -45,30 +45,57 @@ async def get_ability_by_id(self, request: web.Request): ability = self._api_manager.get_object_with_filters('abilities', search=search, include=include, exclude=exclude) if not ability: - raise JsonHttpNotFound(f'Planner not found: {ability_id}') + raise JsonHttpNotFound(f'Ability not found: {ability_id}') return web.json_response(ability) @aiohttp_apispec.docs(tags=['abilities']) @aiohttp_apispec.querystring_schema(BaseGetAllQuerySchema) @aiohttp_apispec.response_schema(AbilitySchema(many=True)) - async def post_abilities(self, request: web.Request): + async def create_abilities(self, request: web.Request): pass @aiohttp_apispec.docs(tags=['abilities']) @aiohttp_apispec.querystring_schema(BaseGetOneQuerySchema) @aiohttp_apispec.response_schema(AbilitySchema) async def put_ability_by_id(self, request: web.Request): - pass + ability_id = request.match_info['ability_id'] + include = request['querystring'].get('include') + exclude = request['querystring'].get('exclude') + + search = dict(ability_id=ability_id) + + ability = self._api_manager.get_object_with_filters('abilities', search=search, include=include, + exclude=exclude) + if not ability: + ability_data = await request.json() + ability = self._api_manager.store_json_as_schema(AbilitySchema, ability_data) + return web.json_response(ability.display) + else: + params = {} + self._api_manager.update_object('abilities', parameters=params) + return web.Response(status=200) @aiohttp_apispec.docs(tags=['abilities']) @aiohttp_apispec.querystring_schema(BaseGetOneQuerySchema) @aiohttp_apispec.response_schema(AbilitySchema) async def patch_ability_by_id(self, request: web.Request): + # Check if ability exists + # If ability exists, update fields. + # Else, return error. pass @aiohttp_apispec.docs(tags=['abilities']) @aiohttp_apispec.querystring_schema(BaseGetOneQuerySchema) @aiohttp_apispec.response_schema(AbilitySchema) async def delete_ability_by_id(self, request: web.Request): - pass + ability_id = request.match_info['ability_id'] + + search = dict(ability_id=ability_id) + + ability = self._api_manager.get_object_with_filters('abilities', search=search) + if not ability: + raise JsonHttpNotFound(f'Ability not found: {ability_id}') + + self._api_manager.delete_object('abilities', search=search) + return web.Response(status=204) diff --git a/app/api/v2/managers/base_api_manager.py b/app/api/v2/managers/base_api_manager.py index e5964d959..ebbdb01c3 100644 --- a/app/api/v2/managers/base_api_manager.py +++ b/app/api/v2/managers/base_api_manager.py @@ -26,6 +26,16 @@ def get_object_with_filters(self, object_name: str, search: dict = None, include if not search or obj.match(search): return self.dump_with_include_exclude(obj, include, exclude) + def delete_object(self, object_name: str, search: dict = None): + for obj in self._data_svc.ram[object_name]: + if not search or obj.match(search): + data = self._data_svc.ram[object_name].copy() + data.remove(obj) + self._data_svc.ram[object_name] = data + + def update_object(self, object_name: str, parameters: dict): + pass + @staticmethod def dump_with_include_exclude(obj, include: List[str] = None, exclude: List[str] = None): dumped = obj.display From 6d7092026a03c173cd0cab861950ae84fbcca206 Mon Sep 17 00:00:00 2001 From: blee Date: Tue, 8 Jun 2021 10:40:46 -0400 Subject: [PATCH 03/58] add access to get requests, remove edits from base_api_manager --- app/api/v2/handlers/ability_api.py | 11 +++++++++-- app/api/v2/managers/base_api_manager.py | 10 ---------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/app/api/v2/handlers/ability_api.py b/app/api/v2/handlers/ability_api.py index 2a68878a4..afc295e75 100644 --- a/app/api/v2/handlers/ability_api.py +++ b/app/api/v2/handlers/ability_api.py @@ -30,7 +30,10 @@ async def get_abilities(self, request: web.Request): include = request['querystring'].get('include') exclude = request['querystring'].get('exclude') - abilities = self._api_manager.get_objects_with_filters('abilities', sort=sort, include=include, exclude=exclude) + access = await self.get_request_permissions(request) + + abilities = self._api_manager.get_objects_with_filters('abilities', search=access, sort=sort, + include=include, exclude=exclude) return web.json_response(abilities) @aiohttp_apispec.docs(tags=['abilities']) @@ -41,7 +44,9 @@ async def get_ability_by_id(self, request: web.Request): include = request['querystring'].get('include') exclude = request['querystring'].get('exclude') - search = dict(ability_id=ability_id) + access = await self.get_request_permissions(request) + query = dict(ability_id=ability_id) + search = {**query, **access} ability = self._api_manager.get_object_with_filters('abilities', search=search, include=include, exclude=exclude) if not ability: @@ -53,12 +58,14 @@ async def get_ability_by_id(self, request: web.Request): @aiohttp_apispec.querystring_schema(BaseGetAllQuerySchema) @aiohttp_apispec.response_schema(AbilitySchema(many=True)) async def create_abilities(self, request: web.Request): + # data = await request.json() pass @aiohttp_apispec.docs(tags=['abilities']) @aiohttp_apispec.querystring_schema(BaseGetOneQuerySchema) @aiohttp_apispec.response_schema(AbilitySchema) async def put_ability_by_id(self, request: web.Request): + # data = await request.json() ability_id = request.match_info['ability_id'] include = request['querystring'].get('include') exclude = request['querystring'].get('exclude') diff --git a/app/api/v2/managers/base_api_manager.py b/app/api/v2/managers/base_api_manager.py index ebbdb01c3..e5964d959 100644 --- a/app/api/v2/managers/base_api_manager.py +++ b/app/api/v2/managers/base_api_manager.py @@ -26,16 +26,6 @@ def get_object_with_filters(self, object_name: str, search: dict = None, include if not search or obj.match(search): return self.dump_with_include_exclude(obj, include, exclude) - def delete_object(self, object_name: str, search: dict = None): - for obj in self._data_svc.ram[object_name]: - if not search or obj.match(search): - data = self._data_svc.ram[object_name].copy() - data.remove(obj) - self._data_svc.ram[object_name] = data - - def update_object(self, object_name: str, parameters: dict): - pass - @staticmethod def dump_with_include_exclude(obj, include: List[str] = None, exclude: List[str] = None): dumped = obj.display From 9598d369678e69591d1fc22d2f530c353a3ca4ca Mon Sep 17 00:00:00 2001 From: blee Date: Tue, 8 Jun 2021 15:02:14 -0400 Subject: [PATCH 04/58] actually implement delete, create ability api manager --- app/api/v2/handlers/ability_api.py | 28 ++++++++++---------- app/api/v2/managers/ability_api_manager.py | 30 ++++++++++++++++++++++ 2 files changed, 44 insertions(+), 14 deletions(-) create mode 100644 app/api/v2/managers/ability_api_manager.py diff --git a/app/api/v2/handlers/ability_api.py b/app/api/v2/handlers/ability_api.py index afc295e75..dbec7f108 100644 --- a/app/api/v2/handlers/ability_api.py +++ b/app/api/v2/handlers/ability_api.py @@ -2,7 +2,7 @@ from aiohttp import web from app.api.v2.handlers.base_api import BaseApi -from app.api.v2.managers.base_api_manager import BaseApiManager +from app.api.v2.managers.ability_api_manager import AbilityApiManager from app.api.v2.responses import JsonHttpNotFound from app.api.v2.schemas.base_schemas import BaseGetAllQuerySchema, BaseGetOneQuerySchema from app.objects.c_ability import AbilitySchema @@ -11,7 +11,7 @@ class AbilityApi(BaseApi): def __init__(self, services): super().__init__(auth_svc=services['auth_svc']) - self._api_manager = BaseApiManager(data_svc=services['data_svc']) + self._api_manager = AbilityApiManager(data_svc=services['data_svc'], rest_svc=services['rest_svc']) def add_routes(self, app: web.Application): router = app.router @@ -48,7 +48,8 @@ async def get_ability_by_id(self, request: web.Request): query = dict(ability_id=ability_id) search = {**query, **access} - ability = self._api_manager.get_object_with_filters('abilities', search=search, include=include, exclude=exclude) + ability = self._api_manager.get_object_with_filters('abilities', search=search, include=include, + exclude=exclude) if not ability: raise JsonHttpNotFound(f'Ability not found: {ability_id}') @@ -58,8 +59,10 @@ async def get_ability_by_id(self, request: web.Request): @aiohttp_apispec.querystring_schema(BaseGetAllQuerySchema) @aiohttp_apispec.response_schema(AbilitySchema(many=True)) async def create_abilities(self, request: web.Request): - # data = await request.json() - pass + ability_list = await request.json() + source = self._api_manager.create_abilities(ability_list) + + return web.json_response(source.display) @aiohttp_apispec.docs(tags=['abilities']) @aiohttp_apispec.querystring_schema(BaseGetOneQuerySchema) @@ -75,13 +78,9 @@ async def put_ability_by_id(self, request: web.Request): ability = self._api_manager.get_object_with_filters('abilities', search=search, include=include, exclude=exclude) if not ability: - ability_data = await request.json() - ability = self._api_manager.store_json_as_schema(AbilitySchema, ability_data) - return web.json_response(ability.display) + pass else: - params = {} - self._api_manager.update_object('abilities', parameters=params) - return web.Response(status=200) + pass @aiohttp_apispec.docs(tags=['abilities']) @aiohttp_apispec.querystring_schema(BaseGetOneQuerySchema) @@ -98,11 +97,12 @@ async def patch_ability_by_id(self, request: web.Request): async def delete_ability_by_id(self, request: web.Request): ability_id = request.match_info['ability_id'] - search = dict(ability_id=ability_id) + ability_dict = dict(ability_id=ability_id) - ability = self._api_manager.get_object_with_filters('abilities', search=search) + ability = self._api_manager.get_object_with_filters('abilities', search=ability_dict) if not ability: raise JsonHttpNotFound(f'Ability not found: {ability_id}') - self._api_manager.delete_object('abilities', search=search) + await self._api_manager.delete_ability(ability_dict) + return web.Response(status=204) diff --git a/app/api/v2/managers/ability_api_manager.py b/app/api/v2/managers/ability_api_manager.py new file mode 100644 index 000000000..8426f8fb4 --- /dev/null +++ b/app/api/v2/managers/ability_api_manager.py @@ -0,0 +1,30 @@ +# from app.api.v2 import validation +from app.api.v2.managers.base_api_manager import BaseApiManager + + +class AbilityUpdateNotAllowed(Exception): + def __init__(self, property, message=None): + super().__init__(message or f'Updating ability property is disallowed: {property}') + self.property = property + + +class AbilityNotFound(Exception): + def __init__(self, ability_id, message=None): + super().__init__(message or f'Ability not found: {ability_id}') + self.ability_id = ability_id + + +class AbilityApiManager(BaseApiManager): + def __init__(self, data_svc, rest_svc): + super().__init__(data_svc=data_svc) + self._rest_svc = rest_svc + + def create_abilities(self, ability_list): + pass + + def update_ability(self, prop, value): + pass + + async def delete_ability(self, ability_id): + result = await self._rest_svc.delete_ability(ability_id) + return result From e11de506ab8c32f8973d4d6c15a3d306a355af5a Mon Sep 17 00:00:00 2001 From: blee Date: Wed, 9 Jun 2021 09:48:26 -0400 Subject: [PATCH 05/58] implement post handler --- app/api/v2/handlers/ability_api.py | 6 ++++-- app/api/v2/managers/ability_api_manager.py | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/app/api/v2/handlers/ability_api.py b/app/api/v2/handlers/ability_api.py index dbec7f108..ef7eb8e82 100644 --- a/app/api/v2/handlers/ability_api.py +++ b/app/api/v2/handlers/ability_api.py @@ -60,9 +60,11 @@ async def get_ability_by_id(self, request: web.Request): @aiohttp_apispec.response_schema(AbilitySchema(many=True)) async def create_abilities(self, request: web.Request): ability_list = await request.json() - source = self._api_manager.create_abilities(ability_list) + access = await self.get_request_permissions(request) + + source = await self._api_manager.create_abilities(access=access, ability_list=ability_list) - return web.json_response(source.display) + return web.json_response(source) @aiohttp_apispec.docs(tags=['abilities']) @aiohttp_apispec.querystring_schema(BaseGetOneQuerySchema) diff --git a/app/api/v2/managers/ability_api_manager.py b/app/api/v2/managers/ability_api_manager.py index 8426f8fb4..46eb71175 100644 --- a/app/api/v2/managers/ability_api_manager.py +++ b/app/api/v2/managers/ability_api_manager.py @@ -19,8 +19,9 @@ def __init__(self, data_svc, rest_svc): super().__init__(data_svc=data_svc) self._rest_svc = rest_svc - def create_abilities(self, ability_list): - pass + async def create_abilities(self, access, ability_list): + data = dict(bulk=ability_list) + return await self._rest_svc.persist_ability(access=access, data=data) def update_ability(self, prop, value): pass From d83116377e2382a09b910463d15b3940027964bc Mon Sep 17 00:00:00 2001 From: blee Date: Thu, 10 Jun 2021 08:20:16 -0400 Subject: [PATCH 06/58] Update post json response --- app/api/v2/handlers/ability_api.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/app/api/v2/handlers/ability_api.py b/app/api/v2/handlers/ability_api.py index ef7eb8e82..f1039fc9a 100644 --- a/app/api/v2/handlers/ability_api.py +++ b/app/api/v2/handlers/ability_api.py @@ -64,7 +64,7 @@ async def create_abilities(self, request: web.Request): source = await self._api_manager.create_abilities(access=access, ability_list=ability_list) - return web.json_response(source) + return web.json_response(source, status=201) @aiohttp_apispec.docs(tags=['abilities']) @aiohttp_apispec.querystring_schema(BaseGetOneQuerySchema) @@ -98,10 +98,9 @@ async def patch_ability_by_id(self, request: web.Request): @aiohttp_apispec.response_schema(AbilitySchema) async def delete_ability_by_id(self, request: web.Request): ability_id = request.match_info['ability_id'] - ability_dict = dict(ability_id=ability_id) - ability = self._api_manager.get_object_with_filters('abilities', search=ability_dict) + if not ability: raise JsonHttpNotFound(f'Ability not found: {ability_id}') From d5b75b500593960117c05a9ba33963b17fa4bff2 Mon Sep 17 00:00:00 2001 From: blee Date: Thu, 10 Jun 2021 13:19:11 -0400 Subject: [PATCH 07/58] Use new base object api functions to implement endpoints --- app/api/v2/handlers/ability_api.py | 104 ++++++--------------- app/api/v2/managers/ability_api_manager.py | 31 ++---- app/objects/c_ability.py | 3 + 3 files changed, 40 insertions(+), 98 deletions(-) diff --git a/app/api/v2/handlers/ability_api.py b/app/api/v2/handlers/ability_api.py index f1039fc9a..8af45176a 100644 --- a/app/api/v2/handlers/ability_api.py +++ b/app/api/v2/handlers/ability_api.py @@ -1,109 +1,67 @@ import aiohttp_apispec from aiohttp import web -from app.api.v2.handlers.base_api import BaseApi +from app.api.v2.handlers.base_object_api import BaseObjectApi from app.api.v2.managers.ability_api_manager import AbilityApiManager -from app.api.v2.responses import JsonHttpNotFound from app.api.v2.schemas.base_schemas import BaseGetAllQuerySchema, BaseGetOneQuerySchema -from app.objects.c_ability import AbilitySchema +from app.objects.c_ability import Ability, AbilitySchema -class AbilityApi(BaseApi): +class AbilityApi(BaseObjectApi): def __init__(self, services): - super().__init__(auth_svc=services['auth_svc']) - self._api_manager = AbilityApiManager(data_svc=services['data_svc'], rest_svc=services['rest_svc']) + super().__init__(description='ability', obj_class=Ability, schema=AbilitySchema, ram_key='abilities', + id_property='ability_id', auth_svc=services['auth_svc']) + self._api_manager = AbilityApiManager(data_svc=services['data_svc'], file_svc=services['file_svc']) def add_routes(self, app: web.Application): router = app.router router.add_get('/abilities', self.get_abilities) router.add_get('/abilities/{ability_id}', self.get_ability_by_id) - router.add_post('/abilities', self.create_abilities) - router.add_put('/abilities/{ability_id}', self.put_ability_by_id) - router.add_patch('/abilities/{ability_id}', self.patch_ability_by_id) - router.add_delete('/abilities/{ability_id}', self.delete_ability_by_id) + router.add_post('/abilities', self.create_ability) + router.add_put('/abilities/{ability_id}', self.create_or_update_ability) + router.add_patch('/abilities/{ability_id}', self.update_ability) + router.add_delete('/abilities/{ability_id}', self.delete_ability) @aiohttp_apispec.docs(tags=['abilities']) @aiohttp_apispec.querystring_schema(BaseGetAllQuerySchema) @aiohttp_apispec.response_schema(AbilitySchema(many=True)) async def get_abilities(self, request: web.Request): - sort = request['querystring'].get('sort', 'name') - include = request['querystring'].get('include') - exclude = request['querystring'].get('exclude') - - access = await self.get_request_permissions(request) - - abilities = self._api_manager.get_objects_with_filters('abilities', search=access, sort=sort, - include=include, exclude=exclude) + abilities = await self.get_all_objects(request) return web.json_response(abilities) @aiohttp_apispec.docs(tags=['abilities']) @aiohttp_apispec.querystring_schema(BaseGetOneQuerySchema) @aiohttp_apispec.response_schema(AbilitySchema) async def get_ability_by_id(self, request: web.Request): - ability_id = request.match_info['ability_id'] - include = request['querystring'].get('include') - exclude = request['querystring'].get('exclude') - - access = await self.get_request_permissions(request) - query = dict(ability_id=ability_id) - search = {**query, **access} - - ability = self._api_manager.get_object_with_filters('abilities', search=search, include=include, - exclude=exclude) - if not ability: - raise JsonHttpNotFound(f'Ability not found: {ability_id}') - + ability = await self.get_object(request) return web.json_response(ability) @aiohttp_apispec.docs(tags=['abilities']) - @aiohttp_apispec.querystring_schema(BaseGetAllQuerySchema) - @aiohttp_apispec.response_schema(AbilitySchema(many=True)) - async def create_abilities(self, request: web.Request): - ability_list = await request.json() - access = await self.get_request_permissions(request) - - source = await self._api_manager.create_abilities(access=access, ability_list=ability_list) - - return web.json_response(source, status=201) + @aiohttp_apispec.request_schema(AbilitySchema) + @aiohttp_apispec.response_schema(AbilitySchema) + async def create_ability(self, request: web.Request): + ability = await self.create_or_update_on_disk_object(request) + ability = await self._api_manager.verify_ability(ability) + return web.json_response(ability.display) @aiohttp_apispec.docs(tags=['abilities']) - @aiohttp_apispec.querystring_schema(BaseGetOneQuerySchema) + @aiohttp_apispec.request_schema(AbilitySchema) @aiohttp_apispec.response_schema(AbilitySchema) - async def put_ability_by_id(self, request: web.Request): - # data = await request.json() - ability_id = request.match_info['ability_id'] - include = request['querystring'].get('include') - exclude = request['querystring'].get('exclude') - - search = dict(ability_id=ability_id) - - ability = self._api_manager.get_object_with_filters('abilities', search=search, include=include, - exclude=exclude) - if not ability: - pass - else: - pass + async def create_or_update_ability(self, request: web.Request): + ability = await self.create_or_update_on_disk_object(request) + ability = await self._api_manager.verify_ability(ability) + return web.json_response(ability) @aiohttp_apispec.docs(tags=['abilities']) - @aiohttp_apispec.querystring_schema(BaseGetOneQuerySchema) + @aiohttp_apispec.request_schema(AbilitySchema) @aiohttp_apispec.response_schema(AbilitySchema) - async def patch_ability_by_id(self, request: web.Request): - # Check if ability exists - # If ability exists, update fields. - # Else, return error. - pass + async def update_ability(self, request: web.Request): + ability = await self.update_on_disk_object(request) + ability = await self._api_manager.verify_ability(ability) + return web.json_response(ability.display) @aiohttp_apispec.docs(tags=['abilities']) - @aiohttp_apispec.querystring_schema(BaseGetOneQuerySchema) @aiohttp_apispec.response_schema(AbilitySchema) - async def delete_ability_by_id(self, request: web.Request): - ability_id = request.match_info['ability_id'] - ability_dict = dict(ability_id=ability_id) - ability = self._api_manager.get_object_with_filters('abilities', search=ability_dict) - - if not ability: - raise JsonHttpNotFound(f'Ability not found: {ability_id}') - - await self._api_manager.delete_ability(ability_dict) - - return web.Response(status=204) + async def delete_ability(self, request: web.Request): + await self.delete_on_disk_object(request) + return web.HTTPNoContent() diff --git a/app/api/v2/managers/ability_api_manager.py b/app/api/v2/managers/ability_api_manager.py index 46eb71175..1c2833eb8 100644 --- a/app/api/v2/managers/ability_api_manager.py +++ b/app/api/v2/managers/ability_api_manager.py @@ -1,31 +1,12 @@ -# from app.api.v2 import validation from app.api.v2.managers.base_api_manager import BaseApiManager - - -class AbilityUpdateNotAllowed(Exception): - def __init__(self, property, message=None): - super().__init__(message or f'Updating ability property is disallowed: {property}') - self.property = property - - -class AbilityNotFound(Exception): - def __init__(self, ability_id, message=None): - super().__init__(message or f'Ability not found: {ability_id}') - self.ability_id = ability_id +from app.objects.c_ability import Ability class AbilityApiManager(BaseApiManager): - def __init__(self, data_svc, rest_svc): + def __init__(self, data_svc, file_svc): super().__init__(data_svc=data_svc) - self._rest_svc = rest_svc - - async def create_abilities(self, access, ability_list): - data = dict(bulk=ability_list) - return await self._rest_svc.persist_ability(access=access, data=data) - - def update_ability(self, prop, value): - pass + self._file_svc = file_svc - async def delete_ability(self, ability_id): - result = await self._rest_svc.delete_ability(ability_id) - return result + async def verify_ability(self, ability: Ability): + ability.verify(log=self.log) + return ability diff --git a/app/objects/c_ability.py b/app/objects/c_ability.py index 9c41e5f7a..baebf44b1 100644 --- a/app/objects/c_ability.py +++ b/app/objects/c_ability.py @@ -154,6 +154,9 @@ async def add_bucket(self, bucket): if bucket not in self.buckets: self.buckets.append(bucket) + def verify(self, log): + pass + @staticmethod def _make_executor_map_key(name, platform): return name, platform From 1339b548ad3529a02db0b313a0cc6582ce8abf98 Mon Sep 17 00:00:00 2001 From: blee Date: Thu, 10 Jun 2021 15:52:25 -0400 Subject: [PATCH 08/58] update ability schema --- app/api/v2/handlers/ability_api.py | 8 ++++---- app/api/v2/managers/ability_api_manager.py | 3 +-- app/objects/c_ability.py | 12 +++++++++--- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/app/api/v2/handlers/ability_api.py b/app/api/v2/handlers/ability_api.py index 8af45176a..fb2e05c71 100644 --- a/app/api/v2/handlers/ability_api.py +++ b/app/api/v2/handlers/ability_api.py @@ -24,14 +24,14 @@ def add_routes(self, app: web.Application): @aiohttp_apispec.docs(tags=['abilities']) @aiohttp_apispec.querystring_schema(BaseGetAllQuerySchema) - @aiohttp_apispec.response_schema(AbilitySchema(many=True)) + @aiohttp_apispec.response_schema(AbilitySchema(many=True, partial=True)) async def get_abilities(self, request: web.Request): abilities = await self.get_all_objects(request) return web.json_response(abilities) @aiohttp_apispec.docs(tags=['abilities']) @aiohttp_apispec.querystring_schema(BaseGetOneQuerySchema) - @aiohttp_apispec.response_schema(AbilitySchema) + @aiohttp_apispec.response_schema(AbilitySchema(partial=True)) async def get_ability_by_id(self, request: web.Request): ability = await self.get_object(request) return web.json_response(ability) @@ -45,7 +45,7 @@ async def create_ability(self, request: web.Request): return web.json_response(ability.display) @aiohttp_apispec.docs(tags=['abilities']) - @aiohttp_apispec.request_schema(AbilitySchema) + @aiohttp_apispec.request_schema(AbilitySchema(partial=True)) @aiohttp_apispec.response_schema(AbilitySchema) async def create_or_update_ability(self, request: web.Request): ability = await self.create_or_update_on_disk_object(request) @@ -53,7 +53,7 @@ async def create_or_update_ability(self, request: web.Request): return web.json_response(ability) @aiohttp_apispec.docs(tags=['abilities']) - @aiohttp_apispec.request_schema(AbilitySchema) + @aiohttp_apispec.request_schema(AbilitySchema(partial=True)) @aiohttp_apispec.response_schema(AbilitySchema) async def update_ability(self, request: web.Request): ability = await self.update_on_disk_object(request) diff --git a/app/api/v2/managers/ability_api_manager.py b/app/api/v2/managers/ability_api_manager.py index 1c2833eb8..a40ff8df7 100644 --- a/app/api/v2/managers/ability_api_manager.py +++ b/app/api/v2/managers/ability_api_manager.py @@ -4,8 +4,7 @@ class AbilityApiManager(BaseApiManager): def __init__(self, data_svc, file_svc): - super().__init__(data_svc=data_svc) - self._file_svc = file_svc + super().__init__(data_svc=data_svc, file_svc=file_svc) async def verify_ability(self, ability: Ability): ability.verify(log=self.log) diff --git a/app/objects/c_ability.py b/app/objects/c_ability.py index baebf44b1..99e766e71 100644 --- a/app/objects/c_ability.py +++ b/app/objects/c_ability.py @@ -12,7 +12,7 @@ class AbilitySchema(ma.Schema): - ability_id = ma.fields.String() + ability_id = ma.fields.String(required=False) tactic = ma.fields.String(missing=None) technique_name = ma.fields.String(missing=None) technique_id = ma.fields.String(missing=None) @@ -27,11 +27,17 @@ class AbilitySchema(ma.Schema): access = ma.fields.Nested(AccessSchema, missing=None) singleton = ma.fields.Bool(missing=None) + @ma.pre_load + def fix_id(self, ability, **_): + if 'id' in ability: + ability['ability_id'] = ability.pop('id') + return ability + @ma.post_load - def build_ability(self, data, **_): + def build_ability(self, data, **kwargs): if 'technique' in data: data['technique_name'] = data.pop('technique') - return Ability(**data) + return None if kwargs.get('partial') is True else Ability(**data) class Ability(FirstClassObjectInterface, BaseObject): From 6718522c77d1ffa5c00d0e309440435caa5e7ee2 Mon Sep 17 00:00:00 2001 From: blee Date: Fri, 11 Jun 2021 09:31:21 -0400 Subject: [PATCH 09/58] add verification to Ability class --- app/objects/c_ability.py | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/app/objects/c_ability.py b/app/objects/c_ability.py index 99e766e71..94eae898f 100644 --- a/app/objects/c_ability.py +++ b/app/objects/c_ability.py @@ -1,5 +1,7 @@ import collections import os +import re +import uuid import marshmallow as ma @@ -12,13 +14,13 @@ class AbilitySchema(ma.Schema): - ability_id = ma.fields.String(required=False) - tactic = ma.fields.String(missing=None) + ability_id = ma.fields.String() + tactic = ma.fields.String(required=True) technique_name = ma.fields.String(missing=None) technique_id = ma.fields.String(missing=None) name = ma.fields.String(missing=None) description = ma.fields.String(missing=None) - executors = ma.fields.List(ma.fields.Nested(ExecutorSchema), missing=None) + executors = ma.fields.List(ma.fields.Nested(ExecutorSchema), required=True) requirements = ma.fields.List(ma.fields.Nested(RequirementSchema), missing=None) privilege = ma.fields.String(missing=None) repeatable = ma.fields.Bool(missing=None) @@ -55,7 +57,7 @@ def unique(self): def executors(self): yield from self._executor_map.values() - def __init__(self, ability_id, name=None, description=None, tactic=None, technique_id=None, technique_name=None, + def __init__(self, ability_id='', name=None, description=None, tactic=None, technique_id=None, technique_name=None, executors=(), requirements=None, privilege=None, repeatable=False, buckets=None, access=None, additional_info=None, tags=None, singleton=False, **kwargs): super().__init__() @@ -161,7 +163,24 @@ async def add_bucket(self, bucket): self.buckets.append(bucket) def verify(self, log): - pass + # Set ability ID if undefined + if not self.ability_id: + self.ability_id = str(uuid.uuid4()) + # Validate ID, used for file creation + validator = re.compile(r'^[a-zA-Z0-9-_]+$') + if not self.ability_id or not validator.match(self.ability_id): + self.log.debug('Invalid ability ID "%s". IDs can only contain ' + 'alphanumeric characters, hyphens, and underscores.' % self.ability_id) + + # Validate tactic, used for directory creation, lower case if present + if not self.tactic or not validator.match(self.tactic): + self.log.debug('Invalid ability tactic "%s". Tactics can only contain ' + 'alphanumeric characters, hyphens, and underscores.' % self.tactic) + self.tactic = self.tactic.lower() + + # Validate platforms, ability will not be loaded if empty + if not self._executor_map: + self.log.debug('At least one executor is required to save ability.') @staticmethod def _make_executor_map_key(name, platform): From bc941b6add939e7a94eaa4abf8c633064ec6e3c5 Mon Sep 17 00:00:00 2001 From: blee Date: Fri, 11 Jun 2021 11:53:02 -0400 Subject: [PATCH 10/58] Add validation before writing to disk --- app/api/v2/handlers/ability_api.py | 7 +--- app/api/v2/managers/ability_api_manager.py | 45 ++++++++++++++++++++++ app/objects/c_ability.py | 4 +- 3 files changed, 49 insertions(+), 7 deletions(-) diff --git a/app/api/v2/handlers/ability_api.py b/app/api/v2/handlers/ability_api.py index fb2e05c71..b95c36b9b 100644 --- a/app/api/v2/handlers/ability_api.py +++ b/app/api/v2/handlers/ability_api.py @@ -40,8 +40,7 @@ async def get_ability_by_id(self, request: web.Request): @aiohttp_apispec.request_schema(AbilitySchema) @aiohttp_apispec.response_schema(AbilitySchema) async def create_ability(self, request: web.Request): - ability = await self.create_or_update_on_disk_object(request) - ability = await self._api_manager.verify_ability(ability) + ability = await self.create_on_disk_object(request) return web.json_response(ability.display) @aiohttp_apispec.docs(tags=['abilities']) @@ -49,15 +48,13 @@ async def create_ability(self, request: web.Request): @aiohttp_apispec.response_schema(AbilitySchema) async def create_or_update_ability(self, request: web.Request): ability = await self.create_or_update_on_disk_object(request) - ability = await self._api_manager.verify_ability(ability) - return web.json_response(ability) + return web.json_response(ability.display) @aiohttp_apispec.docs(tags=['abilities']) @aiohttp_apispec.request_schema(AbilitySchema(partial=True)) @aiohttp_apispec.response_schema(AbilitySchema) async def update_ability(self, request: web.Request): ability = await self.update_on_disk_object(request) - ability = await self._api_manager.verify_ability(ability) return web.json_response(ability.display) @aiohttp_apispec.docs(tags=['abilities']) diff --git a/app/api/v2/managers/ability_api_manager.py b/app/api/v2/managers/ability_api_manager.py index a40ff8df7..f8b7219e1 100644 --- a/app/api/v2/managers/ability_api_manager.py +++ b/app/api/v2/managers/ability_api_manager.py @@ -1,4 +1,10 @@ +import re +import uuid + +from typing import Any + from app.api.v2.managers.base_api_manager import BaseApiManager +from app.api.v2.responses import JsonHttpBadRequest from app.objects.c_ability import Ability @@ -9,3 +15,42 @@ def __init__(self, data_svc, file_svc): async def verify_ability(self, ability: Ability): ability.verify(log=self.log) return ability + + def validate_ability_data(self, create: bool, data: dict): + # If a new ability is being created, ensure required fields present. + if create: + # Set ability ID if undefined + if 'ability_id' not in data: + data['ability_id'] = str(uuid.uuid4()) + if 'tactic' not in data: + raise JsonHttpBadRequest(f'Cannot create ability {data["ability_id"]} due to missing tactic') + if not data['executors']: + raise JsonHttpBadRequest(f'Cannot create ability {data["ability_id"]}: at least one executor required') + # Validate ID, used for file creation + validator = re.compile(r'^[a-zA-Z0-9-_]+$') + if 'ability_id' in data and not validator.match(data['ability_id']): + raise JsonHttpBadRequest(f'Invalid ability ID {data["ability_id"]}. IDs can only contain ' + 'alphanumeric characters, hyphens, and underscores.') + + # Validate tactic, used for directory creation, lower case if present + if 'tactic' in data: + if not validator.match(data['tactic']): + raise JsonHttpBadRequest(f'Invalid ability tactic {data["tactic"]}. Tactics can only contain ' + 'alphanumeric characters, hyphens, and underscores.') + data['tactic'] = data['tactic'].lower() + + # Validate platforms, ability will not be loaded if empty + if 'executors' in data and not data['executors']: + raise JsonHttpBadRequest('At least one executor is required to save ability.') + + async def create_on_disk_object(self, data: dict, access: dict, ram_key: str, id_property: str, obj_class: type): + self.validate_ability_data(create=True, data=data) + return await super().create_on_disk_object(data, access, ram_key, id_property, obj_class) + + async def replace_on_disk_object(self, obj: Any, data: dict, ram_key: str, id_property: str): + self.validate_ability_data(create=False, data=data) + return await super().replace_on_disk_object(obj, data, ram_key, id_property) + + async def update_on_disk_object(self, obj: Any, data: dict, ram_key: str, id_property: str, obj_class: type): + self.validate_ability_data(create=False, data=data) + return await super().update_on_disk_object(obj, data, ram_key, id_property, obj_class) diff --git a/app/objects/c_ability.py b/app/objects/c_ability.py index 94eae898f..aaf7b7b86 100644 --- a/app/objects/c_ability.py +++ b/app/objects/c_ability.py @@ -15,12 +15,12 @@ class AbilitySchema(ma.Schema): ability_id = ma.fields.String() - tactic = ma.fields.String(required=True) + tactic = ma.fields.String() technique_name = ma.fields.String(missing=None) technique_id = ma.fields.String(missing=None) name = ma.fields.String(missing=None) description = ma.fields.String(missing=None) - executors = ma.fields.List(ma.fields.Nested(ExecutorSchema), required=True) + executors = ma.fields.List(ma.fields.Nested(ExecutorSchema)) requirements = ma.fields.List(ma.fields.Nested(RequirementSchema), missing=None) privilege = ma.fields.String(missing=None) repeatable = ma.fields.Bool(missing=None) From 54960f65fa6eb5bdfb58f44eb858273ceef0f198 Mon Sep 17 00:00:00 2001 From: blee Date: Fri, 11 Jun 2021 13:06:59 -0400 Subject: [PATCH 11/58] Remove unused verify function from ability object --- app/api/v2/managers/ability_api_manager.py | 5 ----- app/objects/c_ability.py | 26 ++-------------------- 2 files changed, 2 insertions(+), 29 deletions(-) diff --git a/app/api/v2/managers/ability_api_manager.py b/app/api/v2/managers/ability_api_manager.py index f8b7219e1..2d254e84a 100644 --- a/app/api/v2/managers/ability_api_manager.py +++ b/app/api/v2/managers/ability_api_manager.py @@ -5,17 +5,12 @@ from app.api.v2.managers.base_api_manager import BaseApiManager from app.api.v2.responses import JsonHttpBadRequest -from app.objects.c_ability import Ability class AbilityApiManager(BaseApiManager): def __init__(self, data_svc, file_svc): super().__init__(data_svc=data_svc, file_svc=file_svc) - async def verify_ability(self, ability: Ability): - ability.verify(log=self.log) - return ability - def validate_ability_data(self, create: bool, data: dict): # If a new ability is being created, ensure required fields present. if create: diff --git a/app/objects/c_ability.py b/app/objects/c_ability.py index aaf7b7b86..f1530d844 100644 --- a/app/objects/c_ability.py +++ b/app/objects/c_ability.py @@ -1,7 +1,5 @@ import collections import os -import re -import uuid import marshmallow as ma @@ -15,12 +13,12 @@ class AbilitySchema(ma.Schema): ability_id = ma.fields.String() - tactic = ma.fields.String() + tactic = ma.fields.String(required=True) technique_name = ma.fields.String(missing=None) technique_id = ma.fields.String(missing=None) name = ma.fields.String(missing=None) description = ma.fields.String(missing=None) - executors = ma.fields.List(ma.fields.Nested(ExecutorSchema)) + executors = ma.fields.List(ma.fields.Nested(ExecutorSchema), required=True) requirements = ma.fields.List(ma.fields.Nested(RequirementSchema), missing=None) privilege = ma.fields.String(missing=None) repeatable = ma.fields.Bool(missing=None) @@ -162,26 +160,6 @@ async def add_bucket(self, bucket): if bucket not in self.buckets: self.buckets.append(bucket) - def verify(self, log): - # Set ability ID if undefined - if not self.ability_id: - self.ability_id = str(uuid.uuid4()) - # Validate ID, used for file creation - validator = re.compile(r'^[a-zA-Z0-9-_]+$') - if not self.ability_id or not validator.match(self.ability_id): - self.log.debug('Invalid ability ID "%s". IDs can only contain ' - 'alphanumeric characters, hyphens, and underscores.' % self.ability_id) - - # Validate tactic, used for directory creation, lower case if present - if not self.tactic or not validator.match(self.tactic): - self.log.debug('Invalid ability tactic "%s". Tactics can only contain ' - 'alphanumeric characters, hyphens, and underscores.' % self.tactic) - self.tactic = self.tactic.lower() - - # Validate platforms, ability will not be loaded if empty - if not self._executor_map: - self.log.debug('At least one executor is required to save ability.') - @staticmethod def _make_executor_map_key(name, platform): return name, platform From 2562e2a4ef6062f790446339a27a950073889365 Mon Sep 17 00:00:00 2001 From: blee Date: Fri, 11 Jun 2021 13:30:12 -0400 Subject: [PATCH 12/58] update validation parameters --- app/api/v2/managers/ability_api_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/v2/managers/ability_api_manager.py b/app/api/v2/managers/ability_api_manager.py index 2d254e84a..68f407dcd 100644 --- a/app/api/v2/managers/ability_api_manager.py +++ b/app/api/v2/managers/ability_api_manager.py @@ -43,7 +43,7 @@ async def create_on_disk_object(self, data: dict, access: dict, ram_key: str, id return await super().create_on_disk_object(data, access, ram_key, id_property, obj_class) async def replace_on_disk_object(self, obj: Any, data: dict, ram_key: str, id_property: str): - self.validate_ability_data(create=False, data=data) + self.validate_ability_data(create=True, data=data) return await super().replace_on_disk_object(obj, data, ram_key, id_property) async def update_on_disk_object(self, obj: Any, data: dict, ram_key: str, id_property: str, obj_class: type): From 5a1bbefa60560753daf34d8015e6fadad17ab85f Mon Sep 17 00:00:00 2001 From: blee Date: Fri, 11 Jun 2021 13:41:43 -0400 Subject: [PATCH 13/58] Fix ability required field error in tests --- tests/objects/test_agent.py | 14 +++++++------- tests/services/test_planning_svc.py | 4 ++-- tests/services/test_rest_svc.py | 6 +++--- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/objects/test_agent.py b/tests/objects/test_agent.py index 48ef41b88..99a09fabe 100644 --- a/tests/objects/test_agent.py +++ b/tests/objects/test_agent.py @@ -10,21 +10,21 @@ class TestAgent: def test_task_no_facts(self, loop, data_svc, obfuscator, init_base_world): executor = Executor(name='psh', platform='windows', command='whoami') - ability = Ability(ability_id='123', executors=[executor]) + ability = Ability(ability_id='123', tactic='test', executors=[executor]) agent = Agent(paw='123', sleep_min=2, sleep_max=8, watchdog=0, executors=['pwsh', 'psh'], platform='windows') loop.run_until_complete(agent.task([ability], obfuscator='plain-text')) assert 1 == len(agent.links) def test_task_missing_fact(self, loop, obfuscator, init_base_world): executor = Executor(name='psh', platform='windows', command='net user #{domain.user.name} /domain') - ability = Ability(ability_id='123', executors=[executor]) + ability = Ability(ability_id='123', tactic='test', executors=[executor]) agent = Agent(paw='123', sleep_min=2, sleep_max=8, watchdog=0, executors=['pwsh', 'psh'], platform='windows') loop.run_until_complete(agent.task([ability], obfuscator='plain-text')) assert 0 == len(agent.links) def test_task_with_facts(self, loop, obfuscator, init_base_world): executor = Executor(name='psh', platform='windows', command='net user #{domain.user.name} /domain') - ability = Ability(ability_id='123', executors=[executor]) + ability = Ability(ability_id='123', tactic='test', executors=[executor]) agent = Agent(paw='123', sleep_min=2, sleep_max=8, watchdog=0, executors=['pwsh', 'psh'], platform='windows') fact = Fact(trait='domain.user.name', value='bob') @@ -34,7 +34,7 @@ def test_task_with_facts(self, loop, obfuscator, init_base_world): def test_builtin_fact_replacement(self, loop, obfuscator, init_base_world): executor = Executor(name='psh', platform='windows', command='echo #{paw} #{server} #{group} #{location} #{exe_name} #{upstream_dest}') - ability = Ability(ability_id='123', executors=[executor]) + ability = Ability(ability_id='123', tactic='test', executors=[executor]) agent = Agent(paw='123', sleep_min=2, sleep_max=8, watchdog=0, executors=['pwsh', 'psh'], platform='windows', group='my_group', server='http://localhost:8888', location='testlocation', exe_name='testexe') loop.run_until_complete(agent.task([ability], 'plain-text', [])) @@ -47,7 +47,7 @@ def test_builtin_fact_replacement(self, loop, obfuscator, init_base_world): def test_builtin_fact_replacement_with_upstream_dest(self, loop, obfuscator, init_base_world): executor = Executor(name='psh', platform='windows', command='echo #{paw} #{server} #{group} #{location} #{exe_name} #{upstream_dest}') - ability = Ability(ability_id='123', executors=[executor]) + ability = Ability(ability_id='123', tactic='test', executors=[executor]) agent = Agent(paw='123', sleep_min=2, sleep_max=8, watchdog=0, executors=['pwsh', 'psh'], platform='windows', group='my_group', server='http://10.10.10.10:8888', location='testlocation', exe_name='testexe', upstream_dest='http://127.0.0.1:12345') @@ -62,7 +62,7 @@ def test_preferred_executor_psh(self, loop, ability, executor): executor_test = executor(name='test', platform='windows') executor_cmd = executor(name='cmd', platform='windows') executor_psh = executor(name='psh', platform='windows') - test_ability = ability(ability_id='123', executors=[executor_test, executor_cmd, executor_psh]) + test_ability = ability(ability_id='123', tactic='test', executors=[executor_test, executor_cmd, executor_psh]) agent = Agent(paw='123', sleep_min=2, sleep_max=8, watchdog=0, executors=['psh', 'cmd'], platform='windows') @@ -73,7 +73,7 @@ def test_preferred_executor_from_agent_executor(self, loop, ability, executor): executor_test = executor(name='test', platform='windows') executor_cmd = executor(name='cmd', platform='windows') executor_psh = executor(name='psh', platform='windows') - test_ability = ability(ability_id='123', executors=[executor_test, executor_cmd, executor_psh]) + test_ability = ability(ability_id='123', tactic='test', executors=[executor_test, executor_cmd, executor_psh]) agent = Agent(paw='123', sleep_min=2, sleep_max=8, watchdog=0, executors=['cmd', 'test'], platform='windows') diff --git a/tests/services/test_planning_svc.py b/tests/services/test_planning_svc.py index 730cbd67d..7d0fa28a8 100644 --- a/tests/services/test_planning_svc.py +++ b/tests/services/test_planning_svc.py @@ -61,7 +61,7 @@ def __init__(self, **kwargs): @pytest.fixture def setup_planning_test(loop, executor, ability, agent, operation, data_svc, event_svc, init_base_world): texecutor = executor(name='sh', platform='darwin', command='mkdir test', cleanup='rm -rf test') - tability = ability(ability_id='123', executors=[texecutor], repeatable=True, buckets=['test']) + tability = ability(ability_id='123', tactic='test', executors=[texecutor], repeatable=True, buckets=['test']) tagent = agent(sleep_min=1, sleep_max=2, watchdog=0, executors=['sh'], platform='darwin', server='http://127.0.0.1:8000') tsource = Source(id='123', name='test', facts=[], adjustments=[]) toperation = operation(name='test1', agents=[tagent], @@ -71,7 +71,7 @@ def setup_planning_test(loop, executor, ability, agent, operation, data_svc, eve source=tsource) cexecutor = executor(name='sh', platform='darwin', command=test_string, cleanup='whoami') - cability = ability(ability_id='321', executors=[cexecutor], singleton=True) + cability = ability(ability_id='321', tactic='test', executors=[cexecutor], singleton=True) loop.run_until_complete(data_svc.store(tability)) loop.run_until_complete(data_svc.store(cability)) diff --git a/tests/services/test_rest_svc.py b/tests/services/test_rest_svc.py index d3f8bbb9b..103c1f209 100644 --- a/tests/services/test_rest_svc.py +++ b/tests/services/test_rest_svc.py @@ -22,17 +22,17 @@ def setup_rest_svc_test(loop, data_svc): 'encryption_key': 'ADMIN123', 'exfil_dir': '/tmp'}) loop.run_until_complete(data_svc.store( - Ability(ability_id='123', executors=[ + Ability(ability_id='123', tactic='test', executors=[ Executor(name='psh', platform='windows', command='curl #{app.contact.http}') ]) )) loop.run_until_complete(data_svc.store( - Ability(ability_id='456', executors=[ + Ability(ability_id='456', tactic='test', executors=[ Executor(name='sh', platform='linux', command='whoami') ]) )) loop.run_until_complete(data_svc.store( - Ability(ability_id='789', executors=[ + Ability(ability_id='789', tactic='test', executors=[ Executor(name='sh', platform='linux', command='hostname') ]) )) From bcc54c3b35d46c5a38f600812650f51f71d0fff1 Mon Sep 17 00:00:00 2001 From: blee Date: Wed, 16 Jun 2021 16:08:03 -0400 Subject: [PATCH 14/58] update verification parameters for ability verification --- app/api/v2/managers/ability_api_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/v2/managers/ability_api_manager.py b/app/api/v2/managers/ability_api_manager.py index 68f407dcd..2d254e84a 100644 --- a/app/api/v2/managers/ability_api_manager.py +++ b/app/api/v2/managers/ability_api_manager.py @@ -43,7 +43,7 @@ async def create_on_disk_object(self, data: dict, access: dict, ram_key: str, id return await super().create_on_disk_object(data, access, ram_key, id_property, obj_class) async def replace_on_disk_object(self, obj: Any, data: dict, ram_key: str, id_property: str): - self.validate_ability_data(create=True, data=data) + self.validate_ability_data(create=False, data=data) return await super().replace_on_disk_object(obj, data, ram_key, id_property) async def update_on_disk_object(self, obj: Any, data: dict, ram_key: str, id_property: str, obj_class: type): From 58420db771352adfc5a5aa301ded6d5a97c96f19 Mon Sep 17 00:00:00 2001 From: blee Date: Mon, 28 Jun 2021 18:37:32 -0400 Subject: [PATCH 15/58] update ability writes to disk --- app/api/v2/managers/ability_api_manager.py | 38 ++++++++++++++-------- app/objects/c_ability.py | 9 +++-- 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/app/api/v2/managers/ability_api_manager.py b/app/api/v2/managers/ability_api_manager.py index 2d254e84a..db8847e59 100644 --- a/app/api/v2/managers/ability_api_manager.py +++ b/app/api/v2/managers/ability_api_manager.py @@ -1,5 +1,6 @@ import re import uuid +import os from typing import Any @@ -11,7 +12,30 @@ class AbilityApiManager(BaseApiManager): def __init__(self, data_svc, file_svc): super().__init__(data_svc=data_svc, file_svc=file_svc) - def validate_ability_data(self, create: bool, data: dict): + async def create_on_disk_object(self, data: dict, access: dict, ram_key: str, id_property: str, obj_class: type): + self._validate_ability_data(create=True, data=data) + obj_id = data.get(id_property) or str(uuid.uuid4()) + data[id_property] = obj_id + + tactic_dir = os.path.join('data', 'abilities', data.get('tactic')) + if not os.path.exists(tactic_dir): + os.makedirs(tactic_dir) + file_path = os.path.join(tactic_dir, '%s.yml' % data['ability_id']) + allowed = self._get_allowed_from_access(access) + await self._save_and_reload_object(file_path, data, obj_class, allowed) + return next(self.find_objects(ram_key, {id_property: obj_id})) + + async def replace_on_disk_object(self, obj: Any, data: dict, ram_key: str, id_property: str): + self._validate_ability_data(create=False, data=data) + return await super().replace_on_disk_object(obj, data, ram_key, id_property) + + async def update_on_disk_object(self, obj: Any, data: dict, ram_key: str, id_property: str, obj_class: type): + self._validate_ability_data(create=False, data=data) + return await super().update_on_disk_object(obj, data, ram_key, id_property, obj_class) + + '''Helpers''' + + def _validate_ability_data(self, create: bool, data: dict): # If a new ability is being created, ensure required fields present. if create: # Set ability ID if undefined @@ -37,15 +61,3 @@ def validate_ability_data(self, create: bool, data: dict): # Validate platforms, ability will not be loaded if empty if 'executors' in data and not data['executors']: raise JsonHttpBadRequest('At least one executor is required to save ability.') - - async def create_on_disk_object(self, data: dict, access: dict, ram_key: str, id_property: str, obj_class: type): - self.validate_ability_data(create=True, data=data) - return await super().create_on_disk_object(data, access, ram_key, id_property, obj_class) - - async def replace_on_disk_object(self, obj: Any, data: dict, ram_key: str, id_property: str): - self.validate_ability_data(create=False, data=data) - return await super().replace_on_disk_object(obj, data, ram_key, id_property) - - async def update_on_disk_object(self, obj: Any, data: dict, ram_key: str, id_property: str, obj_class: type): - self.validate_ability_data(create=False, data=data) - return await super().update_on_disk_object(obj, data, ram_key, id_property, obj_class) diff --git a/app/objects/c_ability.py b/app/objects/c_ability.py index f1530d844..28f2c5753 100644 --- a/app/objects/c_ability.py +++ b/app/objects/c_ability.py @@ -13,12 +13,12 @@ class AbilitySchema(ma.Schema): ability_id = ma.fields.String() - tactic = ma.fields.String(required=True) + tactic = ma.fields.String(missing=None) technique_name = ma.fields.String(missing=None) technique_id = ma.fields.String(missing=None) name = ma.fields.String(missing=None) description = ma.fields.String(missing=None) - executors = ma.fields.List(ma.fields.Nested(ExecutorSchema), required=True) + executors = ma.fields.List(ma.fields.Nested(ExecutorSchema), missing=None) requirements = ma.fields.List(ma.fields.Nested(RequirementSchema), missing=None) privilege = ma.fields.String(missing=None) repeatable = ma.fields.Bool(missing=None) @@ -27,6 +27,11 @@ class AbilitySchema(ma.Schema): access = ma.fields.Nested(AccessSchema, missing=None) singleton = ma.fields.Bool(missing=None) + @ma.pre_load + def fix_tactic(self, ability, **_): + if 'tactic' in ability: + ability['tactic'] = ability['tactic'].lower() + @ma.pre_load def fix_id(self, ability, **_): if 'id' in ability: From b43a5468d100c5566e361644c0ea2f6d72cd8f54 Mon Sep 17 00:00:00 2001 From: blee Date: Tue, 29 Jun 2021 12:43:09 -0400 Subject: [PATCH 16/58] update data svc to handle executor lists for loading abilities --- app/api/v2/managers/ability_api_manager.py | 12 +++++- app/objects/c_ability.py | 6 +-- app/service/data_svc.py | 43 ++++++++++++++++++++-- 3 files changed, 52 insertions(+), 9 deletions(-) diff --git a/app/api/v2/managers/ability_api_manager.py b/app/api/v2/managers/ability_api_manager.py index db8847e59..54ce25c94 100644 --- a/app/api/v2/managers/ability_api_manager.py +++ b/app/api/v2/managers/ability_api_manager.py @@ -1,11 +1,13 @@ import re import uuid import os +import yaml from typing import Any from app.api.v2.managers.base_api_manager import BaseApiManager from app.api.v2.responses import JsonHttpBadRequest +from app.utility.base_world import BaseWorld class AbilityApiManager(BaseApiManager): @@ -21,8 +23,7 @@ async def create_on_disk_object(self, data: dict, access: dict, ram_key: str, id if not os.path.exists(tactic_dir): os.makedirs(tactic_dir) file_path = os.path.join(tactic_dir, '%s.yml' % data['ability_id']) - allowed = self._get_allowed_from_access(access) - await self._save_and_reload_object(file_path, data, obj_class, allowed) + await self._save_and_reload_object(file_path, data, obj_class, access) return next(self.find_objects(ram_key, {id_property: obj_id})) async def replace_on_disk_object(self, obj: Any, data: dict, ram_key: str, id_property: str): @@ -61,3 +62,10 @@ def _validate_ability_data(self, create: bool, data: dict): # Validate platforms, ability will not be loaded if empty if 'executors' in data and not data['executors']: raise JsonHttpBadRequest('At least one executor is required to save ability.') + + async def _save_and_reload_object(self, file_path: str, data: dict, obj_type: type, access: BaseWorld.Access): + allowed = self._get_allowed_from_access(access) + await self._file_svc.save_file(file_path, yaml.dump([data], encoding='utf-8', sort_keys=False), + '', encrypt=False) + await self._data_svc.remove('abilities', dict(ability_id=data['ability_id'])) + await self._data_svc.load_ability_file(file_path, allowed) diff --git a/app/objects/c_ability.py b/app/objects/c_ability.py index 28f2c5753..d3af137f6 100644 --- a/app/objects/c_ability.py +++ b/app/objects/c_ability.py @@ -18,13 +18,13 @@ class AbilitySchema(ma.Schema): technique_id = ma.fields.String(missing=None) name = ma.fields.String(missing=None) description = ma.fields.String(missing=None) - executors = ma.fields.List(ma.fields.Nested(ExecutorSchema), missing=None) - requirements = ma.fields.List(ma.fields.Nested(RequirementSchema), missing=None) + executors = ma.fields.List(ma.fields.Nested(ExecutorSchema()), missing=None) + requirements = ma.fields.List(ma.fields.Nested(RequirementSchema()), missing=None) privilege = ma.fields.String(missing=None) repeatable = ma.fields.Bool(missing=None) buckets = ma.fields.List(ma.fields.String(), missing=None) additional_info = ma.fields.Dict(keys=ma.fields.String(), values=ma.fields.String()) - access = ma.fields.Nested(AccessSchema, missing=None) + access = ma.fields.Nested(AccessSchema()) singleton = ma.fields.Bool(missing=None) @ma.pre_load diff --git a/app/service/data_svc.py b/app/service/data_svc.py index 62fd779cd..545f97e18 100644 --- a/app/service/data_svc.py +++ b/app/service/data_svc.py @@ -147,18 +147,31 @@ async def remove(self, object_name, match): async def load_ability_file(self, filename, access): for entries in self.strip_yml(filename): for ab in entries: - ability_id = ab.pop('id', None) + ability_id = ab.pop('id') if 'id' in ab else ab.pop('ability_id', None) name = ab.pop('name', '') description = ab.pop('description', '') tactic = ab.pop('tactic', None) - technique_id = ab.get('technique', dict()).get('attack_id') - technique_name = ab.pop('technique', dict()).get('name') + if 'technique_id' in ab: + technique_id = ab.pop('technique_id') + else: + technique_id = ab.get('technique', dict()).get('attack_id') + if 'technique_name' in ab: + technique_name = ab.pop('technique_name') + else: + technique_name = ab.pop('technique', dict()).get('name') privilege = ab.pop('privilege', None) repeatable = ab.pop('repeatable', False) singleton = ab.pop('singleton', False) requirements = await self._load_ability_requirements(ab.pop('requirements', [])) buckets = ab.pop('buckets', [tactic]) - executors = await self.load_executors_from_platform_dict(ab.pop('platforms', dict())) + if 'executors' in ab: + executors = await self.load_executors_from_list(ab.pop('executors', dict())) + else: + executors = await self.load_executors_from_platform_dict(ab.pop('platforms', dict())) + + if 'access' in ab and ab['access']: + access = ab['access'] + ab.pop('access', None) if tactic and tactic not in filename: self.log.error('Ability=%s has wrong tactic' % id) @@ -203,6 +216,28 @@ async def load_executors_from_platform_dict(self, platforms): parsers=parsers, cleanup=cleanup, variations=variations)) return executors + async def load_executors_from_list(self, executors: list): + output_list = [] + for entry in executors: + name = entry.pop('name', None) + platform = entry.pop('platform', None) + command = entry.pop('command', None) + code = entry.pop('code', None) + language = entry.pop('language', None) + build_target = entry.pop('build_target', None) + payloads = entry.pop('payloads', None) + uploads = entry.pop('uploads', None) + timeout = entry.pop('timeout', None) + parsers = entry.pop('parsers', None) + cleanup = entry.pop('cleanup', None) + variations = entry.pop('variations', None) + additional_info = entry.pop('additional_info', None) + output_list.append(Executor(name=name, platform=platform, command=command, code=code, language=language, + build_target=build_target, payloads=payloads, uploads=uploads, timeout=timeout, + parsers=parsers, cleanup=cleanup, variations=variations, + additional_info=additional_info)) + return output_list + async def load_adversary_file(self, filename, access): warnings.warn("Function deprecated and will be removed in a future update. Use load_yaml_file", DeprecationWarning) await self.load_yaml_file(Adversary, filename, access) From b2f134acd823f1d8bb6a73fb9e85339bf085130d Mon Sep 17 00:00:00 2001 From: blee Date: Tue, 29 Jun 2021 17:24:20 -0400 Subject: [PATCH 17/58] fix post request issues --- app/api/v2/managers/ability_api_manager.py | 6 ++--- app/objects/c_ability.py | 28 ++++++++-------------- 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/app/api/v2/managers/ability_api_manager.py b/app/api/v2/managers/ability_api_manager.py index 54ce25c94..e718ac9c3 100644 --- a/app/api/v2/managers/ability_api_manager.py +++ b/app/api/v2/managers/ability_api_manager.py @@ -23,7 +23,8 @@ async def create_on_disk_object(self, data: dict, access: dict, ram_key: str, id if not os.path.exists(tactic_dir): os.makedirs(tactic_dir) file_path = os.path.join(tactic_dir, '%s.yml' % data['ability_id']) - await self._save_and_reload_object(file_path, data, obj_class, access) + allowed = self._get_allowed_from_access(access) + await self._save_and_reload_object(file_path, data, obj_class, allowed) return next(self.find_objects(ram_key, {id_property: obj_id})) async def replace_on_disk_object(self, obj: Any, data: dict, ram_key: str, id_property: str): @@ -64,8 +65,7 @@ def _validate_ability_data(self, create: bool, data: dict): raise JsonHttpBadRequest('At least one executor is required to save ability.') async def _save_and_reload_object(self, file_path: str, data: dict, obj_type: type, access: BaseWorld.Access): - allowed = self._get_allowed_from_access(access) await self._file_svc.save_file(file_path, yaml.dump([data], encoding='utf-8', sort_keys=False), '', encrypt=False) await self._data_svc.remove('abilities', dict(ability_id=data['ability_id'])) - await self._data_svc.load_ability_file(file_path, allowed) + await self._data_svc.load_ability_file(file_path, access) diff --git a/app/objects/c_ability.py b/app/objects/c_ability.py index d3af137f6..b3d2a9187 100644 --- a/app/objects/c_ability.py +++ b/app/objects/c_ability.py @@ -18,31 +18,26 @@ class AbilitySchema(ma.Schema): technique_id = ma.fields.String(missing=None) name = ma.fields.String(missing=None) description = ma.fields.String(missing=None) - executors = ma.fields.List(ma.fields.Nested(ExecutorSchema()), missing=None) - requirements = ma.fields.List(ma.fields.Nested(RequirementSchema()), missing=None) + executors = ma.fields.List(ma.fields.Nested(ExecutorSchema), missing=None) + requirements = ma.fields.List(ma.fields.Nested(RequirementSchema), missing=None) privilege = ma.fields.String(missing=None) repeatable = ma.fields.Bool(missing=None) buckets = ma.fields.List(ma.fields.String(), missing=None) additional_info = ma.fields.Dict(keys=ma.fields.String(), values=ma.fields.String()) - access = ma.fields.Nested(AccessSchema()) + access = ma.fields.Nested(AccessSchema, missing=None) singleton = ma.fields.Bool(missing=None) @ma.pre_load - def fix_tactic(self, ability, **_): - if 'tactic' in ability: - ability['tactic'] = ability['tactic'].lower() - - @ma.pre_load - def fix_id(self, ability, **_): - if 'id' in ability: - ability['ability_id'] = ability.pop('id') - return ability + def fix_id(self, data, **_): + if 'id' in data: + data['ability_id'] = data.pop('id') + return data @ma.post_load - def build_ability(self, data, **kwargs): + def build_ability(self, data, **_): if 'technique' in data: data['technique_name'] = data.pop('technique') - return None if kwargs.get('partial') is True else Ability(**data) + return Ability(**data) class Ability(FirstClassObjectInterface, BaseObject): @@ -60,7 +55,7 @@ def unique(self): def executors(self): yield from self._executor_map.values() - def __init__(self, ability_id='', name=None, description=None, tactic=None, technique_id=None, technique_name=None, + def __init__(self, ability_id, name=None, description=None, tactic=None, technique_id=None, technique_name=None, executors=(), requirements=None, privilege=None, repeatable=False, buckets=None, access=None, additional_info=None, tags=None, singleton=False, **kwargs): super().__init__() @@ -117,11 +112,9 @@ def find_executor(self, name, platform): def find_executors(self, names, platform): """Find executors for matching platform/executor names - Only the first instance of a matching executor will be returned, as there should not be multiple executors matching a single platform/executor name pair. - :param names: Executors to search. ex: ['psh', 'cmd'] :type names: list(str) :param platform: Platform to search. ex: windows @@ -144,7 +137,6 @@ def find_executors(self, names, platform): def add_executor(self, executor): """Add executor to map - If the executor exists, delete the current entry and add the new executor to the bottom for FIFO """ From cdf4bf5cd8351bdcadc8abb4dfce76c3b0b21798 Mon Sep 17 00:00:00 2001 From: blee Date: Wed, 30 Jun 2021 17:03:55 -0400 Subject: [PATCH 18/58] make access read only --- app/objects/c_ability.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/objects/c_ability.py b/app/objects/c_ability.py index b3d2a9187..318be6ecd 100644 --- a/app/objects/c_ability.py +++ b/app/objects/c_ability.py @@ -24,7 +24,7 @@ class AbilitySchema(ma.Schema): repeatable = ma.fields.Bool(missing=None) buckets = ma.fields.List(ma.fields.String(), missing=None) additional_info = ma.fields.Dict(keys=ma.fields.String(), values=ma.fields.String()) - access = ma.fields.Nested(AccessSchema, missing=None) + access = ma.fields.Nested(AccessSchema, missing=None, dump_only=True) singleton = ma.fields.Bool(missing=None) @ma.pre_load @@ -34,10 +34,10 @@ def fix_id(self, data, **_): return data @ma.post_load - def build_ability(self, data, **_): + def build_ability(self, data, **kwargs): if 'technique' in data: data['technique_name'] = data.pop('technique') - return Ability(**data) + return None if kwargs.get('partial') is True else Ability(**data) class Ability(FirstClassObjectInterface, BaseObject): From d7d2333be1570e8a15514f721cb63eff897a2432 Mon Sep 17 00:00:00 2001 From: blee Date: Wed, 30 Jun 2021 18:20:56 -0400 Subject: [PATCH 19/58] overridee update on disk object --- app/api/v2/managers/ability_api_manager.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/app/api/v2/managers/ability_api_manager.py b/app/api/v2/managers/ability_api_manager.py index e718ac9c3..d3f78088f 100644 --- a/app/api/v2/managers/ability_api_manager.py +++ b/app/api/v2/managers/ability_api_manager.py @@ -33,7 +33,14 @@ async def replace_on_disk_object(self, obj: Any, data: dict, ram_key: str, id_pr async def update_on_disk_object(self, obj: Any, data: dict, ram_key: str, id_property: str, obj_class: type): self._validate_ability_data(create=False, data=data) - return await super().update_on_disk_object(obj, data, ram_key, id_property, obj_class) + obj_id = getattr(obj, id_property) + file_path = await self._get_existing_object_file_path(obj_id, ram_key) + + existing_obj_data = dict(self.strip_yml(file_path)[0][0]) + existing_obj_data.update(data) + + await self._save_and_reload_object(file_path, existing_obj_data, obj_class, obj.access) + return next(self.find_objects(ram_key, {id_property: obj_id})) '''Helpers''' From 3cc1d251c73d21d3d528cf20c8dfbab5bc6d9adf Mon Sep 17 00:00:00 2001 From: blee Date: Thu, 1 Jul 2021 09:38:19 -0400 Subject: [PATCH 20/58] remove dump_only=True from access field --- app/objects/c_ability.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/objects/c_ability.py b/app/objects/c_ability.py index 318be6ecd..1e9e1ec5c 100644 --- a/app/objects/c_ability.py +++ b/app/objects/c_ability.py @@ -24,7 +24,7 @@ class AbilitySchema(ma.Schema): repeatable = ma.fields.Bool(missing=None) buckets = ma.fields.List(ma.fields.String(), missing=None) additional_info = ma.fields.Dict(keys=ma.fields.String(), values=ma.fields.String()) - access = ma.fields.Nested(AccessSchema, missing=None, dump_only=True) + access = ma.fields.Nested(AccessSchema, missing=None) singleton = ma.fields.Bool(missing=None) @ma.pre_load From 2d6813552a041620b6d3729f11ef8681fe300a20 Mon Sep 17 00:00:00 2001 From: blee Date: Fri, 9 Jul 2021 16:16:17 -0400 Subject: [PATCH 21/58] fix id error with post requests --- app/api/v2/managers/ability_api_manager.py | 15 ++++++++------- app/objects/c_ability.py | 5 +++-- app/service/data_svc.py | 9 +++------ 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/app/api/v2/managers/ability_api_manager.py b/app/api/v2/managers/ability_api_manager.py index d3f78088f..e385ebbe9 100644 --- a/app/api/v2/managers/ability_api_manager.py +++ b/app/api/v2/managers/ability_api_manager.py @@ -16,13 +16,12 @@ def __init__(self, data_svc, file_svc): async def create_on_disk_object(self, data: dict, access: dict, ram_key: str, id_property: str, obj_class: type): self._validate_ability_data(create=True, data=data) - obj_id = data.get(id_property) or str(uuid.uuid4()) - data[id_property] = obj_id + obj_id = data.get('id') tactic_dir = os.path.join('data', 'abilities', data.get('tactic')) if not os.path.exists(tactic_dir): os.makedirs(tactic_dir) - file_path = os.path.join(tactic_dir, '%s.yml' % data['ability_id']) + file_path = os.path.join(tactic_dir, '%s.yml' % obj_id) allowed = self._get_allowed_from_access(access) await self._save_and_reload_object(file_path, data, obj_class, allowed) return next(self.find_objects(ram_key, {id_property: obj_id})) @@ -43,17 +42,19 @@ async def update_on_disk_object(self, obj: Any, data: dict, ram_key: str, id_pro return next(self.find_objects(ram_key, {id_property: obj_id})) '''Helpers''' - def _validate_ability_data(self, create: bool, data: dict): + # Prevent errors due to incorrect id property. + data.pop('id', None) + # If a new ability is being created, ensure required fields present. if create: # Set ability ID if undefined - if 'ability_id' not in data: - data['ability_id'] = str(uuid.uuid4()) + data['id'] = data.pop('ability_id', str(uuid.uuid4())) if 'tactic' not in data: raise JsonHttpBadRequest(f'Cannot create ability {data["ability_id"]} due to missing tactic') if not data['executors']: raise JsonHttpBadRequest(f'Cannot create ability {data["ability_id"]}: at least one executor required') + # Validate ID, used for file creation validator = re.compile(r'^[a-zA-Z0-9-_]+$') if 'ability_id' in data and not validator.match(data['ability_id']): @@ -74,5 +75,5 @@ def _validate_ability_data(self, create: bool, data: dict): async def _save_and_reload_object(self, file_path: str, data: dict, obj_type: type, access: BaseWorld.Access): await self._file_svc.save_file(file_path, yaml.dump([data], encoding='utf-8', sort_keys=False), '', encrypt=False) - await self._data_svc.remove('abilities', dict(ability_id=data['ability_id'])) + await self._data_svc.remove('abilities', dict(ability_id=data['id'])) await self._data_svc.load_ability_file(file_path, access) diff --git a/app/objects/c_ability.py b/app/objects/c_ability.py index 1e9e1ec5c..79fa211df 100644 --- a/app/objects/c_ability.py +++ b/app/objects/c_ability.py @@ -1,5 +1,6 @@ import collections import os +import uuid import marshmallow as ma @@ -55,11 +56,11 @@ def unique(self): def executors(self): yield from self._executor_map.values() - def __init__(self, ability_id, name=None, description=None, tactic=None, technique_id=None, technique_name=None, + def __init__(self, ability_id='', name=None, description=None, tactic=None, technique_id=None, technique_name=None, executors=(), requirements=None, privilege=None, repeatable=False, buckets=None, access=None, additional_info=None, tags=None, singleton=False, **kwargs): super().__init__() - self.ability_id = ability_id + self.ability_id = ability_id if ability_id else str(uuid.uuid4()) self.tactic = tactic.lower() if tactic else None self.technique_name = technique_name self.technique_id = technique_id diff --git a/app/service/data_svc.py b/app/service/data_svc.py index 545f97e18..20d74983b 100644 --- a/app/service/data_svc.py +++ b/app/service/data_svc.py @@ -147,7 +147,7 @@ async def remove(self, object_name, match): async def load_ability_file(self, filename, access): for entries in self.strip_yml(filename): for ab in entries: - ability_id = ab.pop('id') if 'id' in ab else ab.pop('ability_id', None) + ability_id = ab.pop('id', None) name = ab.pop('name', '') description = ab.pop('description', '') tactic = ab.pop('tactic', None) @@ -165,13 +165,10 @@ async def load_ability_file(self, filename, access): requirements = await self._load_ability_requirements(ab.pop('requirements', [])) buckets = ab.pop('buckets', [tactic]) if 'executors' in ab: - executors = await self.load_executors_from_list(ab.pop('executors', dict())) + executors = await self.load_executors_from_list(ab.pop('executors')) else: executors = await self.load_executors_from_platform_dict(ab.pop('platforms', dict())) - - if 'access' in ab and ab['access']: - access = ab['access'] - ab.pop('access', None) + access = ab.pop('access', None) if tactic and tactic not in filename: self.log.error('Ability=%s has wrong tactic' % id) From 7dd88b7379801fa9cd4477fb44cece68a28233d2 Mon Sep 17 00:00:00 2001 From: blee Date: Fri, 9 Jul 2021 18:36:48 -0400 Subject: [PATCH 22/58] fix error messages --- app/api/v2/managers/ability_api_manager.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/api/v2/managers/ability_api_manager.py b/app/api/v2/managers/ability_api_manager.py index e385ebbe9..1e9da18d0 100644 --- a/app/api/v2/managers/ability_api_manager.py +++ b/app/api/v2/managers/ability_api_manager.py @@ -43,7 +43,7 @@ async def update_on_disk_object(self, obj: Any, data: dict, ram_key: str, id_pro '''Helpers''' def _validate_ability_data(self, create: bool, data: dict): - # Prevent errors due to incorrect id property. + # Prevent errors due to incorrect expected schema id property. data.pop('id', None) # If a new ability is being created, ensure required fields present. @@ -51,14 +51,14 @@ def _validate_ability_data(self, create: bool, data: dict): # Set ability ID if undefined data['id'] = data.pop('ability_id', str(uuid.uuid4())) if 'tactic' not in data: - raise JsonHttpBadRequest(f'Cannot create ability {data["ability_id"]} due to missing tactic') + raise JsonHttpBadRequest(f'Cannot create ability {data["id"]} due to missing tactic') if not data['executors']: - raise JsonHttpBadRequest(f'Cannot create ability {data["ability_id"]}: at least one executor required') + raise JsonHttpBadRequest(f'Cannot create ability {data["id"]}: at least one executor required') # Validate ID, used for file creation validator = re.compile(r'^[a-zA-Z0-9-_]+$') - if 'ability_id' in data and not validator.match(data['ability_id']): - raise JsonHttpBadRequest(f'Invalid ability ID {data["ability_id"]}. IDs can only contain ' + if 'id' in data and not validator.match(data['id']): + raise JsonHttpBadRequest(f'Invalid ability ID {data["id"]}. IDs can only contain ' 'alphanumeric characters, hyphens, and underscores.') # Validate tactic, used for directory creation, lower case if present From 77896efb6976586161f2cea59b6567cfefbb2483 Mon Sep 17 00:00:00 2001 From: blee Date: Mon, 12 Jul 2021 09:09:48 -0400 Subject: [PATCH 23/58] fix id correction --- app/api/v2/managers/ability_api_manager.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/api/v2/managers/ability_api_manager.py b/app/api/v2/managers/ability_api_manager.py index 1e9da18d0..cd954cb30 100644 --- a/app/api/v2/managers/ability_api_manager.py +++ b/app/api/v2/managers/ability_api_manager.py @@ -45,11 +45,14 @@ async def update_on_disk_object(self, obj: Any, data: dict, ram_key: str, id_pro def _validate_ability_data(self, create: bool, data: dict): # Prevent errors due to incorrect expected schema id property. data.pop('id', None) + # Correct ability_id key for ability file saving. + data['id'] = data.pop('ability_id', '') # If a new ability is being created, ensure required fields present. if create: # Set ability ID if undefined - data['id'] = data.pop('ability_id', str(uuid.uuid4())) + if not data['id']: + data['id'] = str(uuid.uuid4()) if 'tactic' not in data: raise JsonHttpBadRequest(f'Cannot create ability {data["id"]} due to missing tactic') if not data['executors']: From a96a4734d9f2d74acb8ce038b4da1468cf0cbcfe Mon Sep 17 00:00:00 2001 From: blee Date: Thu, 15 Jul 2021 09:24:09 -0400 Subject: [PATCH 24/58] request.has_body is deprecated; use new call --- app/api/v2/handlers/base_object_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/v2/handlers/base_object_api.py b/app/api/v2/handlers/base_object_api.py index 8922dad7a..d6115c795 100644 --- a/app/api/v2/handlers/base_object_api.py +++ b/app/api/v2/handlers/base_object_api.py @@ -144,7 +144,7 @@ async def delete_on_disk_object(self, request: web.Request): async def _parse_common_data_from_request(self, request) -> (dict, dict, str, dict, dict): data = {} - if request.body_exists: + if request.can_read_body: data = await request.json() obj_id = request.match_info.get(self.id_property, '') From a4f7c5bf282d4e23fd01e5417993c1874b5ff528 Mon Sep 17 00:00:00 2001 From: blee Date: Thu, 15 Jul 2021 10:36:06 -0400 Subject: [PATCH 25/58] remove can_read_body call; add extra read parameter to parse data --- app/api/v2/handlers/base_object_api.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/app/api/v2/handlers/base_object_api.py b/app/api/v2/handlers/base_object_api.py index d6115c795..533991655 100644 --- a/app/api/v2/handlers/base_object_api.py +++ b/app/api/v2/handlers/base_object_api.py @@ -34,7 +34,7 @@ async def get_all_objects(self, request: web.Request): return self._api_manager.find_and_dump_objects(self.ram_key, access, sort, include, exclude) async def get_object(self, request: web.Request): - data, access, obj_id, query, search = await self._parse_common_data_from_request(request) + data, access, obj_id, query, search = await self._parse_common_data_from_request(request, read_body=False) obj = self._api_manager.find_object(self.ram_key, query) if not obj: @@ -77,7 +77,7 @@ async def _error_if_object_with_id_exists(self, obj_id: str): """PATCH""" async def update_object(self, request: web.Request): - data, access, obj_id, query, search = await self._parse_common_data_from_request(request) + data, access, obj_id, query, search = await self._parse_common_data_from_request(request, read_body=True) obj = self._api_manager.find_and_update_object(self.ram_key, data, search) if not obj: @@ -85,7 +85,7 @@ async def update_object(self, request: web.Request): return obj async def update_on_disk_object(self, request: web.Request): - data, access, obj_id, query, search = await self._parse_common_data_from_request(request) + data, access, obj_id, query, search = await self._parse_common_data_from_request(request, read_body=True) obj = await self._api_manager.find_and_update_on_disk_object(data, search, self.ram_key, self.id_property, self.obj_class) @@ -96,7 +96,7 @@ async def update_on_disk_object(self, request: web.Request): """PUT""" async def create_or_update_object(self, request: web.Request): - data, access, obj_id, query, search = await self._parse_common_data_from_request(request) + data, access, obj_id, query, search = await self._parse_common_data_from_request(request, read_body=True) matched_obj = self._api_manager.find_object(self.ram_key, query) if matched_obj and matched_obj.access not in access['access']: @@ -105,7 +105,7 @@ async def create_or_update_object(self, request: web.Request): return self._api_manager.create_object_from_schema(self.schema, data, access) async def create_or_update_on_disk_object(self, request: web.Request): - data, access, obj_id, query, search = await self._parse_common_data_from_request(request) + data, access, obj_id, query, search = await self._parse_common_data_from_request(request, read_body=True) matched_obj = self._api_manager.find_object(self.ram_key, query) if not matched_obj: @@ -142,9 +142,9 @@ async def delete_on_disk_object(self, request: web.Request): """Helpers""" - async def _parse_common_data_from_request(self, request) -> (dict, dict, str, dict, dict): + async def _parse_common_data_from_request(self, request, read_body) -> (dict, dict, str, dict, dict): data = {} - if request.can_read_body: + if read_body: data = await request.json() obj_id = request.match_info.get(self.id_property, '') From 36103acfd6a0520a6340ae40513f6c3e2504c656 Mon Sep 17 00:00:00 2001 From: blee Date: Thu, 15 Jul 2021 11:28:40 -0400 Subject: [PATCH 26/58] use request access to set ability access --- app/api/v2/managers/ability_api_manager.py | 3 --- app/service/data_svc.py | 1 - 2 files changed, 4 deletions(-) diff --git a/app/api/v2/managers/ability_api_manager.py b/app/api/v2/managers/ability_api_manager.py index cd954cb30..9673a3941 100644 --- a/app/api/v2/managers/ability_api_manager.py +++ b/app/api/v2/managers/ability_api_manager.py @@ -17,7 +17,6 @@ def __init__(self, data_svc, file_svc): async def create_on_disk_object(self, data: dict, access: dict, ram_key: str, id_property: str, obj_class: type): self._validate_ability_data(create=True, data=data) obj_id = data.get('id') - tactic_dir = os.path.join('data', 'abilities', data.get('tactic')) if not os.path.exists(tactic_dir): os.makedirs(tactic_dir) @@ -34,10 +33,8 @@ async def update_on_disk_object(self, obj: Any, data: dict, ram_key: str, id_pro self._validate_ability_data(create=False, data=data) obj_id = getattr(obj, id_property) file_path = await self._get_existing_object_file_path(obj_id, ram_key) - existing_obj_data = dict(self.strip_yml(file_path)[0][0]) existing_obj_data.update(data) - await self._save_and_reload_object(file_path, existing_obj_data, obj_class, obj.access) return next(self.find_objects(ram_key, {id_property: obj_id})) diff --git a/app/service/data_svc.py b/app/service/data_svc.py index 20d74983b..78f4ca984 100644 --- a/app/service/data_svc.py +++ b/app/service/data_svc.py @@ -168,7 +168,6 @@ async def load_ability_file(self, filename, access): executors = await self.load_executors_from_list(ab.pop('executors')) else: executors = await self.load_executors_from_platform_dict(ab.pop('platforms', dict())) - access = ab.pop('access', None) if tactic and tactic not in filename: self.log.error('Ability=%s has wrong tactic' % id) From 9b6734f04400ed92d17b992ae99e11e2c5a2cbdb Mon Sep 17 00:00:00 2001 From: blee Date: Thu, 15 Jul 2021 12:51:34 -0400 Subject: [PATCH 27/58] read/set access parameter --- app/service/data_svc.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/service/data_svc.py b/app/service/data_svc.py index 78f4ca984..36ca93737 100644 --- a/app/service/data_svc.py +++ b/app/service/data_svc.py @@ -168,6 +168,9 @@ async def load_ability_file(self, filename, access): executors = await self.load_executors_from_list(ab.pop('executors')) else: executors = await self.load_executors_from_platform_dict(ab.pop('platforms', dict())) + if 'access' in ab: + access = ab.get('access') + ab.pop('access', None) if tactic and tactic not in filename: self.log.error('Ability=%s has wrong tactic' % id) From da57b0abaf21c3287bf9e7f70ca3b6ab16d9a783 Mon Sep 17 00:00:00 2001 From: blee Date: Fri, 23 Jul 2021 14:05:50 -0400 Subject: [PATCH 28/58] apply read_body fix --- app/api/v2/handlers/base_object_api.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/app/api/v2/handlers/base_object_api.py b/app/api/v2/handlers/base_object_api.py index 533991655..ae579aa91 100644 --- a/app/api/v2/handlers/base_object_api.py +++ b/app/api/v2/handlers/base_object_api.py @@ -1,4 +1,5 @@ import abc +import json from aiohttp import web @@ -34,7 +35,7 @@ async def get_all_objects(self, request: web.Request): return self._api_manager.find_and_dump_objects(self.ram_key, access, sort, include, exclude) async def get_object(self, request: web.Request): - data, access, obj_id, query, search = await self._parse_common_data_from_request(request, read_body=False) + data, access, obj_id, query, search = await self._parse_common_data_from_request(request) obj = self._api_manager.find_object(self.ram_key, query) if not obj: @@ -77,7 +78,7 @@ async def _error_if_object_with_id_exists(self, obj_id: str): """PATCH""" async def update_object(self, request: web.Request): - data, access, obj_id, query, search = await self._parse_common_data_from_request(request, read_body=True) + data, access, obj_id, query, search = await self._parse_common_data_from_request(request) obj = self._api_manager.find_and_update_object(self.ram_key, data, search) if not obj: @@ -85,7 +86,7 @@ async def update_object(self, request: web.Request): return obj async def update_on_disk_object(self, request: web.Request): - data, access, obj_id, query, search = await self._parse_common_data_from_request(request, read_body=True) + data, access, obj_id, query, search = await self._parse_common_data_from_request(request) obj = await self._api_manager.find_and_update_on_disk_object(data, search, self.ram_key, self.id_property, self.obj_class) @@ -96,7 +97,7 @@ async def update_on_disk_object(self, request: web.Request): """PUT""" async def create_or_update_object(self, request: web.Request): - data, access, obj_id, query, search = await self._parse_common_data_from_request(request, read_body=True) + data, access, obj_id, query, search = await self._parse_common_data_from_request(request) matched_obj = self._api_manager.find_object(self.ram_key, query) if matched_obj and matched_obj.access not in access['access']: @@ -105,7 +106,7 @@ async def create_or_update_object(self, request: web.Request): return self._api_manager.create_object_from_schema(self.schema, data, access) async def create_or_update_on_disk_object(self, request: web.Request): - data, access, obj_id, query, search = await self._parse_common_data_from_request(request, read_body=True) + data, access, obj_id, query, search = await self._parse_common_data_from_request(request) matched_obj = self._api_manager.find_object(self.ram_key, query) if not matched_obj: @@ -142,10 +143,11 @@ async def delete_on_disk_object(self, request: web.Request): """Helpers""" - async def _parse_common_data_from_request(self, request, read_body) -> (dict, dict, str, dict, dict): + async def _parse_common_data_from_request(self, request) -> (dict, dict, str, dict, dict): data = {} - if read_body: - data = await request.json() + raw_body = await request.read() + if raw_body: + data = json.loads(raw_body) obj_id = request.match_info.get(self.id_property, '') if obj_id: From b82ab44e0ccdc63a52ae8cc0c67e64943c58e55c Mon Sep 17 00:00:00 2001 From: blee Date: Fri, 23 Jul 2021 16:00:22 -0400 Subject: [PATCH 29/58] fix validation errors --- app/api/v2/managers/ability_api_manager.py | 11 ++--------- app/objects/c_ability.py | 4 ++-- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/app/api/v2/managers/ability_api_manager.py b/app/api/v2/managers/ability_api_manager.py index 9673a3941..540f2e8bb 100644 --- a/app/api/v2/managers/ability_api_manager.py +++ b/app/api/v2/managers/ability_api_manager.py @@ -26,7 +26,7 @@ async def create_on_disk_object(self, data: dict, access: dict, ram_key: str, id return next(self.find_objects(ram_key, {id_property: obj_id})) async def replace_on_disk_object(self, obj: Any, data: dict, ram_key: str, id_property: str): - self._validate_ability_data(create=False, data=data) + self._validate_ability_data(create=True, data=data) return await super().replace_on_disk_object(obj, data, ram_key, id_property) async def update_on_disk_object(self, obj: Any, data: dict, ram_key: str, id_property: str, obj_class: type): @@ -40,8 +40,6 @@ async def update_on_disk_object(self, obj: Any, data: dict, ram_key: str, id_pro '''Helpers''' def _validate_ability_data(self, create: bool, data: dict): - # Prevent errors due to incorrect expected schema id property. - data.pop('id', None) # Correct ability_id key for ability file saving. data['id'] = data.pop('ability_id', '') @@ -52,9 +50,8 @@ def _validate_ability_data(self, create: bool, data: dict): data['id'] = str(uuid.uuid4()) if 'tactic' not in data: raise JsonHttpBadRequest(f'Cannot create ability {data["id"]} due to missing tactic') - if not data['executors']: + if not data.get('executors'): raise JsonHttpBadRequest(f'Cannot create ability {data["id"]}: at least one executor required') - # Validate ID, used for file creation validator = re.compile(r'^[a-zA-Z0-9-_]+$') if 'id' in data and not validator.match(data['id']): @@ -68,10 +65,6 @@ def _validate_ability_data(self, create: bool, data: dict): 'alphanumeric characters, hyphens, and underscores.') data['tactic'] = data['tactic'].lower() - # Validate platforms, ability will not be loaded if empty - if 'executors' in data and not data['executors']: - raise JsonHttpBadRequest('At least one executor is required to save ability.') - async def _save_and_reload_object(self, file_path: str, data: dict, obj_type: type, access: BaseWorld.Access): await self._file_svc.save_file(file_path, yaml.dump([data], encoding='utf-8', sort_keys=False), '', encrypt=False) diff --git a/app/objects/c_ability.py b/app/objects/c_ability.py index 79fa211df..4d8c3d782 100644 --- a/app/objects/c_ability.py +++ b/app/objects/c_ability.py @@ -14,12 +14,12 @@ class AbilitySchema(ma.Schema): ability_id = ma.fields.String() - tactic = ma.fields.String(missing=None) + tactic = ma.fields.String(required=True) technique_name = ma.fields.String(missing=None) technique_id = ma.fields.String(missing=None) name = ma.fields.String(missing=None) description = ma.fields.String(missing=None) - executors = ma.fields.List(ma.fields.Nested(ExecutorSchema), missing=None) + executors = ma.fields.List(ma.fields.Nested(ExecutorSchema), required=True) requirements = ma.fields.List(ma.fields.Nested(RequirementSchema), missing=None) privilege = ma.fields.String(missing=None) repeatable = ma.fields.Bool(missing=None) From 7342d3e286b9a2cac2f5b589dbb7c72c13b3289e Mon Sep 17 00:00:00 2001 From: blee Date: Sun, 25 Jul 2021 16:46:56 -0400 Subject: [PATCH 30/58] update tests --- tests/objects/test_link.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/objects/test_link.py b/tests/objects/test_link.py index 2eeb8af3f..e1a3076b6 100644 --- a/tests/objects/test_link.py +++ b/tests/objects/test_link.py @@ -103,7 +103,7 @@ def test_emit_status_change_event(self, loop, fake_event_svc, ability, executor) def test_link_agent_reported_time_not_present_when_none_roundtrip(self, ability, executor): test_executor = executor(name='psh', platform='windows') - test_ability = ability(ability_id='123') + test_ability = ability(ability_id='123', tactic='test') test_link = Link(command='sc.exe \\dc create sandsvc binpath= "s4ndc4t.exe -originLinkID 111111"', paw='123456', ability=test_ability, executor=test_executor, id=111111) serialized_link = test_link.display @@ -114,7 +114,7 @@ def test_link_agent_reported_time_not_present_when_none_roundtrip(self, ability, def test_link_agent_reported_time_present_when_set_roundtrip(self, ability, executor): test_executor = executor(name='psh', platform='windows') - test_ability = ability(ability_id='123') + test_ability = ability(ability_id='123', tactic='test') test_link = Link(command='sc.exe \\dc create sandsvc binpath= "s4ndc4t.exe -originLinkID 111111"', paw='123456', ability=test_ability, executor=test_executor, id=111111, agent_reported_time=BaseService.get_timestamp_from_string('2021-02-23 11:50:16')) From f490b7469299c4599775a87ac6488cc4ff16a4b2 Mon Sep 17 00:00:00 2001 From: blee Date: Mon, 26 Jul 2021 13:33:16 -0400 Subject: [PATCH 31/58] fix rest_svc index bug --- app/service/rest_svc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/service/rest_svc.py b/app/service/rest_svc.py index 48805d716..af4b8aa08 100644 --- a/app/service/rest_svc.py +++ b/app/service/rest_svc.py @@ -617,7 +617,7 @@ async def _strip_parsers_from_ability(self, ability): Return ability (minus parsers) and parsers as seperate dict """ parsers = {} - for platform, executors in ability['platforms'].items(): + for platform, executors in ability.get('platforms', {}).items(): parsers[platform] = {} for executor, d in executors.items(): if d.get('parsers', False): @@ -630,7 +630,7 @@ async def _add_parsers_to_ability(self, ability, parsers): not an ability object but just the loaded dict from yaml ability file) """ - for platform, executors in ability['platforms'].items(): + for platform, executors in ability.get('platforms', {}).items(): if parsers.get(platform, False): for executor, _ in executors.items(): if parsers[platform].get(executor, False): From 8b641ee00f9b05a86dfd74711822d40892d71994 Mon Sep 17 00:00:00 2001 From: blee Date: Mon, 26 Jul 2021 13:33:40 -0400 Subject: [PATCH 32/58] update additional fields when storing updated ability --- app/objects/c_ability.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/objects/c_ability.py b/app/objects/c_ability.py index 4d8c3d782..a668b2991 100644 --- a/app/objects/c_ability.py +++ b/app/objects/c_ability.py @@ -99,6 +99,10 @@ def store(self, ram): existing.update('description', self.description) existing.update('_executor_map', self._executor_map) existing.update('privilege', self.privilege) + existing.update('repeatable', self.repeatable) + existing.update('buckets', self.buckets) + existing.update('tags', self.tags) + existing.update('singleton', self.singleton) return existing async def which_plugin(self): From 2a3ce59d2fe5e476f20da0a14f9df3d9540fcb65 Mon Sep 17 00:00:00 2001 From: blee Date: Mon, 26 Jul 2021 13:34:12 -0400 Subject: [PATCH 33/58] change file path if ability tactic is updated --- app/api/v2/managers/ability_api_manager.py | 25 +++++++++++++++++----- app/service/data_svc.py | 2 -- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/app/api/v2/managers/ability_api_manager.py b/app/api/v2/managers/ability_api_manager.py index 540f2e8bb..5fa19a0ba 100644 --- a/app/api/v2/managers/ability_api_manager.py +++ b/app/api/v2/managers/ability_api_manager.py @@ -17,17 +17,20 @@ def __init__(self, data_svc, file_svc): async def create_on_disk_object(self, data: dict, access: dict, ram_key: str, id_property: str, obj_class: type): self._validate_ability_data(create=True, data=data) obj_id = data.get('id') - tactic_dir = os.path.join('data', 'abilities', data.get('tactic')) - if not os.path.exists(tactic_dir): - os.makedirs(tactic_dir) - file_path = os.path.join(tactic_dir, '%s.yml' % obj_id) + file_path = self._create_ability_filepath(data.get('tactic'), obj_id) allowed = self._get_allowed_from_access(access) await self._save_and_reload_object(file_path, data, obj_class, allowed) return next(self.find_objects(ram_key, {id_property: obj_id})) async def replace_on_disk_object(self, obj: Any, data: dict, ram_key: str, id_property: str): self._validate_ability_data(create=True, data=data) - return await super().replace_on_disk_object(obj, data, ram_key, id_property) + obj_id = getattr(obj, id_property) + file_path = await self._get_existing_object_file_path(obj_id, ram_key) + if data.get('tactic') not in file_path: + await self.remove_object_from_disk_by_id(obj_id, ram_key) + file_path = self._create_ability_filepath(data.get('tactic'), obj_id) + await self._save_and_reload_object(file_path, data, type(obj), obj.access) + return next(self.find_objects(ram_key, {id_property: obj_id})) async def update_on_disk_object(self, obj: Any, data: dict, ram_key: str, id_property: str, obj_class: type): self._validate_ability_data(create=False, data=data) @@ -35,6 +38,9 @@ async def update_on_disk_object(self, obj: Any, data: dict, ram_key: str, id_pro file_path = await self._get_existing_object_file_path(obj_id, ram_key) existing_obj_data = dict(self.strip_yml(file_path)[0][0]) existing_obj_data.update(data) + if existing_obj_data.get('tactic') not in file_path: + await self.remove_object_from_disk_by_id(obj_id, ram_key) + file_path = self._create_ability_filepath(data.get('tactic'), obj_id) await self._save_and_reload_object(file_path, existing_obj_data, obj_class, obj.access) return next(self.find_objects(ram_key, {id_property: obj_id})) @@ -65,6 +71,15 @@ def _validate_ability_data(self, create: bool, data: dict): 'alphanumeric characters, hyphens, and underscores.') data['tactic'] = data['tactic'].lower() + if data.get('executor') and len(data.get('executor')) == 0: + raise JsonHttpBadRequest(f'Cannot create ability {data["id"]}: at least one executor required') + + def _create_ability_filepath(self, tactic: str, obj_id: str): + tactic_dir = os.path.join('data', 'abilities', tactic) + if not os.path.exists(tactic_dir): + os.makedirs(tactic_dir) + return os.path.join(tactic_dir, '%s.yml' % obj_id) + async def _save_and_reload_object(self, file_path: str, data: dict, obj_type: type, access: BaseWorld.Access): await self._file_svc.save_file(file_path, yaml.dump([data], encoding='utf-8', sort_keys=False), '', encrypt=False) diff --git a/app/service/data_svc.py b/app/service/data_svc.py index 36ca93737..886df59e1 100644 --- a/app/service/data_svc.py +++ b/app/service/data_svc.py @@ -168,8 +168,6 @@ async def load_ability_file(self, filename, access): executors = await self.load_executors_from_list(ab.pop('executors')) else: executors = await self.load_executors_from_platform_dict(ab.pop('platforms', dict())) - if 'access' in ab: - access = ab.get('access') ab.pop('access', None) if tactic and tactic not in filename: From e6107fc2ae2cf60cded3a2f6eefd7ef2300d7296 Mon Sep 17 00:00:00 2001 From: blee Date: Mon, 26 Jul 2021 18:35:59 -0400 Subject: [PATCH 34/58] make name a required field for consistency w/ ui --- app/api/v2/managers/ability_api_manager.py | 7 ++++++- app/objects/c_ability.py | 7 +++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/app/api/v2/managers/ability_api_manager.py b/app/api/v2/managers/ability_api_manager.py index 5fa19a0ba..eae1b57ee 100644 --- a/app/api/v2/managers/ability_api_manager.py +++ b/app/api/v2/managers/ability_api_manager.py @@ -54,6 +54,8 @@ def _validate_ability_data(self, create: bool, data: dict): # Set ability ID if undefined if not data['id']: data['id'] = str(uuid.uuid4()) + if not data.get('name'): + raise JsonHttpBadRequest(f'Cannot create ability {data["id"]} due to missing name') if 'tactic' not in data: raise JsonHttpBadRequest(f'Cannot create ability {data["id"]} due to missing tactic') if not data.get('executors'): @@ -71,9 +73,12 @@ def _validate_ability_data(self, create: bool, data: dict): 'alphanumeric characters, hyphens, and underscores.') data['tactic'] = data['tactic'].lower() - if data.get('executor') and len(data.get('executor')) == 0: + if 'executors' in data and not data.get('executors'): raise JsonHttpBadRequest(f'Cannot create ability {data["id"]}: at least one executor required') + if 'name' in data and not data.get('name'): + raise JsonHttpBadRequest(f'Cannot create ability {data["id"]} due to missing name') + def _create_ability_filepath(self, tactic: str, obj_id: str): tactic_dir = os.path.join('data', 'abilities', tactic) if not os.path.exists(tactic_dir): diff --git a/app/objects/c_ability.py b/app/objects/c_ability.py index a668b2991..c403d612b 100644 --- a/app/objects/c_ability.py +++ b/app/objects/c_ability.py @@ -17,7 +17,7 @@ class AbilitySchema(ma.Schema): tactic = ma.fields.String(required=True) technique_name = ma.fields.String(missing=None) technique_id = ma.fields.String(missing=None) - name = ma.fields.String(missing=None) + name = ma.fields.String(required=True) description = ma.fields.String(missing=None) executors = ma.fields.List(ma.fields.Nested(ExecutorSchema), required=True) requirements = ma.fields.List(ma.fields.Nested(RequirementSchema), missing=None) @@ -25,7 +25,7 @@ class AbilitySchema(ma.Schema): repeatable = ma.fields.Bool(missing=None) buckets = ma.fields.List(ma.fields.String(), missing=None) additional_info = ma.fields.Dict(keys=ma.fields.String(), values=ma.fields.String()) - access = ma.fields.Nested(AccessSchema, missing=None) + access = ma.fields.Nested(AccessSchema, dump_only=True) singleton = ma.fields.Bool(missing=None) @ma.pre_load @@ -117,9 +117,11 @@ def find_executor(self, name, platform): def find_executors(self, names, platform): """Find executors for matching platform/executor names + Only the first instance of a matching executor will be returned, as there should not be multiple executors matching a single platform/executor name pair. + :param names: Executors to search. ex: ['psh', 'cmd'] :type names: list(str) :param platform: Platform to search. ex: windows @@ -142,6 +144,7 @@ def find_executors(self, names, platform): def add_executor(self, executor): """Add executor to map + If the executor exists, delete the current entry and add the new executor to the bottom for FIFO """ From 61cc27807fa573699811266862b29f11722de734 Mon Sep 17 00:00:00 2001 From: blee Date: Mon, 26 Jul 2021 18:58:39 -0400 Subject: [PATCH 35/58] revert schema changes --- app/objects/c_ability.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/objects/c_ability.py b/app/objects/c_ability.py index c403d612b..be647a565 100644 --- a/app/objects/c_ability.py +++ b/app/objects/c_ability.py @@ -17,7 +17,7 @@ class AbilitySchema(ma.Schema): tactic = ma.fields.String(required=True) technique_name = ma.fields.String(missing=None) technique_id = ma.fields.String(missing=None) - name = ma.fields.String(required=True) + name = ma.fields.String(missing=None) description = ma.fields.String(missing=None) executors = ma.fields.List(ma.fields.Nested(ExecutorSchema), required=True) requirements = ma.fields.List(ma.fields.Nested(RequirementSchema), missing=None) @@ -25,7 +25,7 @@ class AbilitySchema(ma.Schema): repeatable = ma.fields.Bool(missing=None) buckets = ma.fields.List(ma.fields.String(), missing=None) additional_info = ma.fields.Dict(keys=ma.fields.String(), values=ma.fields.String()) - access = ma.fields.Nested(AccessSchema, dump_only=True) + access = ma.fields.Nested(AccessSchema, missing=None) singleton = ma.fields.Bool(missing=None) @ma.pre_load From c7965aa28ec3afa420040acd261d5d027a933556 Mon Sep 17 00:00:00 2001 From: blee Date: Thu, 29 Jul 2021 09:29:08 -0400 Subject: [PATCH 36/58] remove required from tactic/executor fields, reset tests to master --- app/objects/c_ability.py | 4 ++-- app/service/data_svc.py | 25 +++---------------------- tests/objects/test_agent.py | 14 +++++++------- tests/objects/test_link.py | 4 ++-- tests/services/test_planning_svc.py | 4 ++-- tests/services/test_rest_svc.py | 6 +++--- 6 files changed, 19 insertions(+), 38 deletions(-) diff --git a/app/objects/c_ability.py b/app/objects/c_ability.py index be647a565..d4dd8cf1c 100644 --- a/app/objects/c_ability.py +++ b/app/objects/c_ability.py @@ -14,12 +14,12 @@ class AbilitySchema(ma.Schema): ability_id = ma.fields.String() - tactic = ma.fields.String(required=True) + tactic = ma.fields.String(missing=None) technique_name = ma.fields.String(missing=None) technique_id = ma.fields.String(missing=None) name = ma.fields.String(missing=None) description = ma.fields.String(missing=None) - executors = ma.fields.List(ma.fields.Nested(ExecutorSchema), required=True) + executors = ma.fields.List(ma.fields.Nested(ExecutorSchema)) requirements = ma.fields.List(ma.fields.Nested(RequirementSchema), missing=None) privilege = ma.fields.String(missing=None) repeatable = ma.fields.Bool(missing=None) diff --git a/app/service/data_svc.py b/app/service/data_svc.py index 886df59e1..abae5e7cf 100644 --- a/app/service/data_svc.py +++ b/app/service/data_svc.py @@ -15,7 +15,7 @@ from app.objects.c_planner import Planner from app.objects.c_plugin import Plugin from app.objects.c_source import Source -from app.objects.secondclass.c_executor import Executor +from app.objects.secondclass.c_executor import Executor, ExecutorSchema from app.objects.secondclass.c_goal import Goal from app.objects.secondclass.c_parser import Parser from app.objects.secondclass.c_requirement import Requirement @@ -165,7 +165,7 @@ async def load_ability_file(self, filename, access): requirements = await self._load_ability_requirements(ab.pop('requirements', [])) buckets = ab.pop('buckets', [tactic]) if 'executors' in ab: - executors = await self.load_executors_from_list(ab.pop('executors')) + executors = await self.load_executors_from_list(ab.pop('executors', [])) else: executors = await self.load_executors_from_platform_dict(ab.pop('platforms', dict())) ab.pop('access', None) @@ -214,26 +214,7 @@ async def load_executors_from_platform_dict(self, platforms): return executors async def load_executors_from_list(self, executors: list): - output_list = [] - for entry in executors: - name = entry.pop('name', None) - platform = entry.pop('platform', None) - command = entry.pop('command', None) - code = entry.pop('code', None) - language = entry.pop('language', None) - build_target = entry.pop('build_target', None) - payloads = entry.pop('payloads', None) - uploads = entry.pop('uploads', None) - timeout = entry.pop('timeout', None) - parsers = entry.pop('parsers', None) - cleanup = entry.pop('cleanup', None) - variations = entry.pop('variations', None) - additional_info = entry.pop('additional_info', None) - output_list.append(Executor(name=name, platform=platform, command=command, code=code, language=language, - build_target=build_target, payloads=payloads, uploads=uploads, timeout=timeout, - parsers=parsers, cleanup=cleanup, variations=variations, - additional_info=additional_info)) - return output_list + return [ExecutorSchema().load(entry) for entry in executors] async def load_adversary_file(self, filename, access): warnings.warn("Function deprecated and will be removed in a future update. Use load_yaml_file", DeprecationWarning) diff --git a/tests/objects/test_agent.py b/tests/objects/test_agent.py index 4dc4b69ba..a8cf265e5 100644 --- a/tests/objects/test_agent.py +++ b/tests/objects/test_agent.py @@ -10,21 +10,21 @@ class TestAgent: def test_task_no_facts(self, loop, data_svc, obfuscator, init_base_world): executor = Executor(name='psh', platform='windows', command='whoami') - ability = Ability(ability_id='123', tactic='test', executors=[executor]) + ability = Ability(ability_id='123', executors=[executor]) agent = Agent(paw='123', sleep_min=2, sleep_max=8, watchdog=0, executors=['pwsh', 'psh'], platform='windows') loop.run_until_complete(agent.task([ability], obfuscator='plain-text')) assert 1 == len(agent.links) def test_task_missing_fact(self, loop, obfuscator, init_base_world): executor = Executor(name='psh', platform='windows', command='net user #{domain.user.name} /domain') - ability = Ability(ability_id='123', tactic='test', executors=[executor]) + ability = Ability(ability_id='123', executors=[executor]) agent = Agent(paw='123', sleep_min=2, sleep_max=8, watchdog=0, executors=['pwsh', 'psh'], platform='windows') loop.run_until_complete(agent.task([ability], obfuscator='plain-text')) assert 0 == len(agent.links) def test_task_with_facts(self, loop, obfuscator, init_base_world, knowledge_svc): executor = Executor(name='psh', platform='windows', command='net user #{domain.user.name} /domain') - ability = Ability(ability_id='123', tactic='test', executors=[executor]) + ability = Ability(ability_id='123', executors=[executor]) agent = Agent(paw='123', sleep_min=2, sleep_max=8, watchdog=0, executors=['pwsh', 'psh'], platform='windows') fact = Fact(trait='domain.user.name', value='bob') @@ -34,7 +34,7 @@ def test_task_with_facts(self, loop, obfuscator, init_base_world, knowledge_svc) def test_builtin_fact_replacement(self, loop, obfuscator, init_base_world): executor = Executor(name='psh', platform='windows', command='echo #{paw} #{server} #{group} #{location} #{exe_name} #{upstream_dest}') - ability = Ability(ability_id='123', tactic='test', executors=[executor]) + ability = Ability(ability_id='123', executors=[executor]) agent = Agent(paw='123', sleep_min=2, sleep_max=8, watchdog=0, executors=['pwsh', 'psh'], platform='windows', group='my_group', server='http://localhost:8888', location='testlocation', exe_name='testexe') loop.run_until_complete(agent.task([ability], 'plain-text', [])) @@ -47,7 +47,7 @@ def test_builtin_fact_replacement(self, loop, obfuscator, init_base_world): def test_builtin_fact_replacement_with_upstream_dest(self, loop, obfuscator, init_base_world): executor = Executor(name='psh', platform='windows', command='echo #{paw} #{server} #{group} #{location} #{exe_name} #{upstream_dest}') - ability = Ability(ability_id='123', tactic='test', executors=[executor]) + ability = Ability(ability_id='123', executors=[executor]) agent = Agent(paw='123', sleep_min=2, sleep_max=8, watchdog=0, executors=['pwsh', 'psh'], platform='windows', group='my_group', server='http://10.10.10.10:8888', location='testlocation', exe_name='testexe', upstream_dest='http://127.0.0.1:12345') @@ -62,7 +62,7 @@ def test_preferred_executor_psh(self, loop, ability, executor): executor_test = executor(name='test', platform='windows') executor_cmd = executor(name='cmd', platform='windows') executor_psh = executor(name='psh', platform='windows') - test_ability = ability(ability_id='123', tactic='test', executors=[executor_test, executor_cmd, executor_psh]) + test_ability = ability(ability_id='123', executors=[executor_test, executor_cmd, executor_psh]) agent = Agent(paw='123', sleep_min=2, sleep_max=8, watchdog=0, executors=['psh', 'cmd'], platform='windows') @@ -73,7 +73,7 @@ def test_preferred_executor_from_agent_executor(self, loop, ability, executor): executor_test = executor(name='test', platform='windows') executor_cmd = executor(name='cmd', platform='windows') executor_psh = executor(name='psh', platform='windows') - test_ability = ability(ability_id='123', tactic='test', executors=[executor_test, executor_cmd, executor_psh]) + test_ability = ability(ability_id='123', executors=[executor_test, executor_cmd, executor_psh]) agent = Agent(paw='123', sleep_min=2, sleep_max=8, watchdog=0, executors=['cmd', 'test'], platform='windows') diff --git a/tests/objects/test_link.py b/tests/objects/test_link.py index e1a3076b6..2eeb8af3f 100644 --- a/tests/objects/test_link.py +++ b/tests/objects/test_link.py @@ -103,7 +103,7 @@ def test_emit_status_change_event(self, loop, fake_event_svc, ability, executor) def test_link_agent_reported_time_not_present_when_none_roundtrip(self, ability, executor): test_executor = executor(name='psh', platform='windows') - test_ability = ability(ability_id='123', tactic='test') + test_ability = ability(ability_id='123') test_link = Link(command='sc.exe \\dc create sandsvc binpath= "s4ndc4t.exe -originLinkID 111111"', paw='123456', ability=test_ability, executor=test_executor, id=111111) serialized_link = test_link.display @@ -114,7 +114,7 @@ def test_link_agent_reported_time_not_present_when_none_roundtrip(self, ability, def test_link_agent_reported_time_present_when_set_roundtrip(self, ability, executor): test_executor = executor(name='psh', platform='windows') - test_ability = ability(ability_id='123', tactic='test') + test_ability = ability(ability_id='123') test_link = Link(command='sc.exe \\dc create sandsvc binpath= "s4ndc4t.exe -originLinkID 111111"', paw='123456', ability=test_ability, executor=test_executor, id=111111, agent_reported_time=BaseService.get_timestamp_from_string('2021-02-23 11:50:16')) diff --git a/tests/services/test_planning_svc.py b/tests/services/test_planning_svc.py index 72ecbf484..3ed6005f7 100644 --- a/tests/services/test_planning_svc.py +++ b/tests/services/test_planning_svc.py @@ -61,7 +61,7 @@ def __init__(self, **kwargs): @pytest.fixture def setup_planning_test(loop, executor, ability, agent, operation, data_svc, event_svc, init_base_world): texecutor = executor(name='sh', platform='darwin', command='mkdir test', cleanup='rm -rf test') - tability = ability(ability_id='123', tactic='test', executors=[texecutor], repeatable=True, buckets=['test']) + tability = ability(ability_id='123', executors=[texecutor], repeatable=True, buckets=['test']) tagent = agent(sleep_min=1, sleep_max=2, watchdog=0, executors=['sh'], platform='darwin', server='http://127.0.0.1:8000') tsource = Source(id='123', name='test', facts=[], adjustments=[]) toperation = operation(name='test1', agents=[tagent], @@ -71,7 +71,7 @@ def setup_planning_test(loop, executor, ability, agent, operation, data_svc, eve source=tsource) cexecutor = executor(name='sh', platform='darwin', command=test_string, cleanup='whoami') - cability = ability(ability_id='321', tactic='test', executors=[cexecutor], singleton=True) + cability = ability(ability_id='321', executors=[cexecutor], singleton=True) loop.run_until_complete(data_svc.store(tability)) loop.run_until_complete(data_svc.store(cability)) diff --git a/tests/services/test_rest_svc.py b/tests/services/test_rest_svc.py index 103c1f209..d3f8bbb9b 100644 --- a/tests/services/test_rest_svc.py +++ b/tests/services/test_rest_svc.py @@ -22,17 +22,17 @@ def setup_rest_svc_test(loop, data_svc): 'encryption_key': 'ADMIN123', 'exfil_dir': '/tmp'}) loop.run_until_complete(data_svc.store( - Ability(ability_id='123', tactic='test', executors=[ + Ability(ability_id='123', executors=[ Executor(name='psh', platform='windows', command='curl #{app.contact.http}') ]) )) loop.run_until_complete(data_svc.store( - Ability(ability_id='456', tactic='test', executors=[ + Ability(ability_id='456', executors=[ Executor(name='sh', platform='linux', command='whoami') ]) )) loop.run_until_complete(data_svc.store( - Ability(ability_id='789', tactic='test', executors=[ + Ability(ability_id='789', executors=[ Executor(name='sh', platform='linux', command='hostname') ]) )) From fd9b85c63ad4374d9fe3ed31443a0b086e783a44 Mon Sep 17 00:00:00 2001 From: blee Date: Thu, 5 Aug 2021 08:46:03 -0400 Subject: [PATCH 37/58] fix bug when updating v0 file in PATCH request --- app/api/v2/managers/ability_api_manager.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/api/v2/managers/ability_api_manager.py b/app/api/v2/managers/ability_api_manager.py index eae1b57ee..5f432d4a6 100644 --- a/app/api/v2/managers/ability_api_manager.py +++ b/app/api/v2/managers/ability_api_manager.py @@ -7,6 +7,7 @@ from app.api.v2.managers.base_api_manager import BaseApiManager from app.api.v2.responses import JsonHttpBadRequest +from app.objects.c_ability import AbilitySchema from app.utility.base_world import BaseWorld @@ -33,11 +34,11 @@ async def replace_on_disk_object(self, obj: Any, data: dict, ram_key: str, id_pr return next(self.find_objects(ram_key, {id_property: obj_id})) async def update_on_disk_object(self, obj: Any, data: dict, ram_key: str, id_property: str, obj_class: type): - self._validate_ability_data(create=False, data=data) obj_id = getattr(obj, id_property) file_path = await self._get_existing_object_file_path(obj_id, ram_key) - existing_obj_data = dict(self.strip_yml(file_path)[0][0]) + existing_obj_data = AbilitySchema().dump(obj) existing_obj_data.update(data) + self._validate_ability_data(create=False, data=existing_obj_data) if existing_obj_data.get('tactic') not in file_path: await self.remove_object_from_disk_by_id(obj_id, ram_key) file_path = self._create_ability_filepath(data.get('tactic'), obj_id) From 266d2852341e7adb178091a6baea5971fe108834 Mon Sep 17 00:00:00 2001 From: blee Date: Thu, 5 Aug 2021 09:00:18 -0400 Subject: [PATCH 38/58] create functions for v0 ability file conversion --- app/service/data_svc.py | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/app/service/data_svc.py b/app/service/data_svc.py index abae5e7cf..fd513f50f 100644 --- a/app/service/data_svc.py +++ b/app/service/data_svc.py @@ -151,23 +151,14 @@ async def load_ability_file(self, filename, access): name = ab.pop('name', '') description = ab.pop('description', '') tactic = ab.pop('tactic', None) - if 'technique_id' in ab: - technique_id = ab.pop('technique_id') - else: - technique_id = ab.get('technique', dict()).get('attack_id') - if 'technique_name' in ab: - technique_name = ab.pop('technique_name') - else: - technique_name = ab.pop('technique', dict()).get('name') + executors = await self.convert_v0_ability_executor(ab) + technique_id = self.convert_v0_ability_technique_id(ab) + technique_name = self.convert_v0_ability_technique_name(ab) privilege = ab.pop('privilege', None) repeatable = ab.pop('repeatable', False) singleton = ab.pop('singleton', False) requirements = await self._load_ability_requirements(ab.pop('requirements', [])) buckets = ab.pop('buckets', [tactic]) - if 'executors' in ab: - executors = await self.load_executors_from_list(ab.pop('executors', [])) - else: - executors = await self.load_executors_from_platform_dict(ab.pop('platforms', dict())) ab.pop('access', None) if tactic and tactic not in filename: @@ -179,6 +170,24 @@ async def load_ability_file(self, filename, access): repeatable=repeatable, buckets=buckets, access=access, singleton=singleton, **ab) + async def convert_v0_ability_executor(self, ability_data: dict): + """Checks if ability file follows v0 executor format, otherwise assumes v1 ability formatting.""" + if 'platforms' in ability_data: + return await self.load_executors_from_platform_dict(ability_data.pop('platforms', dict())) + return await self.load_executors_from_list(ability_data.pop('executors', [])) + + def convert_v0_ability_technique_name(self, ability_data: dict): + """Checks if ability file follows v0 technique_name format, otherwise assumes v1 ability formatting.""" + if 'technique' in ability_data: + return ability_data.pop('technique', dict()).get('name') + return ability_data.pop('technique_name') + + def convert_v0_ability_technique_id(self, ability_data: dict): + """Checks if ability file follows v0 technique_id format, otherwise assumes v1 ability formatting.""" + if 'technique' in ability_data: + return ability_data.get('technique', dict()).get('attack_id') + return ability_data.pop('technique_id') + async def load_executors_from_platform_dict(self, platforms): executors = [] for platform_names, platform_executors in platforms.items(): From 0249719a30916cf747e4dde5103f1c7829773e7f Mon Sep 17 00:00:00 2001 From: William Booth <18699738+wbooth@users.noreply.github.com> Date: Fri, 13 Aug 2021 09:26:55 -0400 Subject: [PATCH 39/58] remove self-assignment --- app/service/data_svc.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/service/data_svc.py b/app/service/data_svc.py index 62fd779cd..65c182afc 100644 --- a/app/service/data_svc.py +++ b/app/service/data_svc.py @@ -183,8 +183,6 @@ async def load_executors_from_platform_dict(self, platforms): if code_path: _, code_data = await self.get_service('file_svc').read_file(code) code = code_data.decode('utf-8').strip() - else: - code = code language = executor.get('language') build_target = executor.get('build_target') From eccb0d3c5ae87733bd730fd81cd98e8c73a4deac Mon Sep 17 00:00:00 2001 From: William Booth <18699738+wbooth@users.noreply.github.com> Date: Fri, 13 Aug 2021 09:33:39 -0400 Subject: [PATCH 40/58] remove self-assignment --- tests/objects/test_link.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/objects/test_link.py b/tests/objects/test_link.py index 2eeb8af3f..82bbf1615 100644 --- a/tests/objects/test_link.py +++ b/tests/objects/test_link.py @@ -69,7 +69,6 @@ def test_no_status_change_event_fired_when_setting_same_status(self, mock_emit_s executor = executor('psh', 'windows') ability = ability(executor=executor) link = Link(command='net user a', paw='123456', ability=ability, executor=executor, status=-3) - link.status = link.status mock_emit_status_change_method.assert_not_called() @mock.patch.object(Link, '_emit_status_change_event') From 8b316096851ac25e025c2d60021fb42db29b92dc Mon Sep 17 00:00:00 2001 From: William Booth <18699738+wbooth@users.noreply.github.com> Date: Fri, 13 Aug 2021 09:36:52 -0400 Subject: [PATCH 41/58] Update test_link.py --- tests/objects/test_link.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/objects/test_link.py b/tests/objects/test_link.py index 82bbf1615..acfe020e3 100644 --- a/tests/objects/test_link.py +++ b/tests/objects/test_link.py @@ -64,13 +64,6 @@ def test_no_status_change_event_on_instantiation(self, mock_emit_status_change_m Link(command='net user a', paw='123456', ability=ability, executor=executor) mock_emit_status_change_method.assert_not_called() - @mock.patch.object(Link, '_emit_status_change_event') - def test_no_status_change_event_fired_when_setting_same_status(self, mock_emit_status_change_method, ability, executor): - executor = executor('psh', 'windows') - ability = ability(executor=executor) - link = Link(command='net user a', paw='123456', ability=ability, executor=executor, status=-3) - mock_emit_status_change_method.assert_not_called() - @mock.patch.object(Link, '_emit_status_change_event') def test_status_change_event_fired_on_status_change(self, mock_emit_status_change_method, ability, executor): executor = executor('psh', 'windows') From b7dbb22fcbc0ba2fc0742e089fed19b0f0c61dbf Mon Sep 17 00:00:00 2001 From: William Booth <18699738+wbooth@users.noreply.github.com> Date: Fri, 13 Aug 2021 09:45:05 -0400 Subject: [PATCH 42/58] Update adversary_api.py --- app/api/v2/handlers/adversary_api.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/app/api/v2/handlers/adversary_api.py b/app/api/v2/handlers/adversary_api.py index 74a7f6918..2ecbff1b2 100644 --- a/app/api/v2/handlers/adversary_api.py +++ b/app/api/v2/handlers/adversary_api.py @@ -15,12 +15,13 @@ def __init__(self, services): def add_routes(self, app: web.Application): router = app.router + adversaries_by_id_path = '/adversaries/{adversary_id}' router.add_get('/adversaries', self.get_adversaries) - router.add_get('/adversaries/{adversary_id}', self.get_adversary_by_id) + router.add_get(adversaries_by_id, self.get_adversary_by_id) router.add_post('/adversaries', self.create_adversary) - router.add_patch('/adversaries/{adversary_id}', self.update_adversary) - router.add_put('/adversaries/{adversary_id}', self.create_or_update_adversary) - router.add_delete('/adversaries/{adversary_id}', self.delete_adversary) + router.add_patch(adversaries_by_id, self.update_adversary) + router.add_put(adversaries_by_id, self.create_or_update_adversary) + router.add_delete(adversaries_by_id, self.delete_adversary) @aiohttp_apispec.docs(tags=['adversaries']) @aiohttp_apispec.querystring_schema(BaseGetAllQuerySchema) From a528e983de81d988993eae170adc13d6812d6197 Mon Sep 17 00:00:00 2001 From: William Booth <18699738+wbooth@users.noreply.github.com> Date: Fri, 13 Aug 2021 10:12:24 -0400 Subject: [PATCH 43/58] Update adversary_api.py --- app/api/v2/handlers/adversary_api.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/api/v2/handlers/adversary_api.py b/app/api/v2/handlers/adversary_api.py index 2ecbff1b2..50a91735d 100644 --- a/app/api/v2/handlers/adversary_api.py +++ b/app/api/v2/handlers/adversary_api.py @@ -17,11 +17,11 @@ def add_routes(self, app: web.Application): router = app.router adversaries_by_id_path = '/adversaries/{adversary_id}' router.add_get('/adversaries', self.get_adversaries) - router.add_get(adversaries_by_id, self.get_adversary_by_id) + router.add_get(adversaries_by_id_path, self.get_adversary_by_id) router.add_post('/adversaries', self.create_adversary) - router.add_patch(adversaries_by_id, self.update_adversary) - router.add_put(adversaries_by_id, self.create_or_update_adversary) - router.add_delete(adversaries_by_id, self.delete_adversary) + router.add_patch(adversaries_by_id_path, self.update_adversary) + router.add_put(adversaries_by_id_path, self.create_or_update_adversary) + router.add_delete(adversaries_by_id_path, self.delete_adversary) @aiohttp_apispec.docs(tags=['adversaries']) @aiohttp_apispec.querystring_schema(BaseGetAllQuerySchema) From 6f613f9defc4437f46f182a26a64f39e4daa1d4d Mon Sep 17 00:00:00 2001 From: wbooth <18699738+wbooth@users.noreply.github.com> Date: Fri, 13 Aug 2021 10:39:34 -0400 Subject: [PATCH 44/58] standardize time formats --- app/objects/c_agent.py | 4 ++-- app/objects/c_operation.py | 37 ++++++++++++++++++------------------- app/service/contact_svc.py | 2 +- app/utility/base_world.py | 5 +++-- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/app/objects/c_agent.py b/app/objects/c_agent.py index 0da31ba0e..6c2fe23a8 100644 --- a/app/objects/c_agent.py +++ b/app/objects/c_agent.py @@ -43,8 +43,8 @@ class AgentFieldsSchema(ma.Schema): host_ip_addrs = ma.fields.List(ma.fields.String(), allow_none=True) display_name = ma.fields.String(dump_only=True) - created = ma.fields.DateTime(format='%Y-%m-%d %H:%M:%S', dump_only=True) - last_seen = ma.fields.DateTime(format='%Y-%m-%d %H:%M:%S', dump_only=True) + created = ma.fields.DateTime(format=BaseObject.TIME_FORMAT, dump_only=True) + last_seen = ma.fields.DateTime(format=BaseObject.TIME_FORMAT, dump_only=True) links = ma.fields.List(ma.fields.Nested(LinkSchema), dump_only=True) pending_contact = ma.fields.String(dump_only=True) diff --git a/app/objects/c_operation.py b/app/objects/c_operation.py index 7f297eaf5..80810cfb7 100644 --- a/app/objects/c_operation.py +++ b/app/objects/c_operation.py @@ -32,7 +32,7 @@ class OperationSchema(ma.Schema): adversary = ma.fields.Nested(AdversarySchema()) jitter = ma.fields.String() planner = ma.fields.Nested(PlannerSchema()) - start = ma.fields.DateTime(format='%Y-%m-%d %H:%M:%S', dump_only=True) + start = ma.fields.DateTime(format=BaseObject.TIME_FORMAT, dump_only=True) state = ma.fields.String() obfuscator = ma.fields.String() autonomous = ma.fields.Integer() @@ -269,10 +269,10 @@ async def get_skipped_abilities_by_agent(self, data_svc): skipped_abilities.append({agent.paw: list(agent_skipped.values())}) return skipped_abilities - async def report(self, file_svc, data_svc, output=False, redacted=False): + async def report(self, file_svc, data_svc, output=False): try: report = dict(name=self.name, host_group=[a.display for a in self.agents], - start=self.start.strftime('%Y-%m-%d %H:%M:%S'), + start=self.start.strftime(self.TIME_FORMAT), steps=[], finish=self.finish, planner=self.planner.name, adversary=self.adversary.display, jitter=self.jitter, objectives=self.objective.display, facts=[f.display for f in await self.all_facts()]) @@ -281,7 +281,7 @@ async def report(self, file_svc, data_svc, output=False, redacted=False): step_report = dict(link_id=step.id, ability_id=step.ability.ability_id, command=step.command, - delegated=step.decide.strftime('%Y-%m-%d %H:%M:%S'), + delegated=step.decide.strftime(self.TIME_FORMAT), run=step.finish, status=step.status, platform=step.executor.platform, @@ -295,7 +295,7 @@ async def report(self, file_svc, data_svc, output=False, redacted=False): if output and step.output: step_report['output'] = self.decode_bytes(file_svc.read_result_file(step.unique)) if step.agent_reported_time: - step_report['agent_reported_time'] = step.agent_reported_time.strftime('%Y-%m-%d %H:%M:%S') + step_report['agent_reported_time'] = step.agent_reported_time.strftime(self.TIME_FORMAT) agents_steps[step.paw]['steps'].append(step_report) report['steps'] = agents_steps report['skipped_abilities'] = await self.get_skipped_abilities_by_agent(data_svc) @@ -309,20 +309,19 @@ async def event_logs(self, file_svc, data_svc, output=False): return [await self._convert_link_to_event_log(step, file_svc, data_svc, output=output) for step in self.chain if not step.can_ignore()] + async def cede_control_to_planner(self, services): + planner = await self._get_planning_module(services) + await planner.execute() + while not await self.is_closeable(): + await asyncio.sleep(10) + await self.close(services) + async def run(self, services): await self._init_source() - # load objective data_svc = services.get('data_svc') await self._load_objective(data_svc) try: - # Operation cedes control to planner - planner = await self._get_planning_module(services) - await planner.execute() - while not await self.is_closeable(): - await asyncio.sleep(10) - await self.close(services) - - # Automatic event log output + await self.cede_control_to_planner(services) await self.write_event_logs_to_disk(services.get('file_svc'), data_svc, output=True) except Exception as e: logging.error(e, exc_info=True) @@ -348,8 +347,8 @@ async def _load_objective(self, data_svc): async def _convert_link_to_event_log(self, link, file_svc, data_svc, output=False): event_dict = dict(command=link.command, - delegated_timestamp=link.decide.strftime('%Y-%m-%d %H:%M:%S'), - collected_timestamp=link.collect.strftime('%Y-%m-%d %H:%M:%S') if link.collect else None, + delegated_timestamp=link.decide.strftime(self.TIME_FORMAT), + collected_timestamp=link.collect.strftime(self.TIME_FORMAT) if link.collect else None, finished_timestamp=link.finish, status=link.status, platform=link.executor.platform, @@ -362,7 +361,7 @@ async def _convert_link_to_event_log(self, link, file_svc, data_svc, output=Fals if output and link.output: event_dict['output'] = self.decode_bytes(file_svc.read_result_file(link.unique)) if link.agent_reported_time: - event_dict['agent_reported_time'] = link.agent_reported_time.strftime('%Y-%m-%d %H:%M:%S') + event_dict['agent_reported_time'] = link.agent_reported_time.strftime(self.TIME_FORMAT) return event_dict async def _init_source(self): @@ -454,7 +453,7 @@ def _check_reason_skipped(self, agent, ability, op_facts, state, agent_executors def _get_operation_metadata_for_event_log(self): return dict(operation_name=self.name, - operation_start=self.start.strftime('%Y-%m-%d %H:%M:%S'), + operation_start=self.start.strftime(self.TIME_FORMAT), operation_adversary=self.adversary.name) def _emit_state_change_event(self, from_state, to_state): @@ -502,7 +501,7 @@ async def _get_agent_info_for_event_log(agent_paw, data_svc): privilege=agent.privilege, host=agent.host, contact=agent.contact, - created=agent.created.strftime('%Y-%m-%d %H:%M:%S')) + created=agent.created.strftime(BaseObject.TIME_FORMAT)) class Reason(Enum): PLATFORM = 0 diff --git a/app/service/contact_svc.py b/app/service/contact_svc.py index 46869276f..37b00cf34 100644 --- a/app/service/contact_svc.py +++ b/app/service/contact_svc.py @@ -16,7 +16,7 @@ def report(func): async def wrapper(*args, **kwargs): agent, instructions = await func(*args, **kwargs) log = dict(paw=agent.paw, instructions=[BaseWorld.decode_bytes(i.command) for i in instructions], - date=datetime.now().strftime('%Y-%m-%d %H:%M:%S')) + date=BaseWorld.get_current_timestamp()) args[0].report[agent.contact].append(log) return agent, instructions diff --git a/app/utility/base_world.py b/app/utility/base_world.py index 8c51a9c2f..ae599ab65 100644 --- a/app/utility/base_world.py +++ b/app/utility/base_world.py @@ -23,6 +23,7 @@ class BaseWorld: _app_configuration = dict() re_base64 = re.compile('[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}', flags=re.DOTALL) + TIME_FORMAT = '%Y-%m-%d %H:%M:%S' @staticmethod def apply_config(name, config): @@ -78,11 +79,11 @@ def prepend_to_file(filename, line): f.write(line.rstrip('\r\n') + '\n' + content) @staticmethod - def get_current_timestamp(date_format='%Y-%m-%d %H:%M:%S'): + def get_current_timestamp(date_format=TIME_FORMAT): return datetime.now().strftime(date_format) @staticmethod - def get_timestamp_from_string(datetime_str, date_format='%Y-%m-%d %H:%M:%S'): + def get_timestamp_from_string(datetime_str, date_format=TIME_FORMAT): return datetime.strptime(datetime_str, date_format) @staticmethod From de09b7b041c33213428e79e39386af755ba0b318 Mon Sep 17 00:00:00 2001 From: wbooth <18699738+wbooth@users.noreply.github.com> Date: Fri, 13 Aug 2021 10:45:20 -0400 Subject: [PATCH 45/58] time format changes --- app/objects/secondclass/c_fact.py | 2 +- tests/objects/test_operation.py | 22 ++++++++++++---------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/app/objects/secondclass/c_fact.py b/app/objects/secondclass/c_fact.py index df3e88bac..e8a07a00b 100644 --- a/app/objects/secondclass/c_fact.py +++ b/app/objects/secondclass/c_fact.py @@ -44,7 +44,7 @@ class Meta: trait = ma.fields.String(required=True) name = ma.fields.String(dump_only=True) value = ma.fields.Function(lambda x: x.value, deserialize=lambda x: str(x), allow_none=True) - created = ma.fields.DateTime(format='%Y-%m-%d %H:%M:%S', dump_only=True) + created = ma.fields.DateTime(format=BaseObject.TIME_FORMAT, dump_only=True) score = ma.fields.Integer() source = ma.fields.String(allow_none=True) origin_type = ma_enum.EnumField(OriginType, allow_none=True) diff --git a/tests/objects/test_operation.py b/tests/objects/test_operation.py index 456eb3622..32250854a 100644 --- a/tests/objects/test_operation.py +++ b/tests/objects/test_operation.py @@ -18,6 +18,8 @@ from app.objects.secondclass.c_result import Result from app.objects.secondclass.c_fact import Fact +TIME_FORMAT = '%Y-%m-%d %H:%M:%S' + @pytest.fixture def operation_agent(agent): @@ -68,17 +70,17 @@ def op_for_event_logs(operation_agent, operation_adversary, executor, ability, o name='test ability 2', description='test ability 2 desc', executors=[executor_2]) link_1 = operation_link(ability=ability_1, paw=operation_agent.paw, executor=executor_1, command=encoded_command(command_1), status=0, host=operation_agent.host, pid=789, - decide=datetime.strptime('2021-01-01 08:00:00', '%Y-%m-%d %H:%M:%S'), - collect=datetime.strptime('2021-01-01 08:01:00', '%Y-%m-%d %H:%M:%S'), + decide=datetime.strptime('2021-01-01 08:00:00', TIME_FORMAT), + collect=datetime.strptime('2021-01-01 08:01:00', TIME_FORMAT), finish='2021-01-01 08:02:00') link_2 = operation_link(ability=ability_2, paw=operation_agent.paw, executor=executor_2, command=encoded_command(command_2), status=0, host=operation_agent.host, pid=7890, - decide=datetime.strptime('2021-01-01 09:00:00', '%Y-%m-%d %H:%M:%S'), - collect=datetime.strptime('2021-01-01 09:01:00', '%Y-%m-%d %H:%M:%S'), + decide=datetime.strptime('2021-01-01 09:00:00', TIME_FORMAT), + collect=datetime.strptime('2021-01-01 09:01:00', TIME_FORMAT), finish='2021-01-01 09:02:00') discarded_link = operation_link(ability=ability_2, paw=operation_agent.paw, executor=executor_2, command=encoded_command(command_2), status=-2, host=operation_agent.host, pid=7891, - decide=datetime.strptime('2021-01-01 10:00:00', '%Y-%m-%d %H:%M:%S')) + decide=datetime.strptime('2021-01-01 10:00:00', TIME_FORMAT)) op.chain = [link_1, link_2, discarded_link] return op @@ -148,7 +150,7 @@ def op_with_learning_and_seeded(ability, adversary, operation_agent): sc = Source(id='3124', name='test', facts=[Fact(trait='domain.user.name', value='bob')]) op = Operation(id='6789', name='testC', agents=[], adversary=adversary, source=sc, use_learning_parsers=True) # patch operation to make it 'realistic' - op.start = datetime.strptime('2021-01-01 09:00:00', '%Y-%m-%d %H:%M:%S') + op.start = datetime.strptime('2021-01-01 09:00:00', TIME_FORMAT) op.adversary = op.adversary() op.planner = Planner(planner_id='12345', name='test_planner', module='not.an.actual.planner', params=None) @@ -169,8 +171,8 @@ def test_ran_ability_id(self, ability, adversary): def test_event_logs(self, loop, op_for_event_logs, operation_agent, file_svc, data_svc): loop.run_until_complete(data_svc.remove('agents', match=dict(unique=operation_agent.unique))) loop.run_until_complete(data_svc.store(operation_agent)) - start_time = op_for_event_logs.start.strftime('%Y-%m-%d %H:%M:%S') - agent_creation_time = operation_agent.created.strftime('%Y-%m-%d %H:%M:%S') + start_time = op_for_event_logs.start.strftime(TIME_FORMAT) + agent_creation_time = operation_agent.created.strftime(TIME_FORMAT) want_agent_metadata = dict( paw='testpaw', group='red', @@ -239,8 +241,8 @@ def test_writing_event_logs_to_disk(self, loop, op_for_event_logs, operation_age loop.run_until_complete(data_svc.remove('agents', match=dict(unique=operation_agent.unique))) loop.run_until_complete(data_svc.store(operation_agent)) - start_time = op_for_event_logs.start.strftime('%Y-%m-%d %H:%M:%S') - agent_creation_time = operation_agent.created.strftime('%Y-%m-%d %H:%M:%S') + start_time = op_for_event_logs.start.strftime(TIME_FORMAT) + agent_creation_time = operation_agent.created.strftime(TIME_FORMAT) want_agent_metadata = dict( paw='testpaw', group='red', From e853d003dc57e5af8e83638da8c9a8b668dd94ea Mon Sep 17 00:00:00 2001 From: neptunia Date: Fri, 13 Aug 2021 13:08:42 -0400 Subject: [PATCH 46/58] add slack contact - server side --- app/contacts/contact_slack.py | 309 ++++++++++++++++++++++++++++++++++ conf/default.yml | 7 +- 2 files changed, 314 insertions(+), 2 deletions(-) create mode 100644 app/contacts/contact_slack.py diff --git a/app/contacts/contact_slack.py b/app/contacts/contact_slack.py new file mode 100644 index 000000000..d07843a59 --- /dev/null +++ b/app/contacts/contact_slack.py @@ -0,0 +1,309 @@ +import aiohttp +import asyncio +import json +import os +import re +import time + +from base64 import b64encode, b64decode +from collections import defaultdict + +from app.utility.base_world import BaseWorld + + +def api_access(func): + async def process(*args, **kwargs): + async with aiohttp.ClientSession(headers=dict(Authorization='Bearer {}'.format(args[0].key)), + connector=aiohttp.TCPConnector(verify_ssl=False)) as session: + kwargs['session'] = session + return await func(*args, **kwargs) + return process + + +class Contact(BaseWorld): + + class SlackUpload: + def __init__(self, upload_id, filename, num_chunks): + self.upload_id = upload_id + self.filename = filename + self.chunks = [None]*num_chunks + self.required_chunks = num_chunks + self.completed_chunks = 0 + self.exported = False + + def add_chunk(self, chunk_index, contents): + if self.chunks[chunk_index] is None: + self.chunks[chunk_index] = contents + self.completed_chunks += 1 + + def is_complete(self): + return self.completed_chunks == self.required_chunks + + def export_contents(self): + self.exported = True + return b''.join(self.chunks) + + def __init__(self, services): + self.name = 'slack' + self.description = 'Use slack for C2' + self.file_svc = services.get('file_svc') + self.contact_svc = services.get('contact_svc') + self.log = self.create_logger('contact_slack') + self.key = '' + self.channelid = '' + self.botid = '' + + # TODO + # Stores uploaded file chunks. Maps paw to dict that maps upload ID to SlackUpload object + self.pending_uploads = defaultdict(lambda: dict()) + + def retrieve_config(self): + return self.key + + async def start(self): + if await self.valid_config(): + self.key = self.get_config('app.contact.slack') + self.channelid = self.get_config('app.contact.slackchannelid') + self.botid = self.get_config('app.contact.slackbotid') + loop = asyncio.get_event_loop() + loop.create_task(self.slack_operation_loop()) + + async def slack_operation_loop(self): + while True: + await self.handle_beacons(await self.get_results()) + await self.handle_beacons(await self.get_beacons()) + # TODO + await self.handle_uploads(await self.get_uploads()) + await asyncio.sleep(15) + + async def valid_config(self): + return re.compile(pattern='xoxb-[0-9]{13,13}-[0-9]{13,13}-[a-zA-Z0-9]{24,24}').match(self.get_config('app.contact.slack')) + + # TODO: THIS LATER + async def handle_beacons(self, beacons): + """ + Handles various beacons types (beacon and results) + """ + for beacon in beacons: + beacon['contact'] = beacon.get('contact', self.name) + agent, instructions = await self.contact_svc.handle_heartbeat(**beacon) + if 'results' not in beacon: + await self._send_payloads(agent, instructions) + await self._send_instructions(agent, instructions) + + async def get_results(self): + """ + Retrieve all SLACK posted results for a this C2's api key + :return: + """ + try: + # Results are JSON dicts encoded in base64 + s = await self._get_slack_data(comm_type='results') + encoded_json_blobs = [g[0] for g in s] + return [json.loads(self.file_svc.decode_bytes(blob)) for blob in encoded_json_blobs] + except Exception as e: + self.log.error('Retrieving results over c2 (%s) failed: %s' % (self.__class__.__name__, e)) + return [] + + async def get_beacons(self): + """ + Retrieve all SLACK beacons for a particular api key + :return: the beacons + """ + try: + # Beacons are JSON dicts encoded in base64 + s = await self._get_slack_data(comm_type='beacon') + b64_encoded_json_blobs = [g[0] for g in s] + return [json.loads(self.file_svc.decode_bytes(blob)) for blob in b64_encoded_json_blobs] + except Exception as e: + self.log.error('Retrieving beacons over c2 (%s) failed: %s' % (self.__class__.__name__, e)) + return [] + + # TODO + async def handle_uploads(self, upload_slack_info): + for upload in upload_slack_info: + self.log.debug("Handling upload...") + file_contents = upload[0] + metadata = upload[1].split(':') + paw_info = upload[2].split('-') + if len(paw_info) < 2 or len(metadata) < 5: + self.log.error('Parsing SLACK upload data failed. Paw information not provided.') + return + paw = paw_info[1] + upload_id = metadata[1] + filename = self.file_svc.decode_bytes(metadata[2]) + curr_chunk = int(metadata[3]) + num_chunks = int(metadata[4]) + self.log.debug('Received uploaded file chunk %d out of %d for paw %s, upload ID %s, filename %s ' % ( + curr_chunk, num_chunks, paw, upload_id, filename + )) + await self._store_file_chunk(paw, upload_id, filename, file_contents, curr_chunk, num_chunks) + if await self._ready_to_export(paw, upload_id): + self.log.debug('Upload %s complete for paw %s, filename %s' % (upload_id, paw, filename)) + await self._submit_uploaded_file(paw, upload_id) + + # TODO + async def get_uploads(self): + """ + Retrieve all SLACK posted file uploads for this C2's api key + :return: list of (raw content, slack description, slack filename) tuples for upload SLACKs + """ + try: + upload_slacks = await self._get_slack_content(comm_type='upload') + return [(b64decode(g[0]), g[1], g[2]) for g in upload_slacks] + except Exception as e: + self.log.error('Receiving file uploads over c2 (%s) failed: %s' % (self.__class__.__name__, e)) + return [] + + """ PRIVATE """ + + async def _send_instructions(self, agent, instructions): + response = dict(paw=agent.paw, + sleep=await agent.calculate_sleep(), + watchdog=agent.watchdog, + instructions=json.dumps([json.dumps(i.display) for i in instructions])) + if agent.pending_contact != agent.contact: + response['new_contact'] = agent.pending_contact + self.log.debug('Sending agent instructions to switch from C2 channel %s to %s' % (agent.contact, agent.pending_contact)) + await self._post_instructions(self._encode_string(json.dumps(response).encode('utf-8')), agent.paw) + + async def _post_instructions(self, text, paw): + try: + if await self._wait_for_paw(paw, comm_type='instructions'): + return + s = await self._post_slack_message(self._build_slack_message(comm_type='instructions', paw=paw, + data=text)) + return s + except Exception as e: + self.log.warning('Posting instructions over c2 (%s) failed!: %s' % (self.__class__.__name__, e)) + + async def _send_payloads(self, agent, instructions): + for i in instructions: + for p in i.payloads: + filename, payload_contents = await self._get_payload_content(p, agent) + await self._post_payloads(filename, payload_contents, '%s-%s' % (agent.paw, filename)) + + async def _post_payloads(self, filename, payload_contents, paw): + try: + if await self._wait_for_paw(paw, comm_type='payloads'): + return + s = await self._post_slack(self._build_slack_content(comm_type='payloads', paw=paw, files=self._encode_string(payload_contents))) + return s + except Exception as e: + self.log.warning('Posting payload over c2 (%s) failed! %s' % (self.__class__.__name__, e)) + + async def _store_file_chunk(self, paw, upload_id, filename, contents, curr_chunk, total_chunks): + pending_upload = self.pending_uploads[paw].get(upload_id) + if not pending_upload: + # starting brand new upload + pending_upload = self.SlackUpload(upload_id, filename, total_chunks) + self.pending_uploads[paw][upload_id] = pending_upload + pending_upload.add_chunk(curr_chunk - 1, contents) + + async def _ready_to_export(self, paw, upload_id): + pending_upload = self.pending_uploads[paw].get(upload_id) + return pending_upload is not None and pending_upload.is_complete() and not pending_upload.exported + + async def _submit_uploaded_file(self, paw, upload_id): + upload_info = self.pending_uploads[paw].get(upload_id) + if upload_info is not None: + created_dir = os.path.normpath('/' + paw).lstrip('/') + saveto_dir = await self.file_svc.create_exfil_sub_directory(dir_name=created_dir) + unique_filename = ''.join([upload_info.filename, '-', upload_id[0:10]]) + await self.file_svc.save_file(unique_filename, upload_info.export_contents(), saveto_dir) + self.log.debug('Uploaded file %s/%s' % (saveto_dir, upload_info.filename)) + + async def _wait_for_paw(self, paw, comm_type): + for message in await self._get_slack(): + if '{}-{}'.format(comm_type, paw) == message['text'].split(' | ')[0]: + return True + return False + + async def _get_slack_data(self, comm_type): + data = await self._get_raw_slack_data(comm_type=comm_type) + await self._delete_slack_messages(timestamps=[i["ts"] for i in data]) + return [i["text"].split(" | ")[1:] for i in data] + + async def _get_slack_content(self, comm_type): + data = await self._get_raw_slack_data(comm_type=comm_type) + self.log.debug(data) + await self._delete_slack_messages(timestamps=[i["ts"] for i in data]) + return [ + [await self._fetch_content(i["files"][0]["url_private"]), + i["text"].split(" | ")[1], + i["text"].split(" | ")[0]] + for i in data + ] + return [i["text"].split(" | ")[1:] for i in data] + + async def _get_raw_slack_data(self, comm_type): + return [message for message in await self._get_slack() + if (("bot_id" in message and message["bot_id"] == self.botid) and + comm_type in message["text"].split(' | ')[0]) or ( + ("bot_id" not in message) and + comm_type in message["text"].split(' | ')[0])] + + @api_access + async def _get_slack(self, session): + s = json.loads(await self._fetch(session, + 'https://slack.com/api/conversations.history?channel={0}&oldest={1}'.format(self.channelid, int(time.time()-60)))) + return s["messages"] + + async def _get_payload_content(self, payload, beacon): + if payload in self.file_svc.special_payloads: + f = await self.file_svc.special_payloads[payload](dict(file=payload, platform=beacon['platform'])) + return await self.file_svc.read_file(f) + return await self.file_svc.read_file(payload) + + def _build_slack_content(self, comm_type, paw, files): + s = dict(channels=self.channelid, initial_comment='{}-{}'.format(comm_type, paw), content=files) + return s + + def _build_slack_message(self, comm_type, paw, data): + s = dict(channel=self.channelid, text='{}-{} | {}'.format(comm_type, paw, data)) + return s + + def _build_slack_file(self, comm_type, paw, files): + s = dict(channels=self.channelid, initial_comment='{}-{}'.format(comm_type, paw), file=files) + return s + + @api_access + async def _post_slack(self, message_content, session): + return await self._post_form(session, 'https://slack.com/api/files.upload', body=message_content) + + @api_access + async def _post_slack_message(self, message_content, session): + return await self._post(session, 'https://slack.com/api/chat.postMessage', body=message_content) + + @api_access + async def _delete_slack_messages(self, timestamps, session): + for _id in timestamps: + await self._post_form(session, 'https://slack.com/api/chat.delete', dict(channel=self.channelid, ts=_id)) + + @api_access + async def _fetch_content(self, url, session): + return await self._fetch(session, url) + + @staticmethod + async def _delete(session, url): + async with session.delete(url) as response: + return await response.text('ISO-8859-1') + + @staticmethod + async def _fetch(session, url): + async with session.get(url) as response: + return await response.text() + + @staticmethod + async def _post(session, url, body): + async with session.post(url, json=body) as response: + return await response.text() + + @staticmethod + async def _post_form(session, url, body): + async with session.post(url, data=body) as response: + return await response.text() + + @staticmethod + def _encode_string(s): + return str(b64encode(s), 'utf-8') diff --git a/conf/default.yml b/conf/default.yml index 0428a9f4b..6da39a8d8 100644 --- a/conf/default.yml +++ b/conf/default.yml @@ -6,14 +6,18 @@ app.contact.dns.socket: 0.0.0.0:8853 app.contact.gist: API_KEY app.contact.html: /weather app.contact.http: http://0.0.0.0:8888 +app.contact.slack: SLACK_TOKEN +app.contact.slackbotid: SLACK_BOT_ID +app.contact.slackchannelid: SLACK_CHANNEL_ID +app.contact.tcp: 0.0.0.0:7010 app.contact.tunnel.ssh.host_key_file: REPLACE_WITH_KEY_FILE_PATH app.contact.tunnel.ssh.host_key_passphrase: REPLACE_WITH_KEY_FILE_PASSPHRASE app.contact.tunnel.ssh.socket: 0.0.0.0:8022 app.contact.tunnel.ssh.user_name: sandcat app.contact.tunnel.ssh.user_password: s4ndc4t! -app.contact.tcp: 0.0.0.0:7010 app.contact.udp: 0.0.0.0:7011 app.contact.websocket: 0.0.0.0:7012 +auth.login.handler.module: default crypt_salt: REPLACE_WITH_RANDOM_VALUE encryption_key: ADMIN123 exfil_dir: /tmp/caldera @@ -32,7 +36,6 @@ plugins: - training port: 8888 reports_dir: /tmp -auth.login.handler.module: default requirements: go: command: go version From 11fb2bc61055fb4176b50ca3c17911c312485afd Mon Sep 17 00:00:00 2001 From: neptunia Date: Fri, 13 Aug 2021 13:10:51 -0400 Subject: [PATCH 47/58] fix default.yml ordering --- conf/default.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/default.yml b/conf/default.yml index 6da39a8d8..ace42e1b3 100644 --- a/conf/default.yml +++ b/conf/default.yml @@ -9,15 +9,14 @@ app.contact.http: http://0.0.0.0:8888 app.contact.slack: SLACK_TOKEN app.contact.slackbotid: SLACK_BOT_ID app.contact.slackchannelid: SLACK_CHANNEL_ID -app.contact.tcp: 0.0.0.0:7010 app.contact.tunnel.ssh.host_key_file: REPLACE_WITH_KEY_FILE_PATH app.contact.tunnel.ssh.host_key_passphrase: REPLACE_WITH_KEY_FILE_PASSPHRASE app.contact.tunnel.ssh.socket: 0.0.0.0:8022 app.contact.tunnel.ssh.user_name: sandcat app.contact.tunnel.ssh.user_password: s4ndc4t! +app.contact.tcp: 0.0.0.0:7010 app.contact.udp: 0.0.0.0:7011 app.contact.websocket: 0.0.0.0:7012 -auth.login.handler.module: default crypt_salt: REPLACE_WITH_RANDOM_VALUE encryption_key: ADMIN123 exfil_dir: /tmp/caldera @@ -34,6 +33,7 @@ plugins: - sandcat - stockpile - training +auth.login.handler.module: default port: 8888 reports_dir: /tmp requirements: From ff8228092aa6c1d88997402b6404688f912c2fd2 Mon Sep 17 00:00:00 2001 From: neptunia Date: Fri, 13 Aug 2021 13:11:34 -0400 Subject: [PATCH 48/58] fix default.yml ordering --- conf/default.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/default.yml b/conf/default.yml index ace42e1b3..62bebd06c 100644 --- a/conf/default.yml +++ b/conf/default.yml @@ -33,9 +33,9 @@ plugins: - sandcat - stockpile - training -auth.login.handler.module: default port: 8888 reports_dir: /tmp +auth.login.handler.module: default requirements: go: command: go version From 0dc560b542575e251ffc74325e7db4a56a6dad3c Mon Sep 17 00:00:00 2001 From: neptunia Date: Fri, 13 Aug 2021 13:14:42 -0400 Subject: [PATCH 49/58] fix underindent --- app/contacts/contact_slack.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/contacts/contact_slack.py b/app/contacts/contact_slack.py index d07843a59..cf426e558 100644 --- a/app/contacts/contact_slack.py +++ b/app/contacts/contact_slack.py @@ -230,8 +230,8 @@ async def _get_slack_content(self, comm_type): await self._delete_slack_messages(timestamps=[i["ts"] for i in data]) return [ [await self._fetch_content(i["files"][0]["url_private"]), - i["text"].split(" | ")[1], - i["text"].split(" | ")[0]] + i["text"].split(" | ")[1], + i["text"].split(" | ")[0]] for i in data ] return [i["text"].split(" | ")[1:] for i in data] From c4d5b78a5087f53831d2eca5f5c44c0699e8099d Mon Sep 17 00:00:00 2001 From: neptunia Date: Fri, 13 Aug 2021 13:15:39 -0400 Subject: [PATCH 50/58] lol, remove old return --- app/contacts/contact_slack.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/contacts/contact_slack.py b/app/contacts/contact_slack.py index cf426e558..4b915dca1 100644 --- a/app/contacts/contact_slack.py +++ b/app/contacts/contact_slack.py @@ -234,7 +234,6 @@ async def _get_slack_content(self, comm_type): i["text"].split(" | ")[0]] for i in data ] - return [i["text"].split(" | ")[1:] for i in data] async def _get_raw_slack_data(self, comm_type): return [message for message in await self._get_slack() From 34e9d1c4d434ffe1be9e23f7dbcbb999d5edb90c Mon Sep 17 00:00:00 2001 From: William Booth <18699738+wbooth@users.noreply.github.com> Date: Fri, 13 Aug 2021 14:04:59 -0400 Subject: [PATCH 51/58] remove logging of user-controlled data --- app/service/file_svc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/service/file_svc.py b/app/service/file_svc.py index 7bcc48653..e0c7667aa 100644 --- a/app/service/file_svc.py +++ b/app/service/file_svc.py @@ -49,7 +49,7 @@ async def get_file(self, headers): if packer in self.packers: file_path, contents = await self.get_payload_packer(packer).pack(file_path, contents) else: - self.log.warning('packer <%s> not available for payload <%s>, returning unpacked' % (packer, payload)) + self.log.warning('packer not available for payload, returning unpacked') if headers.get('xor_key'): xor_key = headers['xor_key'] contents = xor_bytes(contents, xor_key.encode()) From 419566a755410f5082128c7fc69eaf496372a59d Mon Sep 17 00:00:00 2001 From: neptunia Date: Fri, 13 Aug 2021 14:13:07 -0400 Subject: [PATCH 52/58] fix problems with config field names, remove TODOs --- app/contacts/contact_slack.py | 13 ++++--------- conf/default.yml | 6 +++--- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/app/contacts/contact_slack.py b/app/contacts/contact_slack.py index 4b915dca1..e1cdc0884 100644 --- a/app/contacts/contact_slack.py +++ b/app/contacts/contact_slack.py @@ -53,7 +53,6 @@ def __init__(self, services): self.channelid = '' self.botid = '' - # TODO # Stores uploaded file chunks. Maps paw to dict that maps upload ID to SlackUpload object self.pending_uploads = defaultdict(lambda: dict()) @@ -62,9 +61,9 @@ def retrieve_config(self): async def start(self): if await self.valid_config(): - self.key = self.get_config('app.contact.slack') - self.channelid = self.get_config('app.contact.slackchannelid') - self.botid = self.get_config('app.contact.slackbotid') + self.key = self.get_config('app.contact.slack.api_key') + self.channelid = self.get_config('app.contact.slack.channel_id') + self.botid = self.get_config('app.contact.slack.bot_id') loop = asyncio.get_event_loop() loop.create_task(self.slack_operation_loop()) @@ -72,14 +71,12 @@ async def slack_operation_loop(self): while True: await self.handle_beacons(await self.get_results()) await self.handle_beacons(await self.get_beacons()) - # TODO await self.handle_uploads(await self.get_uploads()) await asyncio.sleep(15) async def valid_config(self): - return re.compile(pattern='xoxb-[0-9]{13,13}-[0-9]{13,13}-[a-zA-Z0-9]{24,24}').match(self.get_config('app.contact.slack')) + return re.compile(pattern='xoxb-[0-9]{13,13}-[0-9]{13,13}-[a-zA-Z0-9]{24,24}').match(self.get_config('app.contact.slack.api_key')) - # TODO: THIS LATER async def handle_beacons(self, beacons): """ Handles various beacons types (beacon and results) @@ -119,7 +116,6 @@ async def get_beacons(self): self.log.error('Retrieving beacons over c2 (%s) failed: %s' % (self.__class__.__name__, e)) return [] - # TODO async def handle_uploads(self, upload_slack_info): for upload in upload_slack_info: self.log.debug("Handling upload...") @@ -142,7 +138,6 @@ async def handle_uploads(self, upload_slack_info): self.log.debug('Upload %s complete for paw %s, filename %s' % (upload_id, paw, filename)) await self._submit_uploaded_file(paw, upload_id) - # TODO async def get_uploads(self): """ Retrieve all SLACK posted file uploads for this C2's api key diff --git a/conf/default.yml b/conf/default.yml index 62bebd06c..8b9d60574 100644 --- a/conf/default.yml +++ b/conf/default.yml @@ -6,9 +6,9 @@ app.contact.dns.socket: 0.0.0.0:8853 app.contact.gist: API_KEY app.contact.html: /weather app.contact.http: http://0.0.0.0:8888 -app.contact.slack: SLACK_TOKEN -app.contact.slackbotid: SLACK_BOT_ID -app.contact.slackchannelid: SLACK_CHANNEL_ID +app.contact.slack.api_key: SLACK_TOKEN +app.contact.slack.bot_id: SLACK_BOT_ID +app.contact.slack.channel_id: SLACK_CHANNEL_ID app.contact.tunnel.ssh.host_key_file: REPLACE_WITH_KEY_FILE_PATH app.contact.tunnel.ssh.host_key_passphrase: REPLACE_WITH_KEY_FILE_PASSPHRASE app.contact.tunnel.ssh.socket: 0.0.0.0:8022 From 27b17408b89bb4d4e375e80b76b56c15428b0aed Mon Sep 17 00:00:00 2001 From: neptunia Date: Fri, 13 Aug 2021 14:18:01 -0400 Subject: [PATCH 53/58] remove debug print --- app/contacts/contact_slack.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/contacts/contact_slack.py b/app/contacts/contact_slack.py index e1cdc0884..48a87c1f7 100644 --- a/app/contacts/contact_slack.py +++ b/app/contacts/contact_slack.py @@ -221,7 +221,6 @@ async def _get_slack_data(self, comm_type): async def _get_slack_content(self, comm_type): data = await self._get_raw_slack_data(comm_type=comm_type) - self.log.debug(data) await self._delete_slack_messages(timestamps=[i["ts"] for i in data]) return [ [await self._fetch_content(i["files"][0]["url_private"]), From 51cba4aeef6f2554a82118cd6d1758dfe0225059 Mon Sep 17 00:00:00 2001 From: William Booth <18699738+wbooth@users.noreply.github.com> Date: Fri, 13 Aug 2021 14:18:16 -0400 Subject: [PATCH 54/58] remove python3.6 tests --- .github/workflows/testing.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 87139b64f..5553a1f78 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -9,8 +9,6 @@ jobs: fail-fast: false matrix: include: - - python-version: 3.6 - toxenv: py36,style,coverage-ci - python-version: 3.7 toxenv: py37,style,coverage-ci - python-version: 3.8 From 8826d006b3a8536420f662784851bf47c8789f8f Mon Sep 17 00:00:00 2001 From: William Booth <18699738+wbooth@users.noreply.github.com> Date: Fri, 13 Aug 2021 14:19:07 -0400 Subject: [PATCH 55/58] Update security.yml --- .github/workflows/security.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index a211353bc..2b1e09e5b 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -9,8 +9,6 @@ jobs: fail-fast: false matrix: include: - - python-version: 3.6 - toxenv: safety - python-version: 3.7 toxenv: safety - python-version: 3.8 From 9e7bd9bd09925995ae35d50c4e6a4e0f903257d4 Mon Sep 17 00:00:00 2001 From: William Booth <18699738+wbooth@users.noreply.github.com> Date: Fri, 13 Aug 2021 14:23:27 -0400 Subject: [PATCH 56/58] Delete codeql-analysis.yml --- .github/workflows/codeql-analysis.yml | 71 --------------------------- 1 file changed, 71 deletions(-) delete mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index ba93f0335..000000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,71 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -name: "CodeQL" - -on: - push: - branches: [master] - pull_request: - # The branches below must be a subset of the branches above - branches: [master] - schedule: - - cron: '0 22 * * 0' - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - # Override automatic language detection by changing the below list - # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] - language: ['python', 'javascript'] - # Learn more... - # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - with: - # We must fetch at least the immediate parents so that if this is - # a pull request then we can checkout the head. - fetch-depth: 2 - - # If this run was triggered by a pull request event, then checkout - # the head of the pull request instead of the merge commit. - - run: git checkout HEAD^2 - if: ${{ github.event_name == 'pull_request' }} - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v1 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v1 - - # ℹī¸ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl - - # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language - - #- run: | - # make bootstrap - # make release - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 From b6fb24462d0b68864191b258abb6119991c8b1f6 Mon Sep 17 00:00:00 2001 From: wbooth <18699738+wbooth@users.noreply.github.com> Date: Fri, 13 Aug 2021 14:24:36 -0400 Subject: [PATCH 57/58] pin plugins to latest version --- plugins/access | 2 +- plugins/atomic | 2 +- plugins/builder | 2 +- plugins/debrief | 2 +- plugins/emu | 2 +- plugins/fieldmanual | 2 +- plugins/human | 2 +- plugins/manx | 2 +- plugins/mock | 2 +- plugins/response | 2 +- plugins/sandcat | 2 +- plugins/ssl | 2 +- plugins/stockpile | 2 +- plugins/training | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) diff --git a/plugins/access b/plugins/access index ee3ab1724..afcb022b6 160000 --- a/plugins/access +++ b/plugins/access @@ -1 +1 @@ -Subproject commit ee3ab172442d706335f0fd73da02f8097a4553e4 +Subproject commit afcb022b6d4e0cbac99297b12f73793f597787ff diff --git a/plugins/atomic b/plugins/atomic index bffeceadd..60f70d197 160000 --- a/plugins/atomic +++ b/plugins/atomic @@ -1 +1 @@ -Subproject commit bffeceaddc77c4c0f066a14b5bda32717b77d939 +Subproject commit 60f70d1970e98196d14452117c7364fb9331fec6 diff --git a/plugins/builder b/plugins/builder index 09b4b8327..afcf2f286 160000 --- a/plugins/builder +++ b/plugins/builder @@ -1 +1 @@ -Subproject commit 09b4b8327a6b5ce2e32bf59bea44131b49763645 +Subproject commit afcf2f286e41f2da8e972050f4ca712d52059088 diff --git a/plugins/debrief b/plugins/debrief index 07a459511..da30c758e 160000 --- a/plugins/debrief +++ b/plugins/debrief @@ -1 +1 @@ -Subproject commit 07a4595114d334c13e231ba443ef7b8a44dac7c9 +Subproject commit da30c758e4097b00f6642600cde33aeb450db538 diff --git a/plugins/emu b/plugins/emu index daed57eb3..cdafa5aa7 160000 --- a/plugins/emu +++ b/plugins/emu @@ -1 +1 @@ -Subproject commit daed57eb3c611515cc44925b393be7a2f556a965 +Subproject commit cdafa5aa79ca03c9ab7171dad591366e18aaeeeb diff --git a/plugins/fieldmanual b/plugins/fieldmanual index 89f19a038..b82117818 160000 --- a/plugins/fieldmanual +++ b/plugins/fieldmanual @@ -1 +1 @@ -Subproject commit 89f19a03898bbf5bcad1eab5faa85e573a9c7b1a +Subproject commit b82117818405785e1fd84f84505ee0c1d3b5c799 diff --git a/plugins/human b/plugins/human index 2d2ce10f7..3622931b6 160000 --- a/plugins/human +++ b/plugins/human @@ -1 +1 @@ -Subproject commit 2d2ce10f7a3ffe7f901fc0a00b096a3fe5d77a4b +Subproject commit 3622931b6e0d25a35c0612667970fe6eb7650a0a diff --git a/plugins/manx b/plugins/manx index 2adb9b55b..949669b8c 160000 --- a/plugins/manx +++ b/plugins/manx @@ -1 +1 @@ -Subproject commit 2adb9b55b99cce4a8aa23d2ea4f15a223615f8b6 +Subproject commit 949669b8c3346077133bd23e242077c22cf93e66 diff --git a/plugins/mock b/plugins/mock index 4cd9368cf..6ffc7e81b 160000 --- a/plugins/mock +++ b/plugins/mock @@ -1 +1 @@ -Subproject commit 4cd9368cf79cb55cf28a35076b29c6f4d7708661 +Subproject commit 6ffc7e81b477372958fda9dda671977c0be669fe diff --git a/plugins/response b/plugins/response index 37b5d6751..ab99a14c4 160000 --- a/plugins/response +++ b/plugins/response @@ -1 +1 @@ -Subproject commit 37b5d67510da3fe9e21fc9fd9c6bbb7797247abd +Subproject commit ab99a14c46a93ab3cbadfd8c57940b7d5161ae0b diff --git a/plugins/sandcat b/plugins/sandcat index af311798e..96fd23ce8 160000 --- a/plugins/sandcat +++ b/plugins/sandcat @@ -1 +1 @@ -Subproject commit af311798e856fb2cfe1c419e92902c14bb62327d +Subproject commit 96fd23ce87cbe165834460f687301e4142bf390f diff --git a/plugins/ssl b/plugins/ssl index 3947c8da5..a9b88ae94 160000 --- a/plugins/ssl +++ b/plugins/ssl @@ -1 +1 @@ -Subproject commit 3947c8da5894b19c04f824a2caf80483c641f2ee +Subproject commit a9b88ae941c507de8046da64d2c7cf037cb2d739 diff --git a/plugins/stockpile b/plugins/stockpile index 3bd714fda..b1dece3f4 160000 --- a/plugins/stockpile +++ b/plugins/stockpile @@ -1 +1 @@ -Subproject commit 3bd714fda444bc72cd39a7ec262ffba923142c46 +Subproject commit b1dece3f470e4fe5d19b7e3dc53a5a0b067fda18 diff --git a/plugins/training b/plugins/training index 0160f026f..e1be8efb3 160000 --- a/plugins/training +++ b/plugins/training @@ -1 +1 @@ -Subproject commit 0160f026f15eea25af1bcc4a8ab0724e16b17ea8 +Subproject commit e1be8efb38894c6db0f11597aec43a80158e1142 From 9f1bb05a6078c011e3c5f6d0c4ed74d869c03356 Mon Sep 17 00:00:00 2001 From: neptunia Date: Fri, 13 Aug 2021 15:40:46 -0400 Subject: [PATCH 58/58] move two shared lines of code to single function --- app/contacts/contact_slack.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/app/contacts/contact_slack.py b/app/contacts/contact_slack.py index 48a87c1f7..8d803f754 100644 --- a/app/contacts/contact_slack.py +++ b/app/contacts/contact_slack.py @@ -214,14 +214,17 @@ async def _wait_for_paw(self, paw, comm_type): return True return False - async def _get_slack_data(self, comm_type): + async def _get_raw_slack_data_and_delete(self, comm_type): data = await self._get_raw_slack_data(comm_type=comm_type) await self._delete_slack_messages(timestamps=[i["ts"] for i in data]) + return data + + async def _get_slack_data(self, comm_type): + data = await self._get_raw_slack_data_and_delete(comm_type=comm_type) return [i["text"].split(" | ")[1:] for i in data] async def _get_slack_content(self, comm_type): - data = await self._get_raw_slack_data(comm_type=comm_type) - await self._delete_slack_messages(timestamps=[i["ts"] for i in data]) + data = await self._get_raw_slack_data_and_delete(comm_type=comm_type) return [ [await self._fetch_content(i["files"][0]["url_private"]), i["text"].split(" | ")[1],