converting private facts into host facts (mitre#431)

app/service/file_svc.py

@@ -67,9 +67,9 @@ async def _compile(self, name, platform):
                 plugin, file_path = await self.find_file_path(name)
                 await self._change_file_hash(file_path)
                 output = 'plugins/%s/payloads/%s-%s' % (plugin, name, platform)
+                os.system('GOOS=%s go build -o %s -ldflags="-s -w" %s' % (platform, output, file_path))
                 self.log.debug('%s compiled for %s with MD5=%s' %
                                (name, platform, md5(open(output, 'rb').read()).hexdigest()))
-                os.system('GOOS=%s go build -o %s -ldflags="-s -w" %s' % (platform, output, file_path))
             return '%s-%s' % (name, platform)
         return name
 

app/service/parsing_svc.py

@@ -4,6 +4,7 @@
 
 from app.service.base_service import BaseService
 
+
 class ParsingService(BaseService):
 
     def __init__(self):
@@ -12,29 +13,47 @@ def __init__(self):
     async def parse_facts(self, operation):
         data_svc = self.get_service('data_svc')
         results = await data_svc.explode_results()
-        op_source = await data_svc.explode_sources(dict(name=operation['name']))
-        for x in [r for r in results if not r['parsed']]:
-            parser = await data_svc.explode_parsers(dict(ability=x['link']['ability']))
-            if parser and x['link']['status']==0:
+        for result in [r for r in results if not r['parsed']]:
+            parser = await data_svc.explode_parsers(dict(ability=result['link']['ability']))
+            if parser and result['link']['status'] == 0:
                 if parser[0]['name'] == 'json':
-                    matched_facts = parsers.json(parser[0], b64decode(x['output']).decode('utf-8'), self.log)
+                    matched_facts = parsers.json(parser[0], b64decode(result['output']).decode('utf-8'), self.log)
                 elif parser[0]['name'] == 'line':
-                    matched_facts = parsers.line(parser[0], b64decode(x['output']).decode('utf-8'), self.log)
+                    matched_facts = parsers.line(parser[0], b64decode(result['output']).decode('utf-8'), self.log)
                 elif parser[0]['name'] == 'mimikatz':
-                    matched_facts = mimikatz_parser.mimikatz(b64decode(x['output']).decode('utf-8'), self.log)
+                    matched_facts = mimikatz_parser.mimikatz(b64decode(result['output']).decode('utf-8'), self.log)
                 else:
-                    matched_facts = parsers.regex(parser[0], b64decode(x['output']).decode('utf-8'), self.log)
+                    matched_facts = parsers.regex(parser[0], b64decode(result['output']).decode('utf-8'), self.log)
 
-                # save facts to DB
+                source = (await data_svc.explode_sources(dict(name=operation['name'])))[0]
                 for match in matched_facts:
-                    if not any(f['property'] == match['fact'] and f['value'] == match['value'] and f['score'] <= 0 for f in
-                               operation['facts']):
-                        await data_svc.create_fact(
-                            source_id=op_source[0]['id'], link_id=x['link_id'], property=match['fact'],
-                            value=match['value'], set_id=match['set_id'], score=1
-                        )
-
-                # mark result as parsed
+                    operation = (await data_svc.explode_operation(dict(id=operation['id'])))[0]
+                    if match['fact'].startswith('host'):
+                        fact = await self._create_host_fact(operation, match, source, result)
+                    else:
+                        fact = await self._create_global_fact(operation, match, source, result)
+                    if fact:
+                        await data_svc.create_fact(**fact)
+
                 update = dict(parsed=self.get_current_timestamp())
-                await data_svc.update('core_result', key='link_id', value=x['link_id'], data=update)
+                await data_svc.update('core_result', key='link_id', value=result['link_id'], data=update)
+
+    """ PRIVATE """
+
+    @staticmethod
+    async def _create_host_fact(operation, match, source, result):
+        already_stashed = [f for f in operation['facts'] if f['property'] == match['fact'] and f['value'] == match['value'] and f['score'] > 0]
+        agents_to_check = []
+        for fact in already_stashed:
+            link = next((lnk for lnk in operation['chain'] if lnk['id'] == fact['link_id']), False)
+            agents_to_check.append(link['paw'])
+        if result['link']['paw'] not in agents_to_check:
+            return dict(source_id=source['id'], link_id=result['link_id'], property=match['fact'], value=match['value'],
+                        set_id=match['set_id'], score=1)
 
+    @staticmethod
+    async def _create_global_fact(operation, match, source, result):
+        if not any(f['property'] == match['fact'] and f['value'] == match['value'] and f['score'] <= 0 for f in
+                   operation['facts']):
+            return dict(source_id=source['id'], link_id=result['link_id'], property=match['fact'],
+                        value=match['value'], set_id=match['set_id'], score=1)

app/service/planning_svc.py

@@ -107,13 +107,12 @@ async def _build_relevant_facts(variables, facts, agent_facts):
         relevant_facts = []
         for v in variables:
             variable_facts = []
-            for fact in facts:
-                if fact['property'] == v:
-                    if 'private' in fact['property']:
-                        if fact['id'] in agent_facts:
-                            variable_facts.append(fact)
-                    else:
+            for fact in [f for f in facts if f['property'] == v]:
+                if fact['property'].startswith('host'):
+                    if fact['id'] in agent_facts:
                         variable_facts.append(fact)
+                else:
+                    variable_facts.append(fact)
             relevant_facts.append(variable_facts)
         return relevant_facts
 

conf/core.sql

@@ -7,7 +7,7 @@ CREATE TABLE if not exists core_agent (id integer primary key AUTOINCREMENT, paw
 CREATE TABLE if not exists core_operation (id integer primary key AUTOINCREMENT, name text, host_group text, adversary_id text, jitter text, start date, finish date, phase integer, stealth integer, planner integer, state text);
 CREATE TABLE if not exists core_chain (id integer primary key AUTOINCREMENT, op_id integer, paw text, ability integer, jitter integer, command text, cleanup integer, score integer, status integer, decide date, collect date, finish date, UNIQUE(op_id, paw, command));
 CREATE TABLE if not exists core_parser (id integer primary key AUTOINCREMENT, ability integer, name text, property text, script text, UNIQUE(ability, property) ON CONFLICT IGNORE);
-CREATE TABLE if not exists core_fact (id integer primary key AUTOINCREMENT, property text, value text, score integer, set_id integer, source_id text, link_id integer, UNIQUE(source_id, property, value) ON CONFLICT IGNORE);
+CREATE TABLE if not exists core_fact (id integer primary key AUTOINCREMENT, property text, value text, score integer, set_id integer, source_id text, link_id integer);
 CREATE TABLE if not exists core_source (id integer primary key AUTOINCREMENT, name text, UNIQUE(name) ON CONFLICT IGNORE);
 CREATE TABLE if not exists core_source_map (id integer primary key AUTOINCREMENT, op_id integer, source_id integer, UNIQUE(op_id, source_id) ON CONFLICT IGNORE);
 CREATE TABLE if not exists core_planner (id integer primary key AUTOINCREMENT, name text, module text, params json, UNIQUE(name) ON CONFLICT IGNORE);