Virts 412 remove set_id (mitre#592)

* bug fix

* removing set_id

* removing set_id from database

* last of set_id remnants

app/parsers/mimikatz.py

@@ -1,17 +1,14 @@
 
 def mimikatz(blob, **kwargs):
-    set_id = 0
     matched_facts = []
     list_lines = blob.split('\n')
     for i, line in enumerate(list_lines):
         if 'Username' in line and '(null)' not in line:
             value = line.split(':')[1].strip()
             if value[-1] is not '$':
-                username_fact = dict(fact='host.user.name', value=value, set_id=set_id)
+                username_fact = dict(fact='host.user.name', value=value)
                 if 'Password' in list_lines[i + 2] and '(null)' not in list_lines[i + 2]:
-                    password_fact = dict(fact='host.user.password', value=list_lines[i + 2].split(':')[1].strip(),
-                                         set_id=set_id)
+                    password_fact = dict(fact='host.user.password', value=list_lines[i + 2].split(':')[1].strip())
                     matched_facts.append(password_fact)
                     matched_facts.append(username_fact)
-                    set_id += 1
     return matched_facts

app/parsers/standard.py

@@ -12,26 +12,26 @@ def json(parser, blob, log):
             return matched_facts
         if isinstance(structured, (list,)):
             for i, entry in enumerate(structured):
-                matched_facts.append((dict(fact=parser['property'], value=entry.get(parser['script']), set_id=i)))
+                matched_facts.append((dict(fact=parser['property'], value=entry.get(parser['script']))))
         elif isinstance(structured, (dict,)):
             dict_match = parser['script']
             dict_match = dict_match.split(',')
             match = structured
             for d in dict_match:
                 match = match[d]
-            matched_facts.append((dict(fact=parser['property'], value=match, set_id=0)))
+            matched_facts.append((dict(fact=parser['property'], value=match)))
         else:
-            matched_facts.append((dict(fact=parser['property'], value=structured[parser['script']], set_id=0)))
+            matched_facts.append((dict(fact=parser['property'], value=structured[parser['script']])))
     return matched_facts
 
 
 def regex(parser, blob, **kwargs):
     matched_facts = []
     for i, v in enumerate([m for m in re.findall(parser['script'], blob.strip())]):
-        matched_facts.append(dict(fact=parser['property'], value=v, set_id=i))
+        matched_facts.append(dict(fact=parser['property'], value=v))
     return matched_facts
 
 
 def line(parser, blob, **kwargs):
-    return [dict(fact=parser['property'], value=f.strip(), set_id=0) for f in blob.split('\n') if f.strip()]
+    return [dict(fact=parser['property'], value=f.strip()) for f in blob.split('\n') if f.strip()]
 

app/service/agent_svc.py

@@ -134,7 +134,7 @@ async def perform_action(self, link: typing.Dict) -> int:
         operation = (await self.data_svc.dao.get('core_operation', dict(id=op_id)))[0]
         while operation['state'] != operation_svc.op_states['RUNNING']:
             if operation['state'] == operation_svc.op_states['RUN_ONE_LINK']:
-                link_id = await self._create_link(link)
+                link_id = await self.data_svc.create('core_chain', link)
                 await self.data_svc.dao.update('core_operation', 'id', op_id,
                                                dict(state=operation_svc.op_states['PAUSED']))
                 return link_id

app/service/data_svc.py

@@ -201,19 +201,18 @@ async def create_operation(self, name, group, adversary_id, jitter='2/8', source
             await self.dao.create('core_source_map', dict(op_id=op_id, source_id=s_id))
         return op_id
 
-    async def create_fact(self, property, value, source_id, score=1, set_id=0, link_id=None):
+    async def create_fact(self, property, value, source_id, score=1, link_id=None):
         """
         Save a new fact to the database
         :param property:
         :param value:
         :param source_id:
         :param score:
-        :param set_id:
         :param link_id:
         :return: the database id
         """
         return await self.dao.create('core_fact', dict(property=property, value=value, source_id=source_id,
-                                                       score=score, set_id=set_id, link_id=link_id))
+                                                       score=score, link_id=link_id))
 
     async def create_rule(self, fact, source_id, action='DENY', match='.*'):
         """

app/service/parsing_svc.py

@@ -90,11 +90,11 @@ async def _create_host_fact(operation, match, source, result):
                 agents_to_check.append(link['paw'])
         if result['link']['paw'] not in agents_to_check:
             return dict(source_id=source['id'], link_id=result['link_id'], property=match['fact'], value=match['value'],
-                        set_id=match['set_id'], score=1)
+                        score=1)
 
     @staticmethod
     async def _create_global_fact(operation, match, source, result):
         if not any(f['property'] == match['fact'] and f['value'] == match['value'] and f['score'] <= 0 for f in
                    operation['facts']):
             return dict(source_id=source['id'], link_id=result['link_id'], property=match['fact'],
-                        value=match['value'], set_id=match['set_id'], score=1)
+                        value=match['value'], score=1)

app/service/planning_svc.py

@@ -155,12 +155,6 @@ async def _add_test_variants(self, links, agent, operation):
                 link['command'] = self.encode_string(decoded_test)
         return links
 
-    @staticmethod
-    def _reward_fact_relationship(combo_set, combo_link, score):
-        if len(combo_set) == 1 and len(combo_link) == 1:
-            score *= 2
-        return score
-
     @staticmethod
     def _is_fact_bound(fact):
         return not fact['link_id']
@@ -205,14 +199,11 @@ async def _build_single_test_variant(self, copy_test, combo):
         """
         Replace all variables with facts from the combo to build a single test variant
         """
-        score, rewards, combo_set_id, combo_link_id = 0, list(), set(), set()
+        score, rewards = 0, list()
         for var in combo:
             score += (score + var['score'])
             rewards.append(var['id'])
             copy_test = copy_test.replace('#{%s}' % var['property'], var['value'])
-            combo_set_id.add(var['set_id'])
-            combo_link_id.add(var['link_id'])
-        score = self._reward_fact_relationship(combo_set_id, combo_link_id, score)
         return copy_test, score, rewards
 
     async def _get_agent_facts(self, op_id, paw):

conf/core.sql

@@ -8,7 +8,7 @@ CREATE TABLE if not exists core_executor (id integer primary key AUTOINCREMENT,
 CREATE TABLE if not exists core_operation (id integer primary key AUTOINCREMENT, name text, host_group text, adversary_id text, jitter text, start date, finish date, phase integer, autonomous integer, planner integer, state text, allow_untrusted integer);
 CREATE TABLE if not exists core_chain (id integer primary key AUTOINCREMENT, op_id integer, paw text, ability integer, jitter integer, command text, executor text, cleanup integer, score integer, status integer, decide date, collect date, finish date, UNIQUE(op_id, paw, command));
 CREATE TABLE if not exists core_parser (ability integer, name text, property text, script text, UNIQUE(ability, property) ON CONFLICT REPLACE);
-CREATE TABLE if not exists core_fact (id integer primary key AUTOINCREMENT, property text, value text, score integer, set_id integer, source_id text, link_id integer DEFAULT 0);
+CREATE TABLE if not exists core_fact (id integer primary key AUTOINCREMENT, property text, value text, score integer, source_id text, link_id integer DEFAULT 0);
 CREATE TABLE if not exists core_source (id integer primary key AUTOINCREMENT, name text, UNIQUE(name) ON CONFLICT IGNORE);
 CREATE TABLE if not exists core_source_map (id integer primary key AUTOINCREMENT, op_id integer, source_id integer, UNIQUE(op_id, source_id) ON CONFLICT IGNORE);
 CREATE TABLE if not exists core_planner (id integer primary key AUTOINCREMENT, name text, module text, params json, UNIQUE(name) ON CONFLICT IGNORE);