Split api.proto into multiple files + minor changes.

* api.proto was split into multiple files. API protobufs are now grouped
semantically into client.proto, hunt.proto, etc.
* CleanClientVersions utility function was added to console_utils to
deal with consequences of having lots of cloned clients.

api_client/python/grr_api_client/client.py

@@ -4,7 +4,9 @@
 from grr_api_client import flow
 from grr_api_client import utils
 from grr_api_client import vfs
-from grr.proto import api_pb2
+from grr.proto.api import client_pb2
+from grr.proto.api import flow_pb2
+from grr.proto.api import user_pb2
 
 
 class ClientApprovalBase(object):
@@ -33,7 +35,7 @@ def __init__(self,
     self._context = context
 
   def Grant(self):
-    args = api_pb2.ApiGrantClientApprovalArgs(
+    args = user_pb2.ApiGrantClientApprovalArgs(
         client_id=self.client_id,
         username=self.username,
         approval_id=self.approval_id)
@@ -48,7 +50,7 @@ class ClientApprovalRef(ClientApprovalBase):
   def Get(self):
     """Fetch and return a proper ClientApproval object."""
 
-    args = api_pb2.ApiGetClientApprovalArgs(
+    args = user_pb2.ApiGetClientApprovalArgs(
         client_id=self.client_id,
         approval_id=self.approval_id,
         username=self.username)
@@ -107,7 +109,7 @@ def CreateFlow(self, name=None, args=None, runner_args=None):
     if not name:
       raise ValueError("name can't be empty")
 
-    request = api_pb2.ApiCreateFlowArgs(client_id=self.client_id)
+    request = flow_pb2.ApiCreateFlowArgs(client_id=self.client_id)
 
     request.flow.name = name
     if runner_args:
@@ -123,7 +125,7 @@ def CreateFlow(self, name=None, args=None, runner_args=None):
   def ListFlows(self):
     """List flows that ran on this client."""
 
-    args = api_pb2.ApiListFlowsArgs(client_id=self.client_id)
+    args = flow_pb2.ApiListFlowsArgs(client_id=self.client_id)
 
     items = self._context.SendIteratorRequest("ListFlows", args)
     return utils.MapItemsIterator(
@@ -151,11 +153,11 @@ def CreateApproval(self,
     if not notified_users:
       raise ValueError("notified_users list can't be empty.")
 
-    approval = api_pb2.ApiClientApproval(
+    approval = user_pb2.ApiClientApproval(
         reason=reason,
         notified_users=notified_users,
         email_cc_addresses=email_cc_addresses or [])
-    args = api_pb2.ApiCreateClientApprovalArgs(
+    args = user_pb2.ApiCreateClientApprovalArgs(
         client_id=self.client_id,
         approval=approval,
         keep_client_alive=keep_client_alive)
@@ -164,8 +166,8 @@ def CreateApproval(self,
     return ClientApproval(
         data=data, username=self._context.username, context=self._context)
 
-  def ListApprovals(self, state=api_pb2.ApiListClientApprovalsArgs.ANY):
-    args = api_pb2.ApiListClientApprovalsArgs(
+  def ListApprovals(self, state=user_pb2.ApiListClientApprovalsArgs.ANY):
+    args = user_pb2.ApiListClientApprovalsArgs(
         client_id=self.client_id, state=state)
     items = self._context.SendIteratorRequest("ListClientApprovals", args)
 
@@ -179,15 +181,15 @@ def AddLabels(self, labels):
     if not labels:
       raise ValueError("labels list can't be empty")
 
-    args = api_pb2.ApiAddClientsLabelsArgs(
+    args = client_pb2.ApiAddClientsLabelsArgs(
         client_ids=[self.client_id], labels=labels)
     self._context.SendRequest("AddClientsLabels", args)
 
   def RemoveLabels(self, labels):
     if not labels:
       raise ValueError("labels list can't be empty")
 
-    args = api_pb2.ApiRemoveClientsLabelsArgs(
+    args = client_pb2.ApiRemoveClientsLabelsArgs(
         client_ids=[self.client_id], labels=labels)
     self._context.SendRequest("RemoveClientsLabels", args)
 
@@ -198,7 +200,7 @@ class ClientRef(ClientBase):
   def Get(self):
     """Fetch client's data and return a proper Client object."""
 
-    args = api_pb2.ApiGetClientArgs(client_id=self.client_id)
+    args = client_pb2.ApiGetClientArgs(client_id=self.client_id)
     result = self._context.SendRequest("GetClient", args)
     return Client(data=result, context=self._context)
 
@@ -220,7 +222,7 @@ def __init__(self, data=None, context=None):
 def SearchClients(query=None, context=None):
   """List clients conforming to a givent query."""
 
-  args = api_pb2.ApiSearchClientsArgs(query=query)
+  args = client_pb2.ApiSearchClientsArgs(query=query)
 
   items = context.SendIteratorRequest("SearchClients", args)
   return utils.MapItemsIterator(lambda data: Client(data=data, context=context),

api_client/python/grr_api_client/connectors/http_connector.py

@@ -9,8 +9,6 @@
 
 from werkzeug import routing
 
-from google.protobuf import wrappers_pb2
-
 from google.protobuf import json_format
 from google.protobuf import symbol_database
 
@@ -20,11 +18,7 @@
 from grr_api_client import errors
 from grr_api_client import utils
 
-from grr.client.components.rekall_support import rekall_pb2
-
-from grr.proto import api_pb2
-from grr.proto import flows_pb2
-from grr.proto import jobs_pb2
+from grr.proto.api import reflection_pb2
 
 logger = logging.getLogger(__name__)
 
@@ -79,16 +73,11 @@ def _FetchRoutingMap(self):
 
     json_str = response.content[len(self.JSON_PREFIX):]
 
-    db = symbol_database.Default()
     # Register descriptors in the database, so that all API-related
     # protos are recognized when Any messages are unpacked.
-    db.RegisterFileDescriptor(api_pb2.DESCRIPTOR)
-    db.RegisterFileDescriptor(flows_pb2.DESCRIPTOR)
-    db.RegisterFileDescriptor(jobs_pb2.DESCRIPTOR)
-    db.RegisterFileDescriptor(wrappers_pb2.DESCRIPTOR)
-    db.RegisterFileDescriptor(rekall_pb2.DESCRIPTOR)
+    utils.RegisterProtoDescriptors(symbol_database.Default())
 
-    proto = api_pb2.ApiListApiMethodsResult()
+    proto = reflection_pb2.ApiListApiMethodsResult()
     json_format.Parse(json_str, proto)
 
     routing_rules = []
@@ -128,8 +117,8 @@ def _GetMethodUrlAndPathParamsNames(self, handler_name, args):
     if args:
       for field, value in args.ListFields():
         if self.handlers_map.is_endpoint_expecting(handler_name, field.name):
-          path_params[field.name] = self._CoerceValueToQueryStringType(field,
-                                                                       value)
+          path_params[field.name] = self._CoerceValueToQueryStringType(
+              field, value)
 
     url = self.urls.build(handler_name, path_params, force_external=True)
 

api_client/python/grr_api_client/flow.py

@@ -2,7 +2,7 @@
 """Flows-related part of GRR API client library."""
 
 from grr_api_client import utils
-from grr.proto import api_pb2
+from grr.proto.api import flow_pb2
 
 
 class FlowResult(object):
@@ -39,18 +39,18 @@ def __init__(self, client_id=None, flow_id=None, context=None):
     self._context = context
 
   def Cancel(self):
-    args = api_pb2.ApiCancelFlowArgs(
+    args = flow_pb2.ApiCancelFlowArgs(
         client_id=self.client_id, flow_id=self.flow_id)
     self._context.SendRequest("CancelFlow", args)
 
   def ListResults(self):
-    args = api_pb2.ApiListFlowResultsArgs(
+    args = flow_pb2.ApiListFlowResultsArgs(
         client_id=self.client_id, flow_id=self.flow_id)
     items = self._context.SendIteratorRequest("ListFlowResults", args)
     return utils.MapItemsIterator(lambda data: FlowResult(data=data), items)
 
   def GetFilesArchive(self):
-    args = api_pb2.ApiGetFlowFilesArchiveArgs(
+    args = flow_pb2.ApiGetFlowFilesArchiveArgs(
         client_id=self.client_id, flow_id=self.flow_id)
     return self._context.SendStreamingRequest("GetFlowFilesArchive", args)
 
@@ -61,7 +61,7 @@ class FlowRef(FlowBase):
   def Get(self):
     """Fetch flow's data and return proper Flow object."""
 
-    args = api_pb2.ApiGetFlowArgs(
+    args = flow_pb2.ApiGetFlowArgs(
         client_id=self.client_id, flow_id=self.flow_id)
     data = self._context.SendRequest("GetFlow", args)
     return Flow(data=data, context=self._context)

api_client/python/grr_api_client/hunt.py

@@ -3,7 +3,7 @@
 
 from grr_api_client import client
 from grr_api_client import utils
-from grr.proto import api_pb2
+from grr.proto.api import hunt_pb2
 
 
 class HuntApprovalBase(object):
@@ -32,7 +32,7 @@ def __init__(self,
     self._context = context
 
   def Grant(self):
-    args = api_pb2.ApiGrantHuntApprovalArgs(
+    args = hunt_pb2.ApiGrantHuntApprovalArgs(
         hunt_id=self.hunt_id,
         username=self.username,
         approval_id=self.approval_id)
@@ -47,7 +47,7 @@ class HuntApprovalRef(HuntApprovalBase):
   def Get(self):
     """Fetch and return a proper HuntApproval object."""
 
-    args = api_pb2.ApiGetHuntApprovalArgs(
+    args = hunt_pb2.ApiGetHuntApprovalArgs(
         hunt_id=self.hunt_id,
         approval_id=self.approval_id,
         username=self.username)
@@ -122,11 +122,11 @@ def CreateApproval(self,
     if not notified_users:
       raise ValueError("notified_users list can't be empty.")
 
-    approval = api_pb2.ApiHuntApproval(
+    approval = hunt_pb2.ApiHuntApproval(
         reason=reason,
         notified_users=notified_users,
         email_cc_addresses=email_cc_addresses or [])
-    args = api_pb2.ApiCreateHuntApprovalArgs(
+    args = hunt_pb2.ApiCreateHuntApprovalArgs(
         hunt_id=self.hunt_id, approval=approval)
 
     data = self._context.SendRequest("CreateHuntApproval", args)
@@ -135,7 +135,7 @@ def CreateApproval(self,
 
   def Modify(self, client_limit=None, client_rate=None, expires=None):
     """Modifies a number of hunt arguments."""
-    args = api_pb2.ApiModifyHuntArgs(hunt_id=self.hunt_id)
+    args = hunt_pb2.ApiModifyHuntArgs(hunt_id=self.hunt_id)
 
     if client_limit is not None:
       args.client_limit = client_limit
@@ -150,25 +150,25 @@ def Modify(self, client_limit=None, client_rate=None, expires=None):
     return Hunt(data=data, context=self._context)
 
   def Start(self):
-    args = api_pb2.ApiModifyHuntArgs(
-        hunt_id=self.hunt_id, state=api_pb2.ApiHunt.STARTED)
+    args = hunt_pb2.ApiModifyHuntArgs(
+        hunt_id=self.hunt_id, state=hunt_pb2.ApiHunt.STARTED)
     data = self._context.SendRequest("ModifyHunt", args)
     return Hunt(data=data, context=self._context)
 
   def Stop(self):
-    args = api_pb2.ApiModifyHuntArgs(
-        hunt_id=self.hunt_id, state=api_pb2.ApiHunt.STOPPED)
+    args = hunt_pb2.ApiModifyHuntArgs(
+        hunt_id=self.hunt_id, state=hunt_pb2.ApiHunt.STOPPED)
     data = self._context.SendRequest("ModifyHunt", args)
     return Hunt(data=data, context=self._context)
 
   def ListResults(self):
-    args = api_pb2.ApiListHuntResultsArgs(hunt_id=self.hunt_id)
+    args = hunt_pb2.ApiListHuntResultsArgs(hunt_id=self.hunt_id)
     items = self._context.SendIteratorRequest("ListHuntResults", args)
     return utils.MapItemsIterator(
         lambda data: HuntResult(data=data, context=self._context), items)
 
   def GetFilesArchive(self):
-    args = api_pb2.ApiGetHuntFilesArchiveArgs(hunt_id=self.hunt_id)
+    args = hunt_pb2.ApiGetHuntFilesArchiveArgs(hunt_id=self.hunt_id)
     return self._context.SendStreamingRequest("GetHuntFilesArchive", args)
 
 
@@ -178,7 +178,7 @@ class HuntRef(HuntBase):
   def Get(self):
     """Fetch hunt's data and return proper Hunt object."""
 
-    args = api_pb2.ApiGetHuntArgs(hunt_id=self.hunt_id)
+    args = hunt_pb2.ApiGetHuntArgs(hunt_id=self.hunt_id)
     data = self._context.SendRequest("GetHunt", args)
     return Hunt(data=data, context=self._context)
 
@@ -221,7 +221,7 @@ def CreateHunt(flow_name=None,
   if not flow_name:
     raise ValueError("flow_name can't be empty")
 
-  request = api_pb2.ApiCreateHuntArgs(flow_name=flow_name)
+  request = hunt_pb2.ApiCreateHuntArgs(flow_name=flow_name)
   if flow_args:
     request.flow_args.value = flow_args.SerializeToString()
     request.flow_args.type_url = utils.GetTypeUrl(flow_args)
@@ -236,16 +236,15 @@ def CreateHunt(flow_name=None,
 def ListHunts(context=None):
   """List all GRR hunts."""
 
-  items = context.SendIteratorRequest("ListHunts", api_pb2.ApiListHuntsArgs())
-  return utils.MapItemsIterator(
-      lambda data: Hunt(data=data, context=context),
-      items)
+  items = context.SendIteratorRequest("ListHunts", hunt_pb2.ApiListHuntsArgs())
+  return utils.MapItemsIterator(lambda data: Hunt(data=data, context=context),
+                                items)
 
 
 def ListHuntApprovals(context=None):
   """List all hunt approvals belonging to requesting user."""
   items = context.SendIteratorRequest("ListHuntApprovals",
-                                      api_pb2.ApiListHuntApprovalsArgs())
+                                      hunt_pb2.ApiListHuntApprovalsArgs())
 
   def MapHuntApproval(data):
     return HuntApproval(data=data, username=context.username, context=context)

api_client/python/grr_api_client/types.py

@@ -3,8 +3,8 @@
 
 from grr_api_client import utils
 
-from grr.proto import api_pb2
 from grr.proto import flows_pb2
+from grr.proto.api import reflection_pb2
 
 
 class Error(Exception):
@@ -40,7 +40,7 @@ def CreateFlowArgs(self, flow_name=None):
     if not self._flow_descriptors:
       self._flow_descriptors = {}
 
-      args = api_pb2.ApiListFlowDescriptorsArgs()
+      args = reflection_pb2.ApiListFlowDescriptorsArgs()
       result = self._context.SendRequest("ListFlowDescriptors", args)
       for item in result.items:
         self._flow_descriptors[item.name] = item

api_client/python/grr_api_client/utils.py

@@ -3,8 +3,27 @@
 
 import itertools
 
+from google.protobuf import wrappers_pb2
+
 from google.protobuf import symbol_database
 
+from grr.client.components.rekall_support import rekall_pb2
+
+from grr.proto import flows_pb2
+from grr.proto import jobs_pb2
+
+from grr.proto.api import artifact_pb2
+from grr.proto.api import client_pb2
+from grr.proto.api import config_pb2
+from grr.proto.api import cron_pb2
+from grr.proto.api import flow_pb2
+from grr.proto.api import hunt_pb2
+from grr.proto.api import output_plugin_pb2
+from grr.proto.api import reflection_pb2
+from grr.proto.api import stats_pb2
+from grr.proto.api import user_pb2
+from grr.proto.api import vfs_pb2
+
 
 class Error(Exception):
   pass
@@ -147,3 +166,26 @@ def UnpackAny(proto_any):
 
   proto_any.Unpack(proto)
   return proto
+
+
+def RegisterProtoDescriptors(db, *additional_descriptors):
+  """Registers all API-releated descriptors in a given symbol DB."""
+  db.RegisterFileDescriptor(artifact_pb2.DESCRIPTOR)
+  db.RegisterFileDescriptor(client_pb2.DESCRIPTOR)
+  db.RegisterFileDescriptor(config_pb2.DESCRIPTOR)
+  db.RegisterFileDescriptor(cron_pb2.DESCRIPTOR)
+  db.RegisterFileDescriptor(flow_pb2.DESCRIPTOR)
+  db.RegisterFileDescriptor(hunt_pb2.DESCRIPTOR)
+  db.RegisterFileDescriptor(output_plugin_pb2.DESCRIPTOR)
+  db.RegisterFileDescriptor(reflection_pb2.DESCRIPTOR)
+  db.RegisterFileDescriptor(stats_pb2.DESCRIPTOR)
+  db.RegisterFileDescriptor(user_pb2.DESCRIPTOR)
+  db.RegisterFileDescriptor(vfs_pb2.DESCRIPTOR)
+
+  db.RegisterFileDescriptor(flows_pb2.DESCRIPTOR)
+  db.RegisterFileDescriptor(jobs_pb2.DESCRIPTOR)
+  db.RegisterFileDescriptor(wrappers_pb2.DESCRIPTOR)
+  db.RegisterFileDescriptor(rekall_pb2.DESCRIPTOR)
+
+  for d in additional_descriptors:
+    db.RegisterFileDescriptor(d)