Change directory structure.

- This is a large change that moves most grr code under a grr subdirectory. This is necessary for python package building. - This commit also contains lots of other development/bugfixing work we have been holding off on pushing since we're bumping the version number. Don't use this commit in production, wait for the pending release.
krodyx · Apr 5, 2016 · a1479d8 · a1479d8
1 parent f5adca8
commit a1479d8
Show file tree

Hide file tree

Showing 1,441 changed files with 25,687 additions and 8,459 deletions.
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -0,0 +1,17 @@
+include requirements.txt
+
+recursive-include * *
+recursive-include gui *
+
+recursive-exclude .git *
+recursive-exclude build *
+recursive-exclude dist *
+recursive-exclude vagrant *
+recursive-exclude travis *
+recursive-exclude test_data *
+recursive-exclude executables *.zip
+recursive-exclude executables/windows/installers *.exe
+recursive-exclude executables *.bin
+
+recursive-exclude client/minicomm *
+recursive-exclude * *.pyc
diff --git a/accelerated/accelerated.c b/accelerated/accelerated.c
@@ -158,6 +158,7 @@ PyObject *py_split_buffer(PyObject *self, PyObject *args, PyObject *kwargs) {
   while (length > 0) {
     Py_ssize_t decoded_length = 0;
     unsigned PY_LONG_LONG tag;
+    int tag_type;
 
     // Read the tag off the buffer.
     varint_decode(&tag, buffer, length, &decoded_length);
@@ -168,17 +169,19 @@ PyObject *py_split_buffer(PyObject *self, PyObject *args, PyObject *kwargs) {
     length -= decoded_length;
 
     // Handle the tag depending on its type.
-    int tag_type = tag & TAG_TYPE_MASK;
+    tag_type = tag & TAG_TYPE_MASK;
     switch (tag_type) {
       case WIRETYPE_VARINT: {
         // Decode the varint and position ourselves at the next tag.
         Py_ssize_t tag_length = 0;
+        PyObject *entry = NULL;
+
         varint_decode(&tag, buffer, length, &tag_length);
 
         // Create an entry to add to the result set. Note: We use
         // PyTuple_SetItem which steals the references here instead of
         // PyTuple_Pack which does not (meaning its more involved to use).
-        PyObject *entry = PyTuple_New(3);
+        entry = PyTuple_New(3);
         PyTuple_SET_ITEM(
             entry, 0, encoded_tag);
 
@@ -257,6 +260,8 @@ PyObject *py_split_buffer(PyObject *self, PyObject *args, PyObject *kwargs) {
         // data.
         Py_ssize_t decoded_length = 0;
         unsigned PY_LONG_LONG data_size;
+        PyObject *entry = NULL;
+
         varint_decode(&data_size, buffer, length, &decoded_length);
 
         // Check that we do not exceed the available buffer here.
@@ -267,7 +272,7 @@ PyObject *py_split_buffer(PyObject *self, PyObject *args, PyObject *kwargs) {
           goto error;
         }
 
-        PyObject *entry = PyTuple_New(3);
+        entry = PyTuple_New(3);
         PyTuple_SET_ITEM(
             entry, 0, encoded_tag);
 

diff --git a/client/linux/requirements.txt b/client/linux/requirements.txt
diff --git a/client/windows/requirements.txt b/client/windows/requirements.txt
diff --git a/config/api_acls.yaml b/config/api_acls.yaml
diff --git a/defaults.py b/defaults.py
@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+"""Autogenerated file. Do not edit."""
+CONFIG_FILE = None
diff --git a/executables/windows/GRRNanny_Win32.exe b/executables/windows/GRRNanny_Win32.exe
diff --git a/executables/windows/GRRNanny_x64.exe b/executables/windows/GRRNanny_x64.exe
diff --git a/__init__.py → grr/__init__.py b/__init__.py → grr/__init__.py
diff --git a/grr/artifacts/README.md b/grr/artifacts/README.md
@@ -0,0 +1,13 @@
+# This directory is reserved for external artifacts
+
+The Makefile removes ``*.yaml`` from this directory when syncing the external
+repo located [here] (https://github.com/ForensicArtifacts/artifacts).
+
+## Where artifacts go
+
+- Private artifacts should go in ``artifacts/local``
+- Public artifacts that are non GRR specific should be submitted to the external
+repo.
+- Public artifacts that call GRR functions with ``LIST_FILES``,
+  ``GRR_CLIENT_ACTION``, ``GREP`` etc. should live in
+  ``artifacts/flow_templates``
diff --git a/artifacts/__init__.py → grr/artifacts/__init__.py b/artifacts/__init__.py → grr/artifacts/__init__.py
diff --git a/artifacts/flow_templates/README → grr/artifacts/flow_templates/README b/artifacts/flow_templates/README → grr/artifacts/flow_templates/README
diff --git a/artifacts/flow_templates/cron.yaml → grr/artifacts/flow_templates/cron.yaml b/artifacts/flow_templates/cron.yaml → grr/artifacts/flow_templates/cron.yaml
diff --git a/artifacts/flow_templates/darwin.yaml → grr/artifacts/flow_templates/darwin.yaml b/artifacts/flow_templates/darwin.yaml → grr/artifacts/flow_templates/darwin.yaml
diff --git a/artifacts/flow_templates/disk.yaml → grr/artifacts/flow_templates/disk.yaml b/artifacts/flow_templates/disk.yaml → grr/artifacts/flow_templates/disk.yaml
diff --git a/artifacts/flow_templates/grr_actions.yaml → ...artifacts/flow_templates/grr_actions.yaml b/artifacts/flow_templates/grr_actions.yaml → ...artifacts/flow_templates/grr_actions.yaml
diff --git a/artifacts/flow_templates/linux.yaml → grr/artifacts/flow_templates/linux.yaml b/artifacts/flow_templates/linux.yaml → grr/artifacts/flow_templates/linux.yaml
diff --git a/artifacts/flow_templates/memory.yaml → grr/artifacts/flow_templates/memory.yaml b/artifacts/flow_templates/memory.yaml → grr/artifacts/flow_templates/memory.yaml
diff --git a/artifacts/flow_templates/processes.yaml → grr/artifacts/flow_templates/processes.yaml b/artifacts/flow_templates/processes.yaml → grr/artifacts/flow_templates/processes.yaml
diff --git a/artifacts/flow_templates/stat.yaml → grr/artifacts/flow_templates/stat.yaml b/artifacts/flow_templates/stat.yaml → grr/artifacts/flow_templates/stat.yaml
diff --git a/artifacts/flow_templates/windows.yaml → grr/artifacts/flow_templates/windows.yaml b/artifacts/flow_templates/windows.yaml → grr/artifacts/flow_templates/windows.yaml
diff --git a/artifacts/local/__init__.py → grr/artifacts/local/__init__.py b/artifacts/local/__init__.py → grr/artifacts/local/__init__.py
diff --git a/artifacts/makefile.py → grr/artifacts/makefile.py b/artifacts/makefile.py → grr/artifacts/makefile.py
diff --git a/checks/__init__.py → grr/checks/__init__.py b/checks/__init__.py → grr/checks/__init__.py
diff --git a/checks/cron.yaml → grr/checks/cron.yaml b/checks/cron.yaml → grr/checks/cron.yaml
diff --git a/checks/cron_test.py → grr/checks/cron_test.py b/checks/cron_test.py → grr/checks/cron_test.py
diff --git a/checks/format_test.py → grr/checks/format_test.py b/checks/format_test.py → grr/checks/format_test.py
diff --git a/checks/local/__init__.py → grr/checks/local/__init__.py b/checks/local/__init__.py → grr/checks/local/__init__.py
diff --git a/checks/mounts.yaml → grr/checks/mounts.yaml b/checks/mounts.yaml → grr/checks/mounts.yaml
diff --git a/checks/mounts_test.py → grr/checks/mounts_test.py b/checks/mounts_test.py → grr/checks/mounts_test.py
diff --git a/checks/nfs.yaml → grr/checks/nfs.yaml b/checks/nfs.yaml → grr/checks/nfs.yaml
diff --git a/checks/nfs_test.py → grr/checks/nfs_test.py b/checks/nfs_test.py → grr/checks/nfs_test.py
diff --git a/checks/pam.yaml → grr/checks/pam.yaml b/checks/pam.yaml → grr/checks/pam.yaml
diff --git a/checks/pam_test.py → grr/checks/pam_test.py b/checks/pam_test.py → grr/checks/pam_test.py
diff --git a/checks/paths.yaml → grr/checks/paths.yaml b/checks/paths.yaml → grr/checks/paths.yaml
diff --git a/checks/paths_test.py → grr/checks/paths_test.py b/checks/paths_test.py → grr/checks/paths_test.py
diff --git a/checks/pkg_sources.yaml → grr/checks/pkg_sources.yaml b/checks/pkg_sources.yaml → grr/checks/pkg_sources.yaml
diff --git a/checks/pkg_sources_test.py → grr/checks/pkg_sources_test.py b/checks/pkg_sources_test.py → grr/checks/pkg_sources_test.py
diff --git a/checks/rsyslog.yaml → grr/checks/rsyslog.yaml b/checks/rsyslog.yaml → grr/checks/rsyslog.yaml
diff --git a/checks/rsyslog_test.py → grr/checks/rsyslog_test.py b/checks/rsyslog_test.py → grr/checks/rsyslog_test.py
diff --git a/checks/services.yaml → grr/checks/services.yaml b/checks/services.yaml → grr/checks/services.yaml
diff --git a/checks/services_test.py → grr/checks/services_test.py b/checks/services_test.py → grr/checks/services_test.py
diff --git a/checks/sshd.yaml → grr/checks/sshd.yaml b/checks/sshd.yaml → grr/checks/sshd.yaml
diff --git a/checks/sshd_test.py → grr/checks/sshd_test.py b/checks/sshd_test.py → grr/checks/sshd_test.py
diff --git a/checks/stat.yaml → grr/checks/stat.yaml b/checks/stat.yaml → grr/checks/stat.yaml
diff --git a/checks/stat_test.py → grr/checks/stat_test.py b/checks/stat_test.py → grr/checks/stat_test.py
diff --git a/checks/sysctl.yaml → grr/checks/sysctl.yaml b/checks/sysctl.yaml → grr/checks/sysctl.yaml
diff --git a/checks/sysctl_test.py → grr/checks/sysctl_test.py b/checks/sysctl_test.py → grr/checks/sysctl_test.py
diff --git a/checks/tests.py → grr/checks/tests.py b/checks/tests.py → grr/checks/tests.py
diff --git a/checks/time.yaml → grr/checks/time.yaml b/checks/time.yaml → grr/checks/time.yaml
diff --git a/checks/time_test.py → grr/checks/time_test.py b/checks/time_test.py → grr/checks/time_test.py
diff --git a/checks/unix_login.yaml → grr/checks/unix_login.yaml b/checks/unix_login.yaml → grr/checks/unix_login.yaml
diff --git a/checks/unix_login_test.py → grr/checks/unix_login_test.py b/checks/unix_login_test.py → grr/checks/unix_login_test.py
diff --git a/client/__init__.py → grr/client/__init__.py b/client/__init__.py → grr/client/__init__.py
diff --git a/client/actions.py → grr/client/actions.py b/client/actions.py → grr/client/actions.py
diff --git a/client/client.py → grr/client/client.py b/client/client.py → grr/client/client.py
diff --git a/client/client_actions/__init__.py → grr/client/client_actions/__init__.py b/client/client_actions/__init__.py → grr/client/client_actions/__init__.py
diff --git a/client/client_actions/action_test.py → grr/client/client_actions/action_test.py b/client/client_actions/action_test.py → grr/client/client_actions/action_test.py
@@ -4,6 +4,7 @@
 """Test client actions."""
 
 import __builtin__
+import collections
 import logging
 import os
 import platform
@@ -73,7 +74,8 @@ def cpu_percent(self):
     return 10.0
 
   def memory_info(self):
-    return (100000, 150000)
+    meminfo = collections.namedtuple("Meminfo", ["rss", "vms"])
+    return meminfo(rss=100000, vms=150000)
 
   def memory_percent(self):
     return 10.0

diff --git a/client/client_actions/admin.py → grr/client/client_actions/admin.py b/client/client_actions/admin.py → grr/client/client_actions/admin.py
@@ -252,8 +252,8 @@ def Run(self, arg):
     proc = psutil.Process(os.getpid())
     meminfo = proc.memory_info()
     response = rdf_client.ClientStats(
-        RSS_size=meminfo[0],
-        VMS_size=meminfo[1],
+        RSS_size=meminfo.rss,
+        VMS_size=meminfo.vms,
         memory_percent=proc.memory_percent(),
         bytes_received=stats.STATS.GetMetricValue(
             "grr_client_received_bytes"),

diff --git a/client/client_actions/admin_test.py → grr/client/client_actions/admin_test.py b/client/client_actions/admin_test.py → grr/client/client_actions/admin_test.py
diff --git a/client/client_actions/components.py → grr/client/client_actions/components.py b/client/client_actions/components.py → grr/client/client_actions/components.py
@@ -16,6 +16,8 @@
 from grr.lib.rdfvalues import client as rdf_client
 from grr.lib.rdfvalues import crypto as rdf_crypto
 
+LOADED_COMPONENTS = {}
+
 
 class Site(object):
   """A copy of the relevant functions of the site Python package.
@@ -92,6 +94,12 @@ class LoadComponent(actions.ActionPlugin):
 
   def LoadComponent(self, summary):
     """Import all the required modules as specified in the request."""
+    if (summary.name in LOADED_COMPONENTS and
+        summary.version != LOADED_COMPONENTS[summary.name]):
+      logging.error("Component %s is already loaded at version %s. Exiting!",
+                    summary.name, LOADED_COMPONENTS[summary.name])
+      os._exit(0)  # pylint: disable=protected-access
+
     for mod_name in summary.modules:
       logging.debug("Will import %s", mod_name)
       importlib.import_module(mod_name)
@@ -138,6 +146,7 @@ def Run(self, request):
     # Add the component path to the site packages:
     site = Site()
     site.AddSiteDir(component_path)
+    LOADED_COMPONENTS[summary.name] = summary.version
 
     try:
       self.LoadComponent(summary)
@@ -193,6 +202,7 @@ def Run(self, request):
 
     # Add the component to the site packages:
     site.AddSiteDir(component_path)
+    LOADED_COMPONENTS[component.summary.name] = component.summary.version
 
     # If this does not work now, we just fail.
     self.LoadComponent(summary)

diff --git a/client/client_actions/components_test.py → grr/client/client_actions/components_test.py b/client/client_actions/components_test.py → grr/client/client_actions/components_test.py
@@ -14,7 +14,6 @@
 from grr.lib import aff4
 from grr.lib import config_lib
 from grr.lib import flags
-from grr.lib import maintenance_utils
 from grr.lib import test_lib
 from grr.lib.rdfvalues import client as rdf_client
 
@@ -40,10 +39,7 @@ class TestComponents(test_lib.EmptyActionTest):
 
   def setUp(self):
     super(TestComponents, self).setUp()
-    self.component = maintenance_utils.SignComponent(
-        os.path.join(self.base_path,
-                     "grr-rekall_0.1_glibc_2.4_amd64_Ubuntu_Linux.bin"),
-        token=self.token)
+    self.component = test_lib.WriteComponent(token=self.token)
 
     # The Rekall component will bring in all these new objects. Since the rekall
     # component code is already loaded when the component is re-imported, Rekall

diff --git a/client/client_actions/enrol.py → grr/client/client_actions/enrol.py b/client/client_actions/enrol.py → grr/client/client_actions/enrol.py
diff --git a/client/client_actions/file_fingerprint.py → ...client/client_actions/file_fingerprint.py b/client/client_actions/file_fingerprint.py → ...client/client_actions/file_fingerprint.py
diff --git a/...t/client_actions/file_fingerprint_test.py → ...t/client_actions/file_fingerprint_test.py b/...t/client_actions/file_fingerprint_test.py → ...t/client_actions/file_fingerprint_test.py
diff --git a/client/client_actions/linux/__init__.py → grr/client/client_actions/linux/__init__.py b/client/client_actions/linux/__init__.py → grr/client/client_actions/linux/__init__.py
diff --git a/client/client_actions/linux/ko_patcher.py → ...client/client_actions/linux/ko_patcher.py b/client/client_actions/linux/ko_patcher.py → ...client/client_actions/linux/ko_patcher.py
diff --git a/client/client_actions/linux/linux.py → grr/client/client_actions/linux/linux.py b/client/client_actions/linux/linux.py → grr/client/client_actions/linux/linux.py
diff --git a/client/client_actions/network.py → grr/client/client_actions/network.py b/client/client_actions/network.py → grr/client/client_actions/network.py
diff --git a/client/client_actions/osx/__init__.py → grr/client/client_actions/osx/__init__.py b/client/client_actions/osx/__init__.py → grr/client/client_actions/osx/__init__.py
diff --git a/client/client_actions/osx/osx.py → grr/client/client_actions/osx/osx.py b/client/client_actions/osx/osx.py → grr/client/client_actions/osx/osx.py
diff --git a/client/client_actions/osx/osx_test.py → grr/client/client_actions/osx/osx_test.py b/client/client_actions/osx/osx_test.py → grr/client/client_actions/osx/osx_test.py
diff --git a/client/client_actions/plist.py → grr/client/client_actions/plist.py b/client/client_actions/plist.py → grr/client/client_actions/plist.py
diff --git a/client/client_actions/plist_test.py → grr/client/client_actions/plist_test.py b/client/client_actions/plist_test.py → grr/client/client_actions/plist_test.py
diff --git a/client/client_actions/searching.py → grr/client/client_actions/searching.py b/client/client_actions/searching.py → grr/client/client_actions/searching.py
diff --git a/client/client_actions/searching_test.py → grr/client/client_actions/searching_test.py b/client/client_actions/searching_test.py → grr/client/client_actions/searching_test.py
diff --git a/client/client_actions/standard.py → grr/client/client_actions/standard.py b/client/client_actions/standard.py → grr/client/client_actions/standard.py
@@ -581,7 +581,9 @@ def Run(self, unused_arg):
         pass
 
       try:
-        response.RSS_size, response.VMS_size = proc.memory_info()
+        pmem = proc.memory_info()
+        response.RSS_size = pmem.rss
+        response.VMS_size = pmem.vms
         response.memory_percent = proc.memory_percent()
       except (psutil.NoSuchProcess, psutil.AccessDenied):
         pass

diff --git a/client/client_actions/standard_test.py → grr/client/client_actions/standard_test.py b/client/client_actions/standard_test.py → grr/client/client_actions/standard_test.py
diff --git a/client/client_actions/tempfiles.py → grr/client/client_actions/tempfiles.py b/client/client_actions/tempfiles.py → grr/client/client_actions/tempfiles.py
@@ -40,17 +40,18 @@ class ErrorNotAFile(Error):
   """Attempt to delete a file that doesn't exist."""
 
 
+def GetTempDirForRoot(root):
+  return os.path.join(root, config_lib.CONFIG["Client.grr_tempdir"])
+
+
 def GetDefaultGRRTempDirectory():
   # Check if any of the roots exists.
   for candidate_dir in config_lib.CONFIG["Client.tempdir_roots"]:
     if os.path.isdir(candidate_dir):
-      return os.path.join(
-          candidate_dir, config_lib.CONFIG["Client.grr_tempdir"])
+      return GetTempDirForRoot(candidate_dir)
 
   # If none of the options exist, fall back to the first directory.
-  return os.path.join(
-      config_lib.CONFIG["Client.tempdir_roots"][0],
-      config_lib.CONFIG["Client.grr_tempdir"])
+  return GetTempDirForRoot(config_lib.CONFIG["Client.tempdir_roots"][0])
 
 
 def CreateGRRTempFile(directory=None, filename=None, lifetime=0, mode="w+b",
@@ -197,7 +198,8 @@ def DeleteGRRTempFile(path):
     raise ErrorBadPath("Path must be absolute")
 
   prefix = config_lib.CONFIG["Client.tempfile_prefix"]
-  directories = config_lib.CONFIG["Client.tempdir_roots"]
+  directories = [GetTempDirForRoot(root)
+                 for root in config_lib.CONFIG["Client.tempdir_roots"]]
   if not _CheckIfPathIsValidForDeletion(
       path, prefix=prefix, directories=directories):
     msg = ("Can't delete temp file %s. Filename must start with %s "
@@ -234,13 +236,16 @@ def Run(self, args):
       ErrorBadPath: if path doesn't exist or is not a regular file or directory
     """
 
-    # Normalize the path, so DeleteGRRTempFile can correctly check if
-    # it is within Client.tempdir.
+    allowed_temp_dirs = [GetTempDirForRoot(root)
+                         for root in config_lib.CONFIG["Client.tempdir_roots"]]
+
     if args.path:
+      # Normalize the path, so DeleteGRRTempFile can correctly check if
+      # it is within Client.tempdir.
       paths = [client_utils.CanonicalPathToLocalPath(
           utils.NormalizePath(args.path))]
     else:
-      paths = config_lib.CONFIG["Client.tempdir_roots"]
+      paths = allowed_temp_dirs
 
     deleted = []
     errors = []
@@ -263,7 +268,7 @@ def Run(self, args):
         DeleteGRRTempFile(path)
         deleted = [path]
 
-      elif path not in config_lib.CONFIG["Client.tempdir_roots"]:
+      elif path not in allowed_temp_dirs:
         if not os.path.exists(path):
           raise ErrorBadPath("File %s does not exist" % path)
         else: