Enable rel-db in docker image, metaclass eradication changes, fix arg…

…ument-parsing bugs in config-updater + other changes.

conftest.py

@@ -5,6 +5,7 @@
 from __future__ import print_function
 from __future__ import unicode_literals
 
+import os
 import sys
 import threading
 import traceback
@@ -40,11 +41,15 @@ def pytest_cmdline_preparse(config, args):
 
 def pytest_cmdline_main(config):
   """A pytest hook that is called when the main function is executed."""
-  del config  # Unused.
 
-  # TODO: `sys.argv` on Python 2 uses `bytes` to represent passed
-  # arguments.
-  sys.argv = [compatibility.NativeStr("pytest")] + test_args
+  if "PYTEST_XDIST_WORKER" in os.environ:
+    # If ran concurrently using pytest-xdist (`-n` cli flag), mainargv is the
+    # result of the execution of pytest_cmdline_main in the main process.
+    sys.argv = config.workerinput["mainargv"]
+  else:
+    # TODO: `sys.argv` on Python 2 uses `bytes` to represent passed
+    # arguments.
+    sys.argv = [compatibility.NativeStr("pytest")] + test_args
 
 
 last_module = None

docker/docker-entrypoint.sh

@@ -16,6 +16,7 @@ initialize() {
     if [[ "${EXTERNAL_HOSTNAME}" ]] && [[ "${ADMIN_PASSWORD}" ]]; then
       grr_config_updater initialize \
         --noprompt \
+        --use_rel_db \
         --external_hostname="${EXTERNAL_HOSTNAME}" \
         --admin_password="${ADMIN_PASSWORD}" \
         --mysql_hostname="${GRR_MYSQL_HOSTNAME}" \

grr/client/grr_response_client/client.py

@@ -74,6 +74,7 @@ def main(unused_args):
 
   if flags.FLAGS.install:
     installer.RunInstaller()
+    sys.exit(0)
 
   errors = config.CONFIG.Validate(["Client", "CA", "Logging"])
 

grr/client/grr_response_client/client_actions/artifact_collector_test.py

@@ -526,7 +526,7 @@ def testWMIArtifact(self, registry):
 
 class TestEchoCmdParser(parser.CommandParser):
 
-  output_types = ["SoftwarePackages"]
+  output_types = [rdf_client.SoftwarePackages]
   supported_artifacts = ["TestEchoCmdArtifact"]
 
   def Parse(self, cmd, args, stdout, stderr, return_val, time_taken,
@@ -545,7 +545,7 @@ def Parse(self, cmd, args, stdout, stderr, return_val, time_taken,
 
 class FakeFileParser(parser.FileParser):
 
-  output_types = ["AttributedDict"]
+  output_types = [rdf_protodict.AttributedDict]
   supported_artifacts = ["FakeFileArtifact"]
 
   def Parse(self, stat, file_obj, knowledge_base):
@@ -564,7 +564,7 @@ def Parse(self, stat, file_obj, knowledge_base):
 
 class FakeFileMultiParser(parser.FileMultiParser):
 
-  output_types = ["AttributedDict"]
+  output_types = [rdf_protodict.AttributedDict]
   supported_artifacts = ["FakeFileArtifact2"]
 
   def ParseMultiple(self, stats, file_objects, knowledge_base):
@@ -587,7 +587,7 @@ def ParseMultiple(self, stats, file_objects, knowledge_base):
 class ParseResponsesTest(client_test_lib.EmptyActionTest):
 
   @mock.patch.object(parsers, "SINGLE_RESPONSE_PARSER_FACTORY",
-                     factory.Factory(parser.SingleResponseParser))
+                     factory.Factory(parsers.SingleResponseParser))
   def testCmdArtifactAction(self):
     """Test the actual client action with parsers."""
     parsers.SINGLE_RESPONSE_PARSER_FACTORY.Register("Cmd", TestEchoCmdParser)
@@ -616,7 +616,7 @@ def testCmdArtifactAction(self):
     self.assertEqual(res.packages[0].description, "1\n")
 
   @mock.patch.object(parsers, "SINGLE_FILE_PARSER_FACTORY",
-                     factory.Factory(parser.SingleFileParser))
+                     factory.Factory(parsers.SingleFileParser))
   def testFakeFileArtifactAction(self):
     """Test collecting a file artifact and parsing the response."""
     parsers.SINGLE_FILE_PARSER_FACTORY.Register("Fake", FakeFileParser)
@@ -642,7 +642,7 @@ def testFakeFileArtifactAction(self):
     self.assertEqual(res.filename, file_path)
 
   @mock.patch.object(parsers, "MULTI_FILE_PARSER_FACTORY",
-                     factory.Factory(parser.MultiFileParser))
+                     factory.Factory(parsers.MultiFileParser))
   def testFakeFileArtifactActionProcessTogether(self):
     """Test collecting a file artifact and parsing the responses together."""
     parsers.MULTI_FILE_PARSER_FACTORY.Register("Fake", FakeFileMultiParser)

grr/client/grr_response_client/client_actions/memory.py

@@ -227,6 +227,9 @@ def DumpProcess(self, psutil_process, args):
             return response
 
           end = start + length
+          # TODO: The filename is parsed on the server side to
+          # extract the memory address again. This should be changed by
+          # saving the `start` and `end` in YaraProcessDumpInformation.
           filename = "%s_%d_%x_%x.tmp" % (psutil_process.name(),
                                           psutil_process.pid, start, end)
           filepath = os.path.join(tmp_dir.path, filename)

grr/client/grr_response_client/client_actions/tempfiles_test.py

@@ -49,7 +49,7 @@ def testCreateAndDelete(self):
     fd = tempfiles.CreateGRRTempFile(filename="process.42.exe", mode="wb")
     fd.close()
     self.assertTrue(os.path.exists(fd.name))
-    self.assertTrue(os.path.basename(fd.name) == "process.42.exe")
+    self.assertEqual(os.path.basename(fd.name), "process.42.exe")
     tempfiles.DeleteGRRTempFile(fd.name)
     self.assertFalse(os.path.exists(fd.name))
 

grr/client/grr_response_client/client_startup.py

@@ -10,7 +10,6 @@
 from grr_response_core.config import contexts
 from grr_response_core.lib import communicator
 from grr_response_core.lib import config_lib
-from grr_response_core.lib import registry
 from grr_response_core.lib.parsers import all as all_parsers
 from grr_response_core.stats import default_stats_collector
 from grr_response_core.stats import stats_collector_instance
@@ -28,7 +27,6 @@ def ClientInit():
 
   client_logging.LogInit()
   all_parsers.Register()
-  registry.Init()
 
   if not config.CONFIG.ContextApplied(contexts.CLIENT_BUILD_CONTEXT):
     config.CONFIG.Persist("Client.labels")

grr/client/grr_response_client/components/chipsec_support/grr_chipsec_test.py

@@ -127,8 +127,8 @@ def testDumpFlashImageUnknownChipsetVerbose(self):
     args = rdf_chipsec_types.DumpFlashImageRequest(log_level=1)
     self.RunAction(self.grr_chipsec_module.DumpFlashImage, args)
     self.assertNotEquals(self.chipsec_mock.logger.logger.call_count, 0)
-    self.assertGreaterEqual(len(self.results), 1)
-    self.assertNotEquals(len(self.results[0].logs), 0)
+    self.assertNotEmpty(self.results)
+    self.assertNotEmpty(self.results[0].logs)
     self.assertEqual(self.results[0].path.path, "")
 
   def testDumpFlashImageOsHelperErrorChipset(self):
@@ -245,8 +245,8 @@ def testDumpACPITableUnknownChipsetVerbose(self):
         table_signature="FACP", logging=True)
     self.RunAction(self.grr_chipsec_module.DumpACPITable, args)
     self.assertNotEquals(self.chipsec_mock.logger.logger.call_count, 0)
-    self.assertGreaterEqual(len(self.results), 1)
-    self.assertNotEquals(len(self.results[0].logs), 0)
+    self.assertNotEmpty(self.results)
+    self.assertNotEmpty(self.results[0].logs)
 
   def testDumpACPITableTriggeringDevMemError(self):
     """Tests the condition where OSError is triggered due to using /dev/mem.
@@ -257,8 +257,8 @@ def testDumpACPITableTriggeringDevMemError(self):
     self.chipsec_mock.acpi.ACPI = MockACPIReadingRestrictedArea
     args = rdf_chipsec_types.DumpACPITableRequest(table_signature="FACP")
     self.RunAction(self.grr_chipsec_module.DumpACPITable, args)
-    self.assertGreaterEqual(len(self.results), 1)
-    self.assertNotEquals(len(self.results[0].logs), 0)
+    self.assertNotEmpty(self.results)
+    self.assertNotEmpty(self.results[0].logs)
 
 
 def main(argv):

grr/client/grr_response_client/grr_fs_client.py

@@ -6,6 +6,7 @@
 from __future__ import unicode_literals
 
 import pdb
+import sys
 
 from absl import app
 from absl import flags
@@ -36,6 +37,7 @@ def main(unused_args):
 
   if flags.FLAGS.install:
     installer.RunInstaller()
+    sys.exit(0)
 
   if not config.CONFIG["Client.fleetspeak_enabled"]:
     raise ValueError(

grr/client/grr_response_client/installer.py

@@ -18,29 +18,20 @@
 
 
 from absl import flags
-from future.utils import with_metaclass
 
 from grr_response_core import config
 from grr_response_core.config import contexts
-from grr_response_core.lib import registry
 
-
-class Installer(
-    with_metaclass(registry.MetaclassRegistry, registry.HookRegistry)):
-  """A GRR installer plugin.
-
-  Modules can register special actions which only run on installation
-  by extending this base class. Execution order is controlled using
-  the same mechanism provided by HookRegistry - i.e. by declaring
-  "pre" and "order" attributes.
-  """
+if sys.platform == "darwin":
+  from grr_response_client.osx import installers  # pylint: disable=g-import-not-at-top
+elif sys.platform == "win32":
+  from grr_response_client.windows import installers  # pylint: disable=g-import-not-at-top
+else:
+  installers = None
 
 
 def RunInstaller():
-  """Run all registered installers.
-
-  Run all the current installers and then exit the process.
-  """
+  """Runs installers for the current platform."""
 
   try:
     os.makedirs(os.path.dirname(config.CONFIG["Installer.logfile"]))
@@ -66,16 +57,8 @@ def RunInstaller():
   config.CONFIG.AddContext(contexts.INSTALLER_CONTEXT,
                            "Context applied when we run the client installer.")
 
-  logging.warning("Starting installation procedure for GRR client.")
-  try:
-    Installer().Init()
-  except Exception as e:  # pylint: disable=broad-except
-    # Ouch! we failed to install... Not a lot we can do
-    # here - just log the error and give up.
-    logging.exception("Installation failed: %s", e)
-
-    # Error return status.
-    sys.exit(-1)
-
-  # Exit successfully.
-  sys.exit(0)
+  if installers is None:
+    logging.info("No installers found for %s.", sys.platform)
+  else:
+    logging.info("Starting installation procedure for GRR client.")
+    installers.Run()

grr/client/grr_response_client/osx/installers.py

@@ -8,67 +8,60 @@
 import os
 import zipfile
 
-from grr_response_client import installer
 from grr_response_core import config
 from grr_response_core.lib import config_lib
 from grr_response_core.lib import type_info
 
 
-class OSXInstaller(installer.Installer):
-  """Tries to find an existing certificate and copies it to the config."""
+def Run():
+  """Extracts a config file from the .pkg file."""
+  logging.info("Extracting config file from .pkg.")
+  pkg_path = os.environ.get("PACKAGE_PATH", None)
+  if pkg_path is None:
+    logging.error("Could not locate package, giving up.")
+    return
 
-  def ExtractConfig(self):
-    """This installer extracts a config file from the .pkg file."""
-    logging.info("Extracting config file from .pkg.")
-    pkg_path = os.environ.get("PACKAGE_PATH", None)
-    if pkg_path is None:
-      logging.error("Could not locate package, giving up.")
-      return
+  zf = zipfile.ZipFile(pkg_path, mode="r")
+  fd = zf.open("config.yaml")
+  install_dir = os.path.dirname(config.CONFIG.parser.filename)
 
-    zf = zipfile.ZipFile(pkg_path, mode="r")
-    fd = zf.open("config.yaml")
-    install_dir = os.path.dirname(config.CONFIG.parser.filename)
+  # We write this config to disk so that Initialize can find the build.yaml
+  # referenced inside the config as a relative path. This config isn't used
+  # after install time.
+  installer_config = os.path.join(install_dir, "installer_config.yaml")
+  with open(installer_config, "wb") as f:
+    f.write(fd.read())
 
-    # We write this config to disk so that Intialize can find the build.yaml
-    # referenced inside the config as a relative path. This config isn't used
-    # after install time.
-    installer_config = os.path.join(install_dir, "installer_config.yaml")
-    with open(installer_config, "wb") as f:
-      f.write(fd.read())
+  packaged_config = config.CONFIG.MakeNewConfig()
+  packaged_config.Initialize(
+      filename=installer_config, parser=config_lib.YamlParser)
 
-    packaged_config = config.CONFIG.MakeNewConfig()
-    packaged_config.Initialize(
-        filename=installer_config, parser=config_lib.YamlParser)
+  new_config = config.CONFIG.MakeNewConfig()
+  new_config.SetWriteBack(config.CONFIG["Config.writeback"])
 
-    new_config = config.CONFIG.MakeNewConfig()
-    new_config.SetWriteBack(config.CONFIG["Config.writeback"])
+  for info in config.CONFIG.type_infos:
+    try:
+      new_value = packaged_config.GetRaw(info.name, None)
+    except type_info.TypeValueError:
+      continue
 
-    for info in config.CONFIG.type_infos:
-      try:
-        new_value = packaged_config.GetRaw(info.name, None)
-      except type_info.TypeValueError:
-        continue
-
-      try:
-        old_value = config.CONFIG.GetRaw(info.name, None)
+    try:
+      old_value = config.CONFIG.GetRaw(info.name, None)
 
-        if not new_value or new_value == old_value:
-          continue
-      except type_info.TypeValueError:
-        pass
-
-      new_config.SetRaw(info.name, new_value)
+      if not new_value or new_value == old_value:
+        continue
+    except type_info.TypeValueError:
+      pass
 
-    new_config.Write()
-    logging.info("Config file extracted successfully.")
+    new_config.SetRaw(info.name, new_value)
 
-    logging.info("Extracting additional files.")
+  new_config.Write()
+  logging.info("Config file extracted successfully.")
 
-    for zinfo in zf.filelist:
-      basename = os.path.basename(zinfo.filename)
-      if basename != "config.yaml":
-        with open(os.path.join(install_dir, basename), "wb") as f:
-          f.write(zf.open(zinfo.filename).read())
+  logging.info("Extracting additional files.")
 
-  def Run(self):
-    self.ExtractConfig()
+  for zinfo in zf.filelist:
+    basename = os.path.basename(zinfo.filename)
+    if basename != "config.yaml":
+      with open(os.path.join(install_dir, basename), "wb") as f:
+        f.write(zf.open(zinfo.filename).read())

grr/client/grr_response_client/osx/registry_init.py

@@ -5,4 +5,3 @@
 from __future__ import unicode_literals
 
 # These need to register plugins so, pylint: disable=unused-import
-from grr_response_client.osx import installers