Fixing Travis/AppVeyor + better Colab error handling.

This change removes the experimental hacks done while debugging
Travis/AppVeyor problems.

Also: bumped the pyinstaller version.

.travis.yml

@@ -47,7 +47,6 @@ matrix:
 
     # OSX builds
     - os: osx
-      osx_image: xcode10.1
       env:
         - GCS_TAG=osx
       before_install:

appveyor/tests/appveyor.yml

@@ -7,8 +7,8 @@ image: ubuntu
 environment:
   CHROME_DEB: google-chrome-stable_current_amd64.deb
 
-#cache:
-# - ${HOME}/.cache/pip
+cache:
+ - ${HOME}/.cache/pip
 
 install:
 - lsb_release -a

grr/client/setup.py

@@ -89,13 +89,7 @@ def make_release_tree(self, base_dir, files):
         # TODO: This is a backport of Python 3.2+ API, should be
         # removed once support for Python 2 is dropped.
         "subprocess32==3.5.3",
-        # TODO: 3.4 has a bug that prevents it from being installed
-        # on macOS and CentOS [1]. On the other hand, 3.2.1 does not work with
-        # Python 3. The issue is already resolved but there has been no release
-        # since then.
-        #
-        # [1]: https://github.com/pyinstaller/pyinstaller/issues/3597
-        "pyinstaller==%s" % ("3.2.1" if sys.version_info < (3, 0) else "3.4"),
+        "pyinstaller==3.5",
     ],
     extras_require={
         # The following requirements are needed in Windows.

grr/client_builder/setup.py

@@ -78,13 +78,7 @@ def make_release_tree(self, base_dir, files):
         "absl-py==0.6.1",
         "grr-response-client==%s" % VERSION.get("Version", "packagedepends"),
         "grr-response-core==%s" % VERSION.get("Version", "packagedepends"),
-        # TODO: 3.4 has a bug that prevents it from being installed
-        # on macOS and CentOS [1]. On the other hand, 3.2.1 does not work with
-        # Python 3. The issue is already resolved but there has been no release
-        # since then.
-        #
-        # [1]: https://github.com/pyinstaller/pyinstaller/issues/3597
-        "pyinstaller==%s" % ("3.2.1" if sys.version_info < (3, 0) else "3.4"),
+        "pyinstaller==3.5",
     ],
 
     # Data files used by GRR. Access these via the config_lib "resource" filter.

grr/proto/setup.py

@@ -85,7 +85,7 @@ def find_all_modules(self):
     return build_py.find_all_modules(self)
 
 
-class Develope(develop):
+class Develop(develop):
 
   def run(self):
     compile_protos()
@@ -107,7 +107,7 @@ def make_release_tree(self, base_dir, files):
   def run(self):
     compile_protos()
     sdist.run(self)
-    
+
 
 VERSION = get_config()
 
@@ -121,6 +121,7 @@ def run(self):
     url="https://github.com/google/grr/tree/master/proto",
     cmdclass={
         "build_py": Build,
+        "develop": Develop,
         "sdist": Sdist,
     },
     packages=find_packages(),

grr/server/grr_response_server/gui/api_plugins/vfs.py

@@ -80,7 +80,32 @@ def __init__(self, client_id, path_type, components):
 
 
 class FileContentNotFoundError(api_call_handler_base.ResourceNotFoundError):
-  """Raised when the content for a specific file could not be found."""
+  """Raised when the content for a specific file could not be found.
+
+  Attributes:
+    client_id: An id of the client for which the file content was not found.
+    path_type: A type of the path for which the file content was not found.
+    components: Components of the path for which the file content was not found.
+    timestamp: A timestamp of the file which content was not found.
+  """
+
+  def __init__(self, client_id, path_type, components, timestamp=None):
+    self.client_id = client_id
+    self.path_type = path_type
+    self.components = components
+    self.timestamp = timestamp
+
+    path = "/" + "/".join(components)
+
+    if timestamp is None:
+      message = "Content for {} file with path '{}' for client '{}' not found"
+      message = message.format(path_type, path, client_id)
+    else:
+      message = ("Content for {} file with path '{}' and timestamp '{}' for "
+                 "client '{}' not found")
+      message = message.format(path_type, path, timestamp, client_id)
+
+    super(FileContentNotFoundError, self).__init__(message)
 
 
 class VfsRefreshOperationNotFoundError(
@@ -511,9 +536,8 @@ def Handle(self, args, token=None):
     try:
       fd = file_store.OpenFile(client_path, max_timestamp=args.timestamp)
     except file_store.FileHasNoContentError:
-      raise FileContentNotFoundError(
-          "File %s with timestamp %s wasn't found on client %s" %
-          (args.file_path, args.timestamp, args.client_id))
+      raise FileContentNotFoundError(args.client_id, path_type, components,
+                                     args.timestamp)
 
     fd.seek(args.offset)
     # No need to protect against args.length == 0 case and large files:
@@ -555,7 +579,14 @@ def Handle(self, args, token=None):
     path_type, components = rdf_objects.ParseCategorizedPath(args.file_path)
     client_path = db.ClientPath(str(args.client_id), path_type, components)
 
-    file_obj = file_store.OpenFile(client_path, max_timestamp=args.timestamp)
+    # TODO: Raise FileNotFoundError if the file does not exist in
+    #  VFS.
+    try:
+      file_obj = file_store.OpenFile(client_path, max_timestamp=args.timestamp)
+    except file_store.FileHasNoContentError:
+      raise FileContentNotFoundError(args.client_id, path_type, components,
+                                     args.timestamp)
+
     size = max(0, file_obj.size - args.offset)
     if args.length and args.length < size:
       size = args.length
@@ -694,6 +725,8 @@ def _FindPathspec(self, args):
 
     for k in sorted(res, key=len, reverse=True):
       path_info = res[k]
+      if path_info is None:
+        raise FileNotFoundError(args.client_id, path_type, components)
       if path_info.stat_entry and path_info.stat_entry.pathspec:
         ps = path_info.stat_entry.pathspec
 

grr/server/grr_response_server/gui/api_plugins/vfs_test.py

@@ -326,6 +326,18 @@ def setUp(self):
     self.file_path = "fs/os/c/Downloads/a.txt"
     self.CreateFileVersions(self.client_id, self.file_path)
 
+  def testRaisesOnNonExistentPath(self):
+    args = vfs_plugin.ApiGetFileBlobArgs(
+        client_id=self.client_id, file_path="fs/os/foo/bar")
+    with self.assertRaises(vfs_plugin.FileContentNotFoundError) as context:
+      self.handler.Handle(args, token=self.token)
+
+    exception = context.exception
+    self.assertEqual(exception.client_id, self.client_id)
+    self.assertEqual(exception.path_type, rdf_objects.PathInfo.PathType.OS)
+    self.assertItemsEqual(exception.components, ["foo", "bar"])
+    self.assertIsNone(exception.timestamp)
+
   def testRaisesOnEmptyPath(self):
     args = vfs_plugin.ApiGetFileBlobArgs(client_id=self.client_id, file_path="")
     with self.assertRaises(ValueError):
@@ -456,6 +468,17 @@ def setUp(self):
     # Choose some directory with pathspec in the ClientFixture.
     self.file_path = "fs/os/Users/Shared"
 
+  def testRaisesOnNonExistentPath(self):
+    args = vfs_plugin.ApiCreateVfsRefreshOperationArgs(
+        client_id=self.client_id, file_path="fs/os/foo/bar")
+    with self.assertRaises(vfs_plugin.FileNotFoundError) as context:
+      self.handler.Handle(args, token=self.token)
+
+    exception = context.exception
+    self.assertEqual(exception.client_id, self.client_id)
+    self.assertEqual(exception.path_type, rdf_objects.PathInfo.PathType.OS)
+    self.assertItemsEqual(exception.components, ["foo", "bar"])
+
   def testRaisesOnEmptyPath(self):
     args = vfs_plugin.ApiCreateVfsRefreshOperationArgs(
         client_id=self.client_id, file_path="")

grr_colab/errors.py

@@ -10,6 +10,7 @@
 from typing import Text, List, Optional
 
 from grr_colab import flags
+from grr_response_proto import jobs_pb2
 
 FLAGS = flags.FLAGS
 
@@ -88,3 +89,11 @@ def __init__(self, client_id, path):
     self.path = path
     msg = 'Path `{}` for client {} is not a directory'.format(client_id, path)
     super(NotDirectoryError, self).__init__(msg)
+
+
+class UnsupportedPathTypeError(Exception):
+
+  def __init__(self, path_type):
+    self.path_type = path_type
+    msg = 'Unsupported path type {}'.format(path_type)
+    super(UnsupportedPathTypeError, self).__init__(msg)

grr_colab/fs.py

@@ -40,7 +40,7 @@ def id(self):
 
   @property
   def cached(self):
-    return vfs.VFS(self._client)
+    return vfs.VFS(self._client, self._path_type)
 
   def ls(self, path, max_depth = 1):
     """Lists contents of a given directory.

grr_colab/vfs.py

@@ -163,8 +163,10 @@ class VFS(object):
   Offers easy to use interface to perform operations on GRR VFS from Colab.
   """
 
-  def __init__(self, client_):
+  def __init__(self, client_,
+               path_type):
     self._client = client_
+    self._path_type = path_type
 
   def ls(self, path, max_depth = 1):
     """Lists contents of a given VFS directory.
@@ -266,11 +268,13 @@ def wget(self, path):
 
     link = '{}/api/clients/{}/vfs-blob/{}'
     return link.format(FLAGS.grr_admin_ui_url, self._client.client_id,
-                       get_vfs_path(path))
+                       get_vfs_path(path, self._path_type))
 
   def _get_file(self, path):
-    return self._client.File(get_vfs_path(path))
+    return self._client.File(get_vfs_path(path, self._path_type))
 
 
-def get_vfs_path(path):
-  return 'fs/os{}'.format(path)
+def get_vfs_path(path, path_type):
+  if path_type == jobs_pb2.PathSpec.OS:
+    return 'fs/os{}'.format(path)
+  raise errors.UnsupportedPathTypeError(path_type)