feat(nodestore): file system-based nodestore backend, with added supp…

…ort for S3 and GCS storage.

src/sentry/nodestore/filesystem/backend.py

@@ -1,53 +1,58 @@
-import os
-from datetime import datetime, timedelta, timezone
+import datetime
+from io import BytesIO
 
 from django.conf import settings
 
+from sentry import options as options_store
+from sentry.models.file import get_storage
 from sentry.nodestore.base import NodeStorage
 
 
 class FileSystemNodeStorage(NodeStorage):
     """
-    A simple backend that saves each node as a file. Only appropriate for
-    debugging and development!
+    A file system-based backend that saves each node as a file, Local fileSystem only appropriate for
+    debugging and development! For production environments, please use S3 or GCS. config.yml Configuration reference:
+    >>> filestore.backend: "s3"
+    ... filestore.options:
+    ...   access_key: "xxx"
+    ...   secret_key: "xxx"
+    ...   endpoint_url: "https://s3.us-east-1.amazonaws.com"
+    ...   bucket_name: "sentry"
+    ...   location: "/sentry"
     """
 
-    def __init__(self, path: str | None = None):
-        self.path: str = ""
-
-        if not settings.DEBUG:
-            raise ValueError("FileSystemNodeStorage should only be used in development!")
-        if path:
-            self.path = os.path.abspath(os.path.expanduser(path))
-        else:
-            self.path = os.path.abspath(os.path.join(os.path.dirname(__file__), "./nodes"))
-
-    def _get_bytes(self, id: str) -> bytes:
-        with open(self.node_path(id), "rb") as file:
-            return file.read()
-
-    def _set_bytes(self, id: str, data: bytes, ttl: timedelta | None = None) -> None:
-        with open(self.node_path(id), "wb") as file:
-            file.write(data)
-
-    def delete(self, id: str) -> None:
-        os.remove(self.node_path(id))
-
-    def cleanup(self, cutoff: datetime) -> None:
-        for filename in os.listdir(self.path):
-            path = os.path.join(self.path, filename)
-            creation_datetime = datetime.fromtimestamp(os.path.getctime(path)).replace(
-                tzinfo=timezone.utc
-            )
-
-            if creation_datetime > cutoff:
-                os.remove(path)
-
-    def bootstrap(self) -> None:
-        try:
-            os.mkdir(self.path)
-        except FileExistsError:
-            pass
-
-    def node_path(self, id: str) -> str:
-        return os.path.join(self.path, f"{id}.json")
+    def __init__(self, prefix_path=None):
+        self.prefix_path: str = "nodestore/"
+        if not settings.DEBUG and options_store.get("filestore.backend") == "filesystem":
+            raise ValueError("Local fileSystem should only be used in development!")
+        if prefix_path:
+            self.prefix_path = prefix_path
+
+    def _get_bytes(self, id: str):
+        storage = get_storage()
+        path = self.node_path(id)
+        return storage.open(path).read()
+
+    def _set_bytes(self, id: str, data: bytes, ttl=0):
+        storage = get_storage()
+        path = self.node_path(id)
+        storage.save(path, BytesIO(data))
+
+    def delete(self, id):
+        storage = get_storage()
+        path = self.node_path(id)
+        storage.delete(path)
+
+    def cleanup(self, cutoff: datetime.datetime):
+        """
+        This driver does not have managed TTLs.  To enable TTLs you will need to enable it on your
+        bucket.
+        """
+        raise NotImplementedError
+
+    def bootstrap(self):
+        # Nothing for filesystem backend to do during bootstrap
+        pass
+
+    def node_path(self, id: str):
+        return f"{self.prefix_path}{id}.json"