Add ContentType option (fsspec#243)

* feat: fsspec#119 Add option for content-type * black Co-authored-by: arpit1997 <[email protected]>
janjagusch · Mar 10, 2020 · 70f5581 · 70f5581
1 parent 2182133
commit 70f5581
Show file tree

Hide file tree

Showing 63 changed files with 7,219 additions and 5,878 deletions.
diff --git a/gcsfs/core.py b/gcsfs/core.py
@@ -980,6 +980,7 @@ def __init__(
         acl=None,
         consistency="md5",
         metadata=None,
+        content_type=None,
         **kwargs
     ):
         """
@@ -1003,6 +1004,9 @@ def __init__(
             'size' ensures that the number of bytes reported by GCS matches
             the number we wrote; 'md5' does a full checksum. Any value other
             than 'size' or 'md5' is assumed to mean no checking.
+        content_type: str
+            default is `application/octet-stream`. See the list of available
+            content types at https://www.iana.org/assignments/media-types/media-types.txt
         metadata: dict
             Custom metadata, in key/value pairs, added at file creation
         """
@@ -1025,6 +1029,7 @@ def __init__(
         self.metadata = metadata
         self.acl = acl
         self.consistency = consistency
+        self.content_type = content_type or "application/octet-stream"
         if self.consistency == "md5":
             self.md5 = md5()
         if mode == "wb":
@@ -1033,15 +1038,6 @@ def __init__(
                 self.blocksize = GCS_MIN_BLOCK_SIZE
             self.location = None
 
-    @property
-    def trim(self):
-        warnings.warn(
-            "GCSFSFile.trim has not effect and will be removed in a future version. "
-            "Access cache settings from GCSFSFile.cache.",
-            FutureWarning,
-        )
-        return True
-
     def info(self):
         """ File information about this path """
         return self.details
@@ -1080,9 +1076,7 @@ def _upload_chunk(self, final=False):
                 elif not final:
                     raise ValueError("Non-final chunk write below min size.")
             head["Content-Range"] = "bytes %i-%i/*" % (self.offset, self.offset + l - 1)
-        head.update(
-            {"Content-Type": "application/octet-stream", "Content-Length": str(l)}
-        )
+        head.update({"Content-Type": self.content_type, "Content-Length": str(l)})
         r = self.gcsfs._call(
             "POST", self.location, uploadType="resumable", headers=head, data=data
         )
@@ -1130,6 +1124,7 @@ def _initiate_upload(self):
             "/v1/b/%s/o" % quote_plus(self.bucket),
             uploadType="resumable",
             json={"name": self.key, "metadata": self.metadata},
+            headers={"X-Upload-Content-Type": self.content_type},
         )
         self.location = r.headers["Location"]
 
@@ -1165,9 +1160,11 @@ def _simple_upload(self):
                 "--==0=="
                 "\nContent-Type: application/json; charset=UTF-8"
                 "\n\n" + metadata + "\n--==0=="
-                "\nContent-Type: application/octet-stream"
+                "\nContent-Type: {0}"
                 "\n\n"
-            ).encode()
+            )
+            .format(self.content_type)
+            .encode()
             + data
             + b"\n--==0==--"
         )

diff --git a/gcsfs/tests/recordings/test_array.yaml b/gcsfs/tests/recordings/test_array.yaml
@@ -17,9 +17,9 @@ interactions:
   response:
     body:
       string: !!binary |
-        H4sIAOvrZ14C/4WOQQ7CIBAAv0L2bOBgPLRHP0IIrpSIXbK7LRjj3xV9gNfJJDNPCDGiiFe64Qqz
-        gd47HAxgr5lRfB7weJqmD5NIFYezqFaZnWut2USUCoaaxUa6u7Dp4i6cdzR/LdxFiUNCe91K8ZFW
-        ZSqj/r3x+vjlzhgYGV5vPzgjxq0AAAA=
+        H4sIALz2Z14C/4WOQQ4CIQxFr0K6NrAwLmaWXoQQrAwRp6TtDBjj3RU9gNuX9/P+E0KMKOKVbrjC
+        bKD3DgcD2GtmFJ8HPJ6m6cMkUsXhLKpVZudaazYRpYKhZrGR7i5surgL7qLEIaG9bqX4SKsyFfN3
+        x3nHUf++8fr45c4YGBleb1uH3f2tAAAA
     headers:
       Cache-Control:
       - private
@@ -156,7 +156,7 @@ interactions:
   response:
     body:
       string: !!binary |
-        H4sIAAAAAAAAANPVTUosSc6IDzb0DAzLKXLLKgiNd3R0K8+IykmOjE/W1eXlAgC2Du/xIwAAAA==
+        H4sIAAAAAAAAANPVTUosSc6IDw73Dq0Kz3UtC8mPd3R0TqzwMTNKLHPV1eXlAgBBcvwoIwAAAA==
     headers:
       Cache-Control:
       - private, max-age=0
@@ -165,7 +165,7 @@ interactions:
       Content-Security-Policy:
       - frame-ancestors 'self'
       Content-Type:
-      - multipart/mixed; boundary=batch_S1IQVlrFjpU_AAFwhZlcY_c
+      - multipart/mixed; boundary=batch_SWKUzWmEvTo_AACaxL62avE
       Server:
       - GSE
       Transfer-Encoding:
@@ -339,6 +339,8 @@ interactions:
       - '40'
       Content-Type:
       - application/json
+      X-Upload-Content-Type:
+      - application/octet-stream
     method: POST
     uri: https://www.googleapis.com/upload/storage/v1/b/gcsfs-testing/o?uploadType=resumable
   response:
@@ -352,7 +354,7 @@ interactions:
       Content-Type:
       - text/plain; charset=utf-8
       Location:
-      - https://www.googleapis.com/upload/storage/v1/b/gcsfs-testing/o?uploadType=resumable&upload_id=AEnB2Ur_9F9Rteooai_VdsJl8F_6UYJJeycbSUsE8gHBw-alc-PS1FJHRG_Z4kNZ677qTjBtyHBcORx0QKkYDAXArUP2WUt2bA
+      - https://www.googleapis.com/upload/storage/v1/b/gcsfs-testing/o?uploadType=resumable&upload_id=AEnB2Up5FUb8qy8nz9xCgZzbl1TDfSdal-CTs9KzZd4kOMEAsuV88ihTwJmCXvQpn-SxOk4ilkOChjy28bLeuwydClnZQ2jGIA
       Pragma:
       - no-cache
       Server:
@@ -379,27 +381,27 @@ interactions:
       Content-Type:
       - application/octet-stream
     method: POST
-    uri: https://www.googleapis.com/upload/storage/v1/b/gcsfs-testing/o?uploadType=resumable&uploadType=resumable&upload_id=AEnB2Ur_9F9Rteooai_VdsJl8F_6UYJJeycbSUsE8gHBw-alc-PS1FJHRG_Z4kNZ677qTjBtyHBcORx0QKkYDAXArUP2WUt2bA
+    uri: https://www.googleapis.com/upload/storage/v1/b/gcsfs-testing/o?uploadType=resumable&uploadType=resumable&upload_id=AEnB2Up5FUb8qy8nz9xCgZzbl1TDfSdal-CTs9KzZd4kOMEAsuV88ihTwJmCXvQpn-SxOk4ilkOChjy28bLeuwydClnZQ2jGIA
   response:
     body:
-      string: "{\n  \"kind\": \"storage#object\",\n  \"id\": \"gcsfs-testing/tmp/test/a/1583868909652857\",\n
+      string: "{\n  \"kind\": \"storage#object\",\n  \"id\": \"gcsfs-testing/tmp/test/a/1583871678573927\",\n
         \ \"selfLink\": \"https://www.googleapis.com/storage/v1/b/gcsfs-testing/o/tmp%2Ftest%2Fa\",\n
-        \ \"mediaLink\": \"https://www.googleapis.com/download/storage/v1/b/gcsfs-testing/o/tmp%2Ftest%2Fa?generation=1583868909652857&alt=media\",\n
+        \ \"mediaLink\": \"https://www.googleapis.com/download/storage/v1/b/gcsfs-testing/o/tmp%2Ftest%2Fa?generation=1583871678573927&alt=media\",\n
         \ \"name\": \"tmp/test/a\",\n  \"bucket\": \"gcsfs-testing\",\n  \"generation\":
-        \"1583868909652857\",\n  \"metageneration\": \"1\",\n  \"storageClass\": \"MULTI_REGIONAL\",\n
-        \ \"size\": \"1000\",\n  \"md5Hash\": \"dkRnLQSSkPA5DZyZPH00PQ==\",\n  \"crc32c\":
-        \"jS1TJA==\",\n  \"etag\": \"CPmW7L3TkOgCEAE=\",\n  \"timeCreated\": \"2020-03-10T19:35:09.652Z\",\n
-        \ \"updated\": \"2020-03-10T19:35:09.652Z\",\n  \"timeStorageClassUpdated\":
-        \"2020-03-10T19:35:09.652Z\"\n}\n"
+        \"1583871678573927\",\n  \"metageneration\": \"1\",\n  \"contentType\": \"application/octet-stream\",\n
+        \ \"storageClass\": \"MULTI_REGIONAL\",\n  \"size\": \"1000\",\n  \"md5Hash\":
+        \"dkRnLQSSkPA5DZyZPH00PQ==\",\n  \"crc32c\": \"jS1TJA==\",\n  \"etag\": \"COfalebdkOgCEAE=\",\n
+        \ \"timeCreated\": \"2020-03-10T20:21:18.573Z\",\n  \"updated\": \"2020-03-10T20:21:18.573Z\",\n
+        \ \"timeStorageClassUpdated\": \"2020-03-10T20:21:18.573Z\"\n}\n"
     headers:
       Cache-Control:
       - no-cache, no-store, max-age=0, must-revalidate
       Content-Length:
-      - '710'
+      - '755'
       Content-Type:
       - application/json; charset=UTF-8
       ETag:
-      - CPmW7L3TkOgCEAE=
+      - COfalebdkOgCEAE=
       Pragma:
       - no-cache
       Server:
@@ -423,24 +425,24 @@ interactions:
     uri: https://www.googleapis.com/storage/v1/b/gcsfs-testing/o/tmp%2Ftest%2Fa
   response:
     body:
-      string: "{\n  \"kind\": \"storage#object\",\n  \"id\": \"gcsfs-testing/tmp/test/a/1583868909652857\",\n
+      string: "{\n  \"kind\": \"storage#object\",\n  \"id\": \"gcsfs-testing/tmp/test/a/1583871678573927\",\n
         \ \"selfLink\": \"https://www.googleapis.com/storage/v1/b/gcsfs-testing/o/tmp%2Ftest%2Fa\",\n
-        \ \"mediaLink\": \"https://www.googleapis.com/download/storage/v1/b/gcsfs-testing/o/tmp%2Ftest%2Fa?generation=1583868909652857&alt=media\",\n
+        \ \"mediaLink\": \"https://www.googleapis.com/download/storage/v1/b/gcsfs-testing/o/tmp%2Ftest%2Fa?generation=1583871678573927&alt=media\",\n
         \ \"name\": \"tmp/test/a\",\n  \"bucket\": \"gcsfs-testing\",\n  \"generation\":
-        \"1583868909652857\",\n  \"metageneration\": \"1\",\n  \"storageClass\": \"MULTI_REGIONAL\",\n
-        \ \"size\": \"1000\",\n  \"md5Hash\": \"dkRnLQSSkPA5DZyZPH00PQ==\",\n  \"crc32c\":
-        \"jS1TJA==\",\n  \"etag\": \"CPmW7L3TkOgCEAE=\",\n  \"timeCreated\": \"2020-03-10T19:35:09.652Z\",\n
-        \ \"updated\": \"2020-03-10T19:35:09.652Z\",\n  \"timeStorageClassUpdated\":
-        \"2020-03-10T19:35:09.652Z\"\n}\n"
+        \"1583871678573927\",\n  \"metageneration\": \"1\",\n  \"contentType\": \"application/octet-stream\",\n
+        \ \"storageClass\": \"MULTI_REGIONAL\",\n  \"size\": \"1000\",\n  \"md5Hash\":
+        \"dkRnLQSSkPA5DZyZPH00PQ==\",\n  \"crc32c\": \"jS1TJA==\",\n  \"etag\": \"COfalebdkOgCEAE=\",\n
+        \ \"timeCreated\": \"2020-03-10T20:21:18.573Z\",\n  \"updated\": \"2020-03-10T20:21:18.573Z\",\n
+        \ \"timeStorageClassUpdated\": \"2020-03-10T20:21:18.573Z\"\n}\n"
     headers:
       Cache-Control:
       - private, max-age=0, must-revalidate, no-transform
       Content-Length:
-      - '710'
+      - '755'
       Content-Type:
       - application/json; charset=UTF-8
       ETag:
-      - CPmW7L3TkOgCEAE=
+      - COfalebdkOgCEAE=
       Server:
       - UploadServer
       Vary:
@@ -461,7 +463,7 @@ interactions:
       Range:
       - bytes=0-999
     method: GET
-    uri: https://www.googleapis.com/download/storage/v1/b/gcsfs-testing/o/tmp%2Ftest%2Fa?alt=media&generation=1583868909652857
+    uri: https://www.googleapis.com/download/storage/v1/b/gcsfs-testing/o/tmp%2Ftest%2Fa?alt=media&generation=1583871678573927
   response:
     body:
       string: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
@@ -477,7 +479,7 @@ interactions:
       Content-Type:
       - application/octet-stream
       ETag:
-      - CPmW7L3TkOgCEAE=
+      - COfalebdkOgCEAE=
       Pragma:
       - no-cache
       Server:
@@ -486,7 +488,7 @@ interactions:
       - Origin
       - X-Origin
       X-Goog-Generation:
-      - '1583868909652857'
+      - '1583871678573927'
       X-Goog-Metageneration:
       - '1'
       X-Goog-Storage-Class:
@@ -568,20 +570,21 @@ interactions:
   response:
     body:
       string: "{\n  \"kind\": \"storage#objects\",\n  \"items\": [\n    {\n      \"kind\":
-        \"storage#object\",\n      \"id\": \"gcsfs-testing/tmp/test/a/1583868909652857\",\n
+        \"storage#object\",\n      \"id\": \"gcsfs-testing/tmp/test/a/1583871678573927\",\n
         \     \"selfLink\": \"https://www.googleapis.com/storage/v1/b/gcsfs-testing/o/tmp%2Ftest%2Fa\",\n
-        \     \"mediaLink\": \"https://www.googleapis.com/download/storage/v1/b/gcsfs-testing/o/tmp%2Ftest%2Fa?generation=1583868909652857&alt=media\",\n
+        \     \"mediaLink\": \"https://www.googleapis.com/download/storage/v1/b/gcsfs-testing/o/tmp%2Ftest%2Fa?generation=1583871678573927&alt=media\",\n
         \     \"name\": \"tmp/test/a\",\n      \"bucket\": \"gcsfs-testing\",\n      \"generation\":
-        \"1583868909652857\",\n      \"metageneration\": \"1\",\n      \"storageClass\":
-        \"MULTI_REGIONAL\",\n      \"size\": \"1000\",\n      \"md5Hash\": \"dkRnLQSSkPA5DZyZPH00PQ==\",\n
-        \     \"crc32c\": \"jS1TJA==\",\n      \"etag\": \"CPmW7L3TkOgCEAE=\",\n      \"timeCreated\":
-        \"2020-03-10T19:35:09.652Z\",\n      \"updated\": \"2020-03-10T19:35:09.652Z\",\n
-        \     \"timeStorageClassUpdated\": \"2020-03-10T19:35:09.652Z\"\n    }\n  ]\n}\n"
+        \"1583871678573927\",\n      \"metageneration\": \"1\",\n      \"contentType\":
+        \"application/octet-stream\",\n      \"storageClass\": \"MULTI_REGIONAL\",\n
+        \     \"size\": \"1000\",\n      \"md5Hash\": \"dkRnLQSSkPA5DZyZPH00PQ==\",\n
+        \     \"crc32c\": \"jS1TJA==\",\n      \"etag\": \"COfalebdkOgCEAE=\",\n      \"timeCreated\":
+        \"2020-03-10T20:21:18.573Z\",\n      \"updated\": \"2020-03-10T20:21:18.573Z\",\n
+        \     \"timeStorageClassUpdated\": \"2020-03-10T20:21:18.573Z\"\n    }\n  ]\n}\n"
     headers:
       Cache-Control:
       - private, max-age=0, must-revalidate, no-transform
       Content-Length:
-      - '832'
+      - '881'
       Content-Type:
       - application/json; charset=UTF-8
       Server: