Improve typing for typeQuota

generate_class_from_json.py

@@ -18,10 +18,7 @@
 def to_py_type(v: Dict[str, Any]) -> str:
     if 'type' in v:
         t = TYPE_MAPPING.get(v['type'], v['type'])
-        if v.get('required'):
-            return t
-        else:
-            return 'Optional[{}]'.format(t)
+        return t
     if 'enum' in v:
         return 'str'
     raise AssertionError(v)

pyhdfs.py

@@ -188,6 +188,18 @@ def __ne__(self, other: object) -> bool:
         return not self.__eq__(other)
 
 
+class TypeQuota(_BoilerplateClass):
+    """
+    :param consumed: The storage type space consumed.
+    :type consumed: int
+    :param quota: The storage type quota.
+    :type quota: int
+    """
+
+    consumed: int
+    quota: int
+
+
 class ContentSummary(_BoilerplateClass):
     """
     :param directoryCount: The number of directories.
@@ -202,8 +214,8 @@ class ContentSummary(_BoilerplateClass):
     :type spaceConsumed: int
     :param spaceQuota: The disk space quota.
     :type spaceQuota: int
-    :param typeQuota:
-    :type typeQuota: Optional[object]
+    :param typeQuota: Quota usage for ARCHIVE, DISK, SSD
+    :type typeQuota: Dict[str, TypeQuota]
     """
 
     directoryCount: int
@@ -212,7 +224,7 @@ class ContentSummary(_BoilerplateClass):
     quota: int
     spaceConsumed: int
     spaceQuota: int
-    typeQuota: Optional[object]
+    typeQuota: Dict[str, TypeQuota]
 
 
 class FileChecksum(_BoilerplateClass):
@@ -618,8 +630,10 @@ def list_status(self, path: str, **kwargs: _PossibleArgumentTypes) -> List[FileS
 
     def get_content_summary(self, path: str, **kwargs: _PossibleArgumentTypes) -> ContentSummary:
         """Return the :py:class:`ContentSummary` of a given Path."""
-        return ContentSummary(
-            **_json(self._get(path, 'GETCONTENTSUMMARY', **kwargs))['ContentSummary'])
+        data = _json(self._get(path, 'GETCONTENTSUMMARY', **kwargs))['ContentSummary']
+        if 'typeQuota' in data:
+            data['typeQuota'] = {k: TypeQuota(**v) for k, v in data['typeQuota'].items()}
+        return ContentSummary(**data)
 
     def get_file_checksum(self, path: str, **kwargs: _PossibleArgumentTypes) -> FileChecksum:
         """Get the checksum of a file.

test_pyhdfs.py

@@ -30,6 +30,7 @@
 from pyhdfs import HdfsPathIsNotEmptyDirectoryException
 from pyhdfs import HdfsSnapshotException
 from pyhdfs import HdfsUnsupportedOperationException
+from pyhdfs import TypeQuota
 
 TEST_DIR = '/tmp/pyhdfs_test'
 TEST_FILE = posixpath.join(TEST_DIR, 'some file')
@@ -141,6 +142,19 @@ def test_basic_operations(self) -> None:
         assert client.delete(TEST_DIR, recursive=True)
         assert not client.delete(TEST_DIR, recursive=True)
 
+    @unittest.skipIf(os.environ.get('VERSION') == '2.9.2', "Not supported on Hadoop 2")
+    def test_get_content_summary_quota(self) -> None:
+        # WebHDFS doesn't support the dfsadmin command to set quotas, so this test uses the CLI.
+        client = make_client()
+        self._make_empty_dir(client)
+        subprocess.check_call([
+            'hdfs', 'dfsadmin', '-setSpaceQuota', '10', '-storageType', 'ARCHIVE', TEST_DIR
+        ])
+        content_summary = client.get_content_summary(TEST_DIR)
+        assert content_summary.typeQuota == {
+            'ARCHIVE': TypeQuota(consumed=0, quota=10),
+        }
+
     def test_list_file(self) -> None:
         client = make_client()
         self._make_dir_and_file(client)
@@ -382,8 +396,8 @@ def test_snapshots(self) -> None:
         with self.assertRaises(HdfsSnapshotException):
             client.create_snapshot(TEST_DIR)
 
-        # WebHDFS doesn't support the dfsadmin command to enable snapshots, so this test makes
-        # some assumptions to turn them on.
+        # WebHDFS doesn't support the dfsadmin command to enable snapshots, so this test uses the
+        # CLI.
         subprocess.check_call(['hdfs', 'dfsadmin', '-allowSnapshot', TEST_DIR])
 
         path = client.create_snapshot(TEST_DIR, snapshotname='x')