Merge pull request python-diamond#693 from somechris/diskusage-nvme

diskusage: Support Amazon AWS NVMe disks

src/collectors/diskusage/diskusage.py

@@ -63,7 +63,7 @@ def get_default_config(self):
  '|sd[a-z]+[0-9]*$' +
  '|x?vd[a-z]+[0-9]*$' +
  '|disk[0-9]+$' +
- '|nvme[0-9]+n[0-9]+$' +
+ '|nvme[0-9]+n[0-9]+(p[0-9]+)?$' +
  '|dm\-[0-9]+$'),
  'sector_size': 512,
  'send_zero': False,

src/collectors/diskusage/test/fixtures/diskstats

@@ -31,3 +31,6 @@
  8 48 sdd 11350168 128344871 1119269534 168467240 9130464 31774918 328709208 167823720 0 18473250 337157280
  8 49 sdd1 11350141 128344871 1119269318 168467050 9130464 31774918 328709208 167823720 0 18473200 337157090
  9 0 md0 150707 0 2818762 0 111152210 0 889217680 0 0 0 0
+ 259 0 nvme0n1 32040 0 906912 78220 280728 0 5219224 221588 0 44112 299804
+ 259 1 nvme0n1p1 0 0 0 0 0 0 0 0 0 0 0
+ 259 2 nvme1n1 349 0 3354 776 583565 0 67618880 1735280 0 127152 1736008

src/collectors/diskusage/test/fixtures/proc_diskstats_1_nvme

@@ -0,0 +1,29 @@
+ 1 0 ram0 0 0 0 0 0 0 0 0 0 0 0
+ 1 1 ram1 0 0 0 0 0 0 0 0 0 0 0
+ 1 2 ram2 0 0 0 0 0 0 0 0 0 0 0
+ 1 3 ram3 0 0 0 0 0 0 0 0 0 0 0
+ 1 4 ram4 0 0 0 0 0 0 0 0 0 0 0
+ 1 5 ram5 0 0 0 0 0 0 0 0 0 0 0
+ 1 6 ram6 0 0 0 0 0 0 0 0 0 0 0
+ 1 7 ram7 0 0 0 0 0 0 0 0 0 0 0
+ 1 8 ram8 0 0 0 0 0 0 0 0 0 0 0
+ 1 9 ram9 0 0 0 0 0 0 0 0 0 0 0
+ 1 10 ram10 0 0 0 0 0 0 0 0 0 0 0
+ 1 11 ram11 0 0 0 0 0 0 0 0 0 0 0
+ 1 12 ram12 0 0 0 0 0 0 0 0 0 0 0
+ 1 13 ram13 0 0 0 0 0 0 0 0 0 0 0
+ 1 14 ram14 0 0 0 0 0 0 0 0 0 0 0
+ 1 15 ram15 0 0 0 0 0 0 0 0 0 0 0
+ 8 0 nvme0n1 1650390 2176303 51741706 8213219 11766889 27366923 313073680 207996798 0 8426453 216207918
+ 8 1 nvme0n1p1 1650369 2176272 51741290 8213067 11766889 27366923 313073680 207996798 0 8426237 216207629
+ 8 16 nvme1n2 1642335748 6635 47517782005 2485957567 13136509974 3665796878 237171463357 1799839451 1 1299123651 4283211976
+ 8 17 nvme1n2p3 1642335722 6588 47517781421 2485957476 13136509974 3665796878 237171463357 1799839451 1 1299064267 4283078696
+ 9 0 md0 0 0 0 0 0 0 0 0 0 0 0
+ 7 0 loop0 0 0 0 0 0 0 0 0 0 0 0
+ 7 1 loop1 0 0 0 0 0 0 0 0 0 0 0
+ 7 2 loop2 0 0 0 0 0 0 0 0 0 0 0
+ 7 3 loop3 0 0 0 0 0 0 0 0 0 0 0
+ 7 4 loop4 0 0 0 0 0 0 0 0 0 0 0
+ 7 5 loop5 0 0 0 0 0 0 0 0 0 0 0
+ 7 6 loop6 0 0 0 0 0 0 0 0 0 0 0
+ 7 7 loop7 0 0 0 0 0 0 0 0 0 0 0

src/collectors/diskusage/test/fixtures/proc_diskstats_2_nvme

@@ -0,0 +1,29 @@
+ 1 0 ram0 0 0 0 0 0 0 0 0 0 0 0
+ 1 1 ram1 0 0 0 0 0 0 0 0 0 0 0
+ 1 2 ram2 0 0 0 0 0 0 0 0 0 0 0
+ 1 3 ram3 0 0 0 0 0 0 0 0 0 0 0
+ 1 4 ram4 0 0 0 0 0 0 0 0 0 0 0
+ 1 5 ram5 0 0 0 0 0 0 0 0 0 0 0
+ 1 6 ram6 0 0 0 0 0 0 0 0 0 0 0
+ 1 7 ram7 0 0 0 0 0 0 0 0 0 0 0
+ 1 8 ram8 0 0 0 0 0 0 0 0 0 0 0
+ 1 9 ram9 0 0 0 0 0 0 0 0 0 0 0
+ 1 10 ram10 0 0 0 0 0 0 0 0 0 0 0
+ 1 11 ram11 0 0 0 0 0 0 0 0 0 0 0
+ 1 12 ram12 0 0 0 0 0 0 0 0 0 0 0
+ 1 13 ram13 0 0 0 0 0 0 0 0 0 0 0
+ 1 14 ram14 0 0 0 0 0 0 0 0 0 0 0
+ 1 15 ram15 0 0 0 0 0 0 0 0 0 0 0
+ 8 0 nvme0n1 1650390 2176303 51741706 8213219 11766892 27366928 313073744 207996798 0 8426453 216207918
+ 8 1 nvme0n1p1 1650369 2176272 51741290 8213067 11766892 27366928 313073744 207996798 0 8426237 216207629
+ 8 16 nvme1n2 1642336912 6635 47517819253 2485964730 13136518024 3665798895 237171541643 1799839788 0 1299128608 4283219468
+ 8 17 nvme1n2p3 1642336886 6588 47517818669 2485964639 13136518024 3665798895 237171541643 1799839788 0 1299069224 4283086187
+ 9 0 md0 0 0 0 0 0 0 0 0 0 0 0
+ 7 0 loop0 0 0 0 0 0 0 0 0 0 0 0
+ 7 1 loop1 0 0 0 0 0 0 0 0 0 0 0
+ 7 2 loop2 0 0 0 0 0 0 0 0 0 0 0
+ 7 3 loop3 0 0 0 0 0 0 0 0 0 0 0
+ 7 4 loop4 0 0 0 0 0 0 0 0 0 0 0
+ 7 5 loop5 0 0 0 0 0 0 0 0 0 0 0
+ 7 6 loop6 0 0 0 0 0 0 0 0 0 0 0
+ 7 7 loop7 0 0 0 0 0 0 0 0 0 0 0

src/collectors/diskusage/test/fixtures/test_verify_supporting_nvme.pkl

@@ -0,0 +1,152 @@
+(dp1
+S'nvme1n2.reads_kilobyte'
+p2
+F18624
+sS'nvme1n2.write_requests_merged_per_second'
+p3
+F201.69999999999999
+sS'nvme1n2.io_in_progress'
+p4
+F0
+sS'nvme0n1.reads'
+p5
+F0
+sS'nvme1n2.reads'
+p6
+F1164
+sS'nvme0n1.writes_kilobyte'
+p7
+F32
+sS'nvme0n1.util_percentage'
+p8
+F0
+sS'nvme0n1.write_requests_merged_per_second'
+p9
+F0.5
+sS'nvme1n2.read_requests_merged_per_second'
+p10
+F0
+sS'nvme1n2.concurrent_io'
+p11
+F0.5
+sS'nvme1n2.writes_per_second'
+p12
+F805
+sS'nvme0n1.read_kilobyte_per_second'
+p13
+F0
+sS'nvme1n2.await'
+p14
+F0.80000000000000004
+sS'nvme0n1.io_milliseconds'
+p15
+F0
+sS'nvme0n1.service_time'
+p16
+F0
+sS'nvme0n1.writes_per_second'
+p17
+F0.29999999999999999
+sS'nvme1n2.service_time'
+p18
+F0.5
+sS'nvme0n1.reads_kilobyte'
+p19
+F0
+sS'nvme1n2.writes_milliseconds'
+p20
+F337
+sS'nvme1n2.reads_merged'
+p21
+F0
+sS'nvme0n1.concurrent_io'
+p22
+F0
+sS'nvme1n2.write_kilobyte_per_second'
+p23
+F3914.3000000000002
+sS'nvme1n2.reads_milliseconds'
+p24
+F7163
+sS'nvme1n2.average_queue_length'
+p25
+F0.74920000000000009
+sS'nvme1n2.writes_kilobyte'
+p26
+F39143
+sS'nvme1n2.read_kilobyte_per_second'
+p27
+F1862.4000000000001
+sS'nvme0n1.reads_per_second'
+p28
+F0
+sS'nvme1n2.util_percentage'
+p29
+F49.57
+sS'nvme1n2.io'
+p30
+F9214
+sS'nvme0n1.iops'
+p31
+F0.29999999999999999
+sS'nvme1n2.average_request_size_kilobyte'
+p32
+F6.2999999999999998
+sS'nvme1n2.io_milliseconds_weighted'
+p33
+F7492
+sS'nvme0n1.io'
+p34
+F3
+sS'nvme0n1.writes_merged'
+p35
+F5
+sS'nvme0n1.average_request_size_kilobyte'
+p36
+F10.6
+sS'nvme0n1.read_requests_merged_per_second'
+p37
+F0
+sS'nvme0n1.reads_milliseconds'
+p38
+F0
+sS'nvme0n1.average_queue_length'
+p39
+F0
+sS'nvme1n2.writes_merged'
+p40
+F2017
+sS'nvme1n2.iops'
+p41
+F921.39999999999998
+sS'nvme0n1.writes'
+p42
+F3
+sS'nvme0n1.writes_milliseconds'
+p43
+F0
+sS'nvme0n1.io_in_progress'
+p44
+F0
+sS'nvme0n1.io_milliseconds_weighted'
+p45
+F0
+sS'nvme0n1.write_kilobyte_per_second'
+p46
+F3.2000000000000002
+sS'nvme1n2.writes'
+p47
+F8050
+sS'nvme0n1.await'
+p48
+F0
+sS'nvme0n1.reads_merged'
+p49
+F0
+sS'nvme1n2.io_milliseconds'
+p50
+F4957
+sS'nvme1n2.reads_per_second'
+p51
+F116.40000000000001
+s.

src/collectors/diskusage/test/testdiskusage.py

@@ -47,7 +47,8 @@ def test_get_disk_statistics(self):
  self.assertEqual(
  sorted(result.keys()),
  [(8, 0), (8, 1), (8, 16), (8, 17), (8, 32),
- (8, 33), (8, 48), (8, 49), (9, 0)])
+ (8, 33), (8, 48), (8, 49), (9, 0), (259, 0),
+ (259, 1), (259, 2)])
 
  return result
 
@@ -124,6 +125,42 @@ def test_verify_supporting_vda_and_xvdb(self, publish_mock):
 
  self.assertPublishedMany(publish_mock, metrics)
 
+ @patch('os.access', Mock(return_value=True))
+ @patch.object(Collector, 'publish')
+ def test_verify_supporting_nvme(self, publish_mock):
+ patch_open = patch(
+ '__builtin__.open',
+ Mock(
+ return_value=self.getFixture(
+ 'proc_diskstats_1_nvme')))
+ patch_time = patch('time.time', Mock(return_value=10))
+
+ patch_open.start()
+ patch_time.start()
+ self.collector.collect()
+ patch_open.stop()
+ patch_time.stop()
+
+ self.assertPublishedMany(publish_mock, {})
+
+ patch_open = patch(
+ '__builtin__.open',
+ Mock(
+ return_value=self.getFixture(
+ 'proc_diskstats_2_nvme')))
+ patch_time = patch('time.time', Mock(return_value=20))
+
+ patch_open.start()
+ patch_time.start()
+ self.collector.collect()
+ patch_open.stop()
+ patch_time.stop()
+
+ metrics = self.getPickledResults(
+ 'test_verify_supporting_nvme.pkl')
+
+ self.assertPublishedMany(publish_mock, metrics)
+
  @patch('os.access', Mock(return_value=True))
  @patch.object(Collector, 'publish')
  def test_verify_supporting_md_dm(self, publish_mock):