forked from cisco-system-traffic-generator/trex-core
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathdpdk_setup_ports.py
executable file
·1665 lines (1445 loc) · 72.6 KB
/
dpdk_setup_ports.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#! /bin/bash
"source" "find_python.sh" "--local"
"exec" "$PYTHON" "$0" "$@"
# hhaim
import sys
try:
xrange # Python 2
except NameError:
xrange = range # Python 3
import os
python_ver = 'python%s' % sys.version_info[0]
yaml_path = os.path.join('external_libs', 'pyyaml-3.11', python_ver)
if yaml_path not in sys.path:
sys.path.append(yaml_path)
import yaml
import dpdk_nic_bind
import re
import argparse
import copy
import shlex
import traceback
from collections import defaultdict, OrderedDict
import subprocess
import platform
import stat
import time
import shutil
import signal
import glob
from dpdk_nic_bind import is_napatech
march = os.uname()[4]
# exit code is Important should be
# -1 : don't continue
# 0 : no errors - no need to load mlx share object
# 32 : no errors - mlx share object should be loaded
# 64 : no errors - napatech 3GD should be running
MLX_EXIT_CODE = 32
NTACC_EXIT_CODE = 64
class VFIOBindErr(Exception): pass
PATH_ARR = os.getenv('PATH', '').split(':')
for path in ['/usr/local/sbin', '/usr/sbin', '/sbin']:
if path not in PATH_ARR:
PATH_ARR.append(path)
os.environ['PATH'] = ':'.join(PATH_ARR)
def if_list_remove_sub_if(if_list):
return if_list
class ConfigCreator(object):
mandatory_interface_fields = ['Slot_str', 'Device_str', 'NUMA']
_2hex_re = '[\da-fA-F]{2}'
mac_re = re.compile('^({0}:){{5}}{0}$'.format(_2hex_re))
if march == 'ppc64le':
MAX_LCORE_NUM = 159
else:
MAX_LCORE_NUM = 63
# cpu_topology - dict: physical processor -> physical core -> logical processing unit (thread)
# interfaces - array of dicts per interface, should include "mandatory_interface_fields" values
def __init__(self, cpu_topology, interfaces, include_lcores = [], exclude_lcores = [], only_first_thread = False, zmq_rpc_port = None, zmq_pub_port = None, prefix = None, ignore_numa = False):
self.cpu_topology = copy.deepcopy(cpu_topology)
self.interfaces = copy.deepcopy(interfaces)
del cpu_topology
del interfaces
assert isinstance(self.cpu_topology, dict), 'Type of cpu_topology should be dict, got: %s' % type(self.cpu_topology)
assert len(self.cpu_topology.keys()) > 0, 'cpu_topology should contain at least one processor'
assert isinstance(self.interfaces, list), 'Type of interfaces should be list, got: %s' % type(list)
assert len(self.interfaces) % 2 == 0, 'Should be even number of interfaces, got: %s' % len(self.interfaces)
assert len(self.interfaces) >= 2, 'Should be at least two interfaces, got: %s' % len(self.interfaces)
assert isinstance(include_lcores, list), 'include_lcores should be list, got: %s' % type(include_lcores)
assert isinstance(exclude_lcores, list), 'exclude_lcores should be list, got: %s' % type(exclude_lcores)
assert len(self.interfaces) >= 2, 'Should be at least two interfaces, got: %s' % len(self.interfaces)
if only_first_thread:
for cores in self.cpu_topology.values():
for core in cores.keys():
cores[core] = cores[core][:1]
include_lcores = [int(x) for x in include_lcores]
exclude_lcores = [int(x) for x in exclude_lcores]
self.has_zero_lcore = False
self.lcores_per_numa = {}
total_lcores = 0
for numa, cores in self.cpu_topology.items():
self.lcores_per_numa[numa] = {'main': [], 'siblings': [], 'all': []}
for core, lcores in cores.items():
total_lcores += len(lcores)
for lcore in list(lcores):
if include_lcores and lcore not in include_lcores:
cores[core].remove(lcore)
if exclude_lcores and lcore in exclude_lcores:
cores[core].remove(lcore)
if lcore > self.MAX_LCORE_NUM:
cores[core].remove(lcore)
if 0 in lcores:
self.has_zero_lcore = True
lcores.remove(0)
self.lcores_per_numa[numa]['siblings'].extend(lcores)
else:
self.lcores_per_numa[numa]['main'].extend(lcores[:1])
self.lcores_per_numa[numa]['siblings'].extend(lcores[1:])
self.lcores_per_numa[numa]['all'].extend(lcores)
for interface in self.interfaces:
for mandatory_interface_field in ConfigCreator.mandatory_interface_fields:
if mandatory_interface_field not in interface:
raise DpdkSetup("Expected '%s' field in interface dictionary, got: %s" % (mandatory_interface_field, interface))
Device_str = self._verify_devices_same_type(self.interfaces)
if '100Gb' in Device_str:
self.speed = 100
elif '50Gb' in Device_str:
self.speed = 50
elif '40Gb' in Device_str:
self.speed = 40
elif '25Gb' in Device_str:
self.speed = 25
elif '20Gb' in Device_str:
self.speed = 20
else:
self.speed = 10
minimum_required_lcores = len(self.interfaces) // 2 + 2
if total_lcores < minimum_required_lcores:
raise DpdkSetup('Your system should have at least %s cores for %s interfaces, and it has: %s.' %
(minimum_required_lcores, len(self.interfaces), total_lcores))
interfaces_per_numa = defaultdict(int)
for i in range(0, len(self.interfaces), 2):
if self.interfaces[i]['Slot_str'] == 'dummy':
numa = self.interfaces[i+1]['NUMA']
other_if_numa = self.interfaces[i]['NUMA']
else:
numa = self.interfaces[i]['NUMA']
other_if_numa = self.interfaces[i+1]['NUMA']
if numa != other_if_numa and not ignore_numa and self.interfaces[i]['Slot_str'] != 'dummy' and self.interfaces[i+1]['Slot_str'] != 'dummy':
raise DpdkSetup('NUMA of each pair of interfaces should be the same. Got NUMA %s for client interface %s, NUMA %s for server interface %s' %
(numa, self.interfaces[i]['Slot_str'], self.interfaces[i+1]['NUMA'], self.interfaces[i+1]['Slot_str']))
interfaces_per_numa[numa] += 2
self.interfaces_per_numa = interfaces_per_numa
self.prefix = prefix
self.zmq_pub_port = zmq_pub_port
self.zmq_rpc_port = zmq_rpc_port
self.ignore_numa = ignore_numa
@staticmethod
def verify_mac(mac_string):
if not ConfigCreator.mac_re.match(mac_string):
raise DpdkSetup('MAC address should be in format of 12:34:56:78:9a:bc, got: %s' % mac_string)
return mac_string.lower()
@staticmethod
def _exit_if_bad_ip(ip):
if not ConfigCreator._verify_ip(ip):
raise DpdkSetup("Got bad IP %s" % ip)
@staticmethod
def _verify_ip(ip):
a = ip.split('.')
if len(a) != 4:
return False
for x in a:
if not x.isdigit():
return False
i = int(x)
if i < 0 or i > 255:
return False
return True
@staticmethod
def _verify_devices_same_type(interfaces_list):
Device_str = interfaces_list[0]['Device_str']
if Device_str == 'dummy':
return Device_str
for interface in interfaces_list:
if interface['Device_str'] == 'dummy':
continue
if Device_str != interface['Device_str']:
raise DpdkSetup('Interfaces should be of same type, got:\n\t* %s\n\t* %s' % (Device_str, interface['Device_str']))
return Device_str
def create_config(self, filename = None, print_config = False):
config_str = '### Config file generated by dpdk_setup_ports.py ###\n\n'
config_str += '- version: 2\n'
config_str += " interfaces: ['%s']\n" % "', '".join([interface['Slot_str'] + interface.get("sub_interface", "") for interface in self.interfaces])
if self.speed > 10:
config_str += ' port_bandwidth_gb: %s\n' % self.speed
if self.prefix:
config_str += ' prefix: %s\n' % self.prefix
if self.zmq_pub_port:
config_str += ' zmq_pub_port: %s\n' % self.zmq_pub_port
if self.zmq_rpc_port:
config_str += ' zmq_rpc_port: %s\n' % self.zmq_rpc_port
config_str += ' port_info:\n'
for index, interface in enumerate(self.interfaces):
if 'ip' in interface:
self._exit_if_bad_ip(interface['ip'])
self._exit_if_bad_ip(interface['def_gw'])
config_str += ' '*6 + '- ip: %s\n' % interface['ip']
config_str += ' '*8 + 'default_gw: %s\n' % interface['def_gw']
else:
config_str += ' '*6 + '- dest_mac: %s' % self.verify_mac(interface['dest_mac'])
if interface.get('loopback_dest'):
config_str += " # MAC OF LOOPBACK TO IT'S DUAL INTERFACE\n"
else:
config_str += '\n'
config_str += ' '*8 + 'src_mac: %s\n' % self.verify_mac(interface['src_mac'])
if index % 2:
config_str += '\n' # dual if barrier
if not self.ignore_numa:
config_str += ' platform:\n'
if len(self.interfaces_per_numa.keys()) == 1 and -1 in self.interfaces_per_numa: # VM, use any cores
lcores_pool = sorted([lcore for lcores in self.lcores_per_numa.values() for lcore in lcores['all']])
config_str += ' '*6 + 'master_thread_id: %s\n' % (0 if self.has_zero_lcore else lcores_pool.pop(0))
config_str += ' '*6 + 'latency_thread_id: %s\n' % lcores_pool.pop(0)
lcores_per_dual_if = int(len(lcores_pool) * 2 / len(self.interfaces))
config_str += ' '*6 + 'dual_if:\n'
for i in range(0, len(self.interfaces), 2):
lcores_for_this_dual_if = list(map(str, sorted(lcores_pool[:lcores_per_dual_if])))
lcores_pool = lcores_pool[lcores_per_dual_if:]
if not lcores_for_this_dual_if:
raise DpdkSetup('lcores_for_this_dual_if is empty (internal bug, please report with details of setup)')
config_str += ' '*8 + '- socket: 0\n'
config_str += ' '*10 + 'threads: [%s]\n\n' % ','.join(lcores_for_this_dual_if)
else:
# we will take common minimum among all NUMAs, to satisfy all
lcores_per_dual_if = 99
extra_lcores = 1 if self.has_zero_lcore else 2
# worst case 3 iterations, to ensure master and "rx" have cores left
while (lcores_per_dual_if * sum(self.interfaces_per_numa.values()) / 2) + extra_lcores > sum([len(lcores['all']) for lcores in self.lcores_per_numa.values()]):
lcores_per_dual_if -= 1
for numa, lcores_dict in self.lcores_per_numa.items():
if not self.interfaces_per_numa[numa]:
continue
lcores_per_dual_if = min(lcores_per_dual_if, int(2 * len(lcores_dict['all']) / self.interfaces_per_numa[numa]))
lcores_pool = copy.deepcopy(self.lcores_per_numa)
# first, allocate lcores for dual_if section
dual_if_section = ' '*6 + 'dual_if:\n'
for i in range(0, len(self.interfaces), 2):
if self.interfaces[i]['Device_str'] == 'dummy':
numa = self.interfaces[i+1]['NUMA']
else:
numa = self.interfaces[i]['NUMA']
dual_if_section += ' '*8 + '- socket: %s\n' % numa
lcores_for_this_dual_if = lcores_pool[numa]['all'][:lcores_per_dual_if]
lcores_pool[numa]['all'] = lcores_pool[numa]['all'][lcores_per_dual_if:]
for lcore in lcores_for_this_dual_if:
if lcore in lcores_pool[numa]['main']:
lcores_pool[numa]['main'].remove(lcore)
elif lcore in lcores_pool[numa]['siblings']:
lcores_pool[numa]['siblings'].remove(lcore)
else:
raise DpdkSetup('lcore not in main nor in siblings list (internal bug, please report with details of setup)')
if not lcores_for_this_dual_if:
raise DpdkSetup('Not enough cores at NUMA %s. This NUMA has %s processing units and %s interfaces.' % (numa, len(self.lcores_per_numa[numa]), self.interfaces_per_numa[numa]))
dual_if_section += ' '*10 + 'threads: [%s]\n\n' % ','.join(list(map(str, sorted(lcores_for_this_dual_if))))
# take the cores left to master and rx
mains_left = [lcore for lcores in lcores_pool.values() for lcore in lcores['main']]
siblings_left = [lcore for lcores in lcores_pool.values() for lcore in lcores['siblings']]
if mains_left:
rx_core = mains_left.pop(0)
else:
rx_core = siblings_left.pop(0)
if self.has_zero_lcore:
master_core = 0
elif mains_left:
master_core = mains_left.pop(0)
else:
master_core = siblings_left.pop(0)
config_str += ' '*6 + 'master_thread_id: %s\n' % master_core
config_str += ' '*6 + 'latency_thread_id: %s\n' % rx_core
# add the dual_if section
config_str += dual_if_section
# verify config is correct YAML format
try:
yaml.safe_load(config_str)
except Exception as e:
raise DpdkSetup('Could not create correct yaml config.\nGenerated YAML:\n%s\nEncountered error:\n%s' % (config_str, e))
if print_config:
print(config_str)
if filename:
if os.path.exists(filename):
if not dpdk_nic_bind.confirm('File %s already exist, overwrite? (y/N)' % filename):
print('Skipping.')
return config_str
with open(filename, 'w') as f:
f.write(config_str)
print('Saved to %s.' % filename)
return config_str
# only load igb_uio if it's available
def load_igb_uio():
loaded_mods = dpdk_nic_bind.get_loaded_modules()
if 'igb_uio' in loaded_mods:
return True
if 'uio' not in loaded_mods:
ret = os.system('modprobe uio')
if ret:
return False
km = './ko/%s/igb_uio.ko' % dpdk_nic_bind.kernel_ver
if os.path.exists(km):
return os.system('insmod %s' % km) == 0
# try to compile igb_uio if it's missing
def compile_and_load_igb_uio():
loaded_mods = dpdk_nic_bind.get_loaded_modules()
if 'igb_uio' in loaded_mods:
return
if 'uio' not in loaded_mods:
ret = os.system('modprobe uio')
if ret:
print('Failed inserting uio module, please check if it is installed')
sys.exit(-1)
km = './ko/%s/igb_uio.ko' % dpdk_nic_bind.kernel_ver
if not os.path.exists(km):
print("ERROR: We don't have precompiled igb_uio.ko module for your kernel version")
print('Will try compiling automatically...')
build_path = '/tmp/trex-ko'
ret = os.system('mkdir -p %s' % build_path)
assert not ret, 'Makedirs failed'
ret = os.system('chmod -R 755 %s' % build_path)
assert not ret, 'chmod failed'
build_src_path = build_path + '/src'
shutil.rmtree(build_src_path, ignore_errors = True)
shutil.copytree('./ko/src', build_src_path)
try:
subprocess.check_output('make', cwd = build_src_path, stderr = subprocess.STDOUT, universal_newlines = True)
subprocess.check_output(['make', 'install'], cwd = build_src_path, stderr = subprocess.STDOUT, universal_newlines = True)
print('Success.\n')
except subprocess.CalledProcessError as e:
print('\n ERROR: Automatic compilation failed (return code: %s)' % e.returncode)
print(' Output:\n %s' % '\n '.join(e.output.splitlines()))
print('\nYou can try compiling yourself, using the following commands:')
print(' $mkdir -p /tmp/trex-ko')
print(' $cp -r ./ko/src /tmp/trex-ko')
print(' $cd /tmp/trex-ko/src')
print(' $make')
print(' $make install')
print(' $cd -')
print('Then, try to run TRex again.')
print('Note: you might need additional Linux packages for that:')
print(' * yum based (Fedora, CentOS, RedHat):')
print(' sudo yum install kernel-devel-`uname -r`')
print(' sudo yum group install "Development tools"')
print(' * apt based (Ubuntu):')
print(' sudo apt install linux-headers-`uname -r` build-essential')
sys.exit(-1)
km = os.path.join(build_path, dpdk_nic_bind.kernel_ver, 'igb_uio.ko')
ret = os.system('insmod %s' % km)
if ret:
print('Failed inserting igb_uio module')
sys.exit(-1)
class map_driver(object):
args=None;
cfg_file='/etc/trex_cfg.yaml'
parent_args = None
def pa():
return map_driver.parent_args
class DpdkSetup(Exception):
pass
class CIfMap:
def __init__(self, cfg_file):
self.m_cfg_file =cfg_file;
self.m_cfg_dict={};
self.m_devices={};
self.m_is_mellanox_mode=False;
def dump_error (self,err):
s="""%s
From this TRex version a configuration file must exist in /etc/ folder "
The name of the configuration file should be /etc/trex_cfg.yaml "
The minimum configuration file should include something like this
- version : 2 # version 2 of the configuration file
interfaces : ["03:00.0","03:00.1","13:00.1","13:00.0"] # list of the interfaces to bind run ./dpdk_nic_bind.py --status to see the list
port_limit : 2 # number of ports to use valid is 2,4,6,8,10,12
example of already bind devices
$ ./dpdk_nic_bind.py --status
Network devices using DPDK-compatible driver
============================================
0000:03:00.0 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv=igb_uio unused=
0000:03:00.1 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv=igb_uio unused=
0000:13:00.0 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv=igb_uio unused=
0000:13:00.1 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv=igb_uio unused=
Network devices using kernel driver
===================================
0000:02:00.0 '82545EM Gigabit Ethernet Controller (Copper)' if=eth2 drv=e1000 unused=igb_uio *Active*
Other network devices
=====================
""" % (err);
return s;
def raise_error (self,err):
s= self.dump_error (err)
raise DpdkSetup(s)
def set_only_mellanox_nics(self):
self.m_is_mellanox_mode=True;
def get_only_mellanox_nics(self):
return self.m_is_mellanox_mode
def read_pci (self,pci_id,reg_id):
out=subprocess.check_output(['setpci', '-s',pci_id, '%s.w' %(reg_id)])
out=out.decode(errors='replace');
return (out.strip());
def write_pci (self,pci_id,reg_id,val):
out=subprocess.check_output(['setpci','-s',pci_id, '%s.w=%s' %(reg_id,val)])
out=out.decode(errors='replace');
return (out.strip());
def tune_mlx_device (self,pci_id):
# set PCIe Read to 4K and not 512 ... need to add it to startup s
val=self.read_pci (pci_id,68)
if val[0]=='0':
#hypervisor does not give the right to write to this register
return;
if val[0]!='5':
val='5'+val[1:]
self.write_pci (pci_id,68,val)
assert(self.read_pci (pci_id,68)==val);
def get_mtu_mlx (self,dev_id):
if len(dev_id)>0:
try:
out=subprocess.check_output(['ifconfig', dev_id])
except Exception as e:
raise DpdkSetup(' "ifconfig %s" utility does not works, try to install it using "$yum install net-tools -y" on CentOS system' %(dev_id) )
out=out.decode(errors='replace');
obj=re.search(r'MTU:(\d+)',out,flags=re.MULTILINE|re.DOTALL);
if obj:
return int(obj.group(1));
else:
obj=re.search(r'mtu (\d+)',out,flags=re.MULTILINE|re.DOTALL);
if obj:
return int(obj.group(1));
else:
return -1
def set_mtu_mlx (self,dev_id,new_mtu):
if len(dev_id)>0:
out=subprocess.check_output(['ifconfig', dev_id,'mtu',str(new_mtu)])
out=out.decode(errors='replace');
def set_max_mtu_mlx_device(self,dev_id):
mtu=9*1024+22
dev_mtu=self.get_mtu_mlx (dev_id);
if (dev_mtu>0) and (dev_mtu!=mtu):
self.set_mtu_mlx(dev_id,mtu);
if self.get_mtu_mlx(dev_id) != mtu:
print("Could not set MTU to %d" % mtu)
sys.exit(-1);
def disable_flow_control_mlx_device (self,dev_id):
if len(dev_id)>0:
my_stderr = open("/dev/null","wb")
cmd ='ethtool -A '+dev_id + ' rx off tx off '
subprocess.call(cmd, stdout=my_stderr,stderr=my_stderr, shell=True)
my_stderr.close();
def check_ofed_version (self):
ofed_info='/usr/bin/ofed_info'
ofed_ver_re = re.compile('.*[-](\d)[.](\d)[-].*')
ofed_ver = 46
ofed_ver_show = '4.6'
if not os.path.isfile(ofed_info):
print("OFED %s is not installed on this setup" % ofed_info)
sys.exit(-1);
try:
out = subprocess.check_output([ofed_info])
except Exception as e:
print("OFED %s can't run " % (ofed_info))
sys.exit(-1);
lines=out.splitlines();
if len(lines)>1:
m= ofed_ver_re.match(str(lines[0]))
if m:
ver=int(m.group(1))*10+int(m.group(2))
if ver < ofed_ver:
print("installed OFED version is '%s' should be at least '%s' and up" % (lines[0],ofed_ver_show))
sys.exit(-1);
else:
print("not found valid OFED version '%s' " % (lines[0]))
sys.exit(-1);
def verify_ofed_os(self):
err_msg = 'Warning: Mellanox NICs where tested only with RedHat/CentOS 7.6\n'
err_msg += 'Correct usage with other Linux distributions is not guaranteed.'
try:
dist = platform.dist()
if dist[0] not in ('redhat', 'centos') or not dist[1].startswith('7.6'):
print(err_msg)
except Exception as e:
print('Error while determining OS type: %s' % e)
def load_config_file (self):
fcfg=self.m_cfg_file
if not os.path.isfile(fcfg) :
self.raise_error ("There is no valid configuration file %s\n" % fcfg)
try:
stream = open(fcfg, 'r')
self.m_cfg_dict= yaml.safe_load(stream)
except Exception as e:
print(e);
raise e
stream.close();
cfg_dict = self.m_cfg_dict[0]
if 'version' not in cfg_dict:
raise DpdkSetup("Configuration file %s is old, it should include version field\n" % fcfg )
if int(cfg_dict['version'])<2 :
raise DpdkSetup("Configuration file %s is old, expected version 2, got: %s\n" % (fcfg, cfg_dict['version']))
if 'interfaces' not in self.m_cfg_dict[0]:
raise DpdkSetup("Configuration file %s is old, it should include interfaces field with even number of elements" % fcfg)
if_list= if_list_remove_sub_if(self.m_cfg_dict[0]['interfaces']);
l=len(if_list);
if l > 24:
raise DpdkSetup("Configuration file %s should include interfaces field with maximum 24 elements, got: %s." % (fcfg,l))
if l % 2:
raise DpdkSetup("Configuration file %s should include even number of interfaces, got: %s" % (fcfg,l))
if 'port_limit' in cfg_dict:
if cfg_dict['port_limit'] > len(if_list):
raise DpdkSetup('Error: port_limit should not be higher than number of interfaces in config file: %s\n' % fcfg)
if cfg_dict['port_limit'] % 2:
raise DpdkSetup('Error: port_limit in config file must be even number, got: %s\n' % cfg_dict['port_limit'])
if cfg_dict['port_limit'] <= 0:
raise DpdkSetup('Error: port_limit in config file must be positive number, got: %s\n' % cfg_dict['port_limit'])
if pa() and pa().limit_ports is not None and pa().limit_ports > len(if_list):
raise DpdkSetup('Error: --limit-ports CLI argument (%s) must not be higher than number of interfaces (%s) in config file: %s\n' % (pa().limit_ports, len(if_list), fcfg))
if cfg_dict.get('stack') == 'linux_based':
self.verify_ethtool()
def verify_ethtool(self):
try:
FNULL = open(os.devnull, 'w')
subprocess.check_call(['ethtool', '--version'], stdout=FNULL, stderr=subprocess.STDOUT)
except Exception as e:
raise DpdkSetup('Error: "linux_based" stack mode requires ethtool installed on your machine. Install it or do not use this mode.')
def do_bind_all(self, drv, pci, force = False):
assert type(pci) is list
cmd = '{ptn} dpdk_nic_bind.py --bind={drv} {pci} {frc}'.format(
ptn = sys.executable,
drv = drv,
pci = ' '.join(pci),
frc = '--force' if force else '')
print(cmd)
return os.system(cmd)
# pros: no need to compile .ko per Kernel version
# cons: need special config/hw (not always works)
def try_bind_to_vfio_pci(self, to_bind_list):
krnl_params_file = '/proc/cmdline'
if not os.path.exists(krnl_params_file):
raise VFIOBindErr('Could not find file with Kernel boot parameters: %s' % krnl_params_file)
with open(krnl_params_file) as f:
krnl_params = f.read()
# IOMMU is always enabled on Power systems
if march != 'ppc64le' and 'iommu=' not in krnl_params:
raise VFIOBindErr('vfio-pci is not an option here')
if 'vfio_pci' not in dpdk_nic_bind.get_loaded_modules():
ret = os.system('modprobe vfio_pci')
if ret:
raise VFIOBindErr('Could not load vfio_pci')
ret = self.do_bind_all('vfio-pci', to_bind_list)
if ret:
raise VFIOBindErr('Binding to vfio_pci failed')
def pci_name_to_full_name (self,pci_name):
if pci_name == 'dummy':
return pci_name
c='[0-9A-Fa-f]';
sp='[:]'
s_short=c+c+sp+c+c+'[.]'+c;
s_full=c+c+c+c+sp+s_short
re_full = re.compile(s_full)
re_short = re.compile(s_short)
if re_short.match(pci_name):
return '0000:'+pci_name
if re_full.match(pci_name):
return pci_name
err=" %s is not a valid pci address \n" %pci_name;
raise DpdkSetup(err)
def run_dpdk_lspci (self):
dpdk_nic_bind.get_nic_details()
self.m_devices= dpdk_nic_bind.devices
def get_prefix(self):
if pa().prefix:
return pa().prefix
return self.m_cfg_dict[0].get('prefix', '')
def preprocess_astf_file_if_needed(self):
""" check if we are in astf batch mode, in case we are convert the profile to json in tmp"""
if not pa() or not pa().astf or pa().interactive:
return
input_file = pa().file
if not input_file:
return
instance_name = ''
prefix = self.get_prefix()
if prefix:
instance_name = '-' + prefix
dst_json_file = "/tmp/astf{instance}.json".format(instance=instance_name)
extension = os.path.splitext(input_file)[1]
if extension == '.json':
shutil.copyfile(input_file, dst_json_file)
os.chmod(dst_json_file, 0o777)
return
elif extension != '.py':
raise DpdkSetup('ERROR when running with --astf mode, you need to have a new Python profile format (.py) and not YAML')
print('converting astf profile %s to json %s' % (input_file, dst_json_file))
# imports from trex.astf
cur_path = os.path.abspath(os.path.dirname(__file__))
trex_path = os.path.join(cur_path, 'automation', 'trex_control_plane', 'interactive')
if trex_path not in sys.path:
sys.path.insert(1, trex_path)
from trex.astf.trex_astf_profile import ASTFProfile
from trex.astf.sim import decode_tunables
tunables = {}
if pa().tunable:
tunables = decode_tunables(pa().tunable)
try:
profile = ASTFProfile.load(input_file, **tunables)
json_content = profile.to_json_str()
except Exception as e:
raise DpdkSetup('ERROR: Could not convert astf profile to JSON:\n%s' % e)
with open(dst_json_file, 'w') as f:
f.write(json_content)
os.chmod(dst_json_file, 0o777)
def verify_stf_file(self):
""" check the input file of STF """
if not pa() or not pa().file or pa().astf:
return
extension = os.path.splitext(pa().file)[1]
if extension == '.py':
raise DpdkSetup('ERROR: Python files can not be used with STF mode, did you forget "--astf" flag?')
elif extension != '.yaml':
pass # should we fail here?
def is_hugepage_file_exits(self,socket_id):
t = ['2048','1048576']
for obj in t:
filename = '/sys/devices/system/node/node{}/hugepages/hugepages-{}kB/nr_hugepages'.format(socket_id,obj)
if os.path.isfile(filename):
return (True,filename,int(obj))
return (False,None,None)
def config_hugepages(self, wanted_count = None):
mount_output = subprocess.check_output('mount', stderr = subprocess.STDOUT).decode(errors='replace')
if 'hugetlbfs' not in mount_output:
huge_mnt_dir = '/mnt/huge'
if not os.path.isdir(huge_mnt_dir):
print("Creating huge node")
os.makedirs(huge_mnt_dir)
os.system('mount -t hugetlbfs nodev %s' % huge_mnt_dir)
for socket_id in range(2):
r = self.is_hugepage_file_exits(socket_id)
if not r[0]:
if socket_id == 0:
print('WARNING: hugepages config file does not exist!')
continue
if wanted_count is None:
if self.m_cfg_dict[0].get('low_end', False):
if socket_id == 0:
if pa() and pa().limit_ports:
if_count = pa().limit_ports
else:
if_count = self.m_cfg_dict[0].get('port_limit', len(self.m_cfg_dict[0]['interfaces']))
wanted_count = 20 + 40 * if_count
else:
wanted_count = 1 # otherwise, DPDK will not be able to see the device
else:
wanted_count = 2048
if r[2] > 2048:
wanted_count = wanted_count / 1024
if wanted_count < 1 :
wanted_count = 1
filename = r[1]
with open(filename) as f:
configured_hugepages = int(f.read())
if configured_hugepages < wanted_count:
os.system('echo %d > %s' % (wanted_count, filename))
time.sleep(0.1)
with open(filename) as f: # verify
configured_hugepages = int(f.read())
if configured_hugepages < wanted_count:
print('WARNING: tried to configure %d hugepages for socket %d, but result is: %d' % (wanted_count, socket_id, configured_hugepages))
def get_core_for_services(self):
default_core = 0
cfg_dict = self.m_cfg_dict[0]
if cfg_dict.get('services_core', False):
# 1st priority from services_core
return cfg_dict['services_core']
elif cfg_dict.get('low_end', False):
# if low_end, use low_end_core or default if not supplied
return cfg_dict.get('low_end_core', default_core)
elif cfg_dict.get('platform', False):
# 2nd priority from master_core
return cfg_dict['platform'].get('master_thread_id', default_core)
else:
# 3rd priority is zero core
return default_core
def run_servers(self):
''' Run both scapy, bird & Emu servers according to pa'''
if not pa():
return
services_core = self.get_core_for_services()
if should_scapy_server_run():
ret = os.system('{sys_exe} general_daemon_server restart -n {name} -c {core} --py -e "{exe}" -r -d -i'.format(sys_exe = sys.executable,
core = services_core,
name = 'Scapy',
exe = '-m trex.scapy_server.scapy_zmq_server'))
if ret:
print("Could not start scapy daemon server, which is needed by GUI to create packets.\nIf you don't need it, use --no-scapy-server flag.")
sys.exit(-1)
if pa().bird_server:
ret = os.system('{sys_exe} general_daemon_server restart -n {name} -c {core} --py -e "{exe}" -i'.format(sys_exe = sys.executable,
name = 'PyBird',
core = services_core,
exe = '-m trex.pybird_server.pybird_zmq_server'))
if ret:
print("Could not start bird server\nIf you don't need it, don't use --bird-server flag.")
sys.exit(-1)
if pa().emu:
emu_zmq_tcp_flag = '--emu-zmq-tcp' if pa().emu_zmq_tcp else ''
exe = './trex-emu {emu_zmq_tcp}'.format(emu_zmq_tcp = emu_zmq_tcp_flag)
ret = os.system('{sys_exe} general_daemon_server restart -n {name} -c {core} --sudo -e "{exe}"'.format(sys_exe = sys.executable,
core = services_core,
name = 'Emu',
exe = exe))
if ret:
print("Could not start emu service\nIf you don't need it, don't use -emu flag.")
sys.exit(-1)
# check vdev Linux interfaces status
# return True if interfaces are vdev
def check_vdev(self, if_list):
if not if_list:
return
af_names = []
ifname_re = re.compile('iface\s*=\s*([^\s,]+)')
found_vdev = False
found_pdev = False
for iface in if_list:
if iface == 'dummy':
continue
elif '--vdev' in iface:
found_vdev = True
if 'net_af_packet' in iface:
res = ifname_re.search(iface)
if res:
af_names.append(res.group(1))
elif ':' not in iface: # no PCI => assume af_packet
found_vdev = True
af_names.append(iface)
else:
found_pdev = True
if found_vdev:
if found_pdev:
raise DpdkSetup('You have mix of vdev and pdev interfaces in config file!')
for name in af_names:
if not os.path.exists('/sys/class/net/%s' % name):
raise DpdkSetup('ERROR: Could not find Linux interface %s.' % name)
oper_state = '/sys/class/net/%s/operstate' % name
if os.path.exists(oper_state):
with open(oper_state) as f:
f_cont = f.read().strip()
if f_cont in ('down', 'DOWN'):
raise DpdkSetup('ERROR: Requested Linux interface %s is DOWN.' % name)
return found_vdev
def check_trex_running(self, if_list):
if if_list and map_driver.args.parent and self.m_cfg_dict[0].get('enable_zmq_pub', True):
publisher_port = self.m_cfg_dict[0].get('zmq_pub_port', 4500)
pid = dpdk_nic_bind.get_tcp_port_usage(publisher_port)
if pid:
cmdline = dpdk_nic_bind.read_pid_cmdline(pid)
print('ZMQ port is used by following process:\npid: %s, cmd: %s' % (pid, cmdline))
sys.exit(-1)
# verify that all interfaces of i40e NIC are in use by current instance of TRex
def check_i40e_binds(self, if_list):
# i40e device IDs taked from dpdk/drivers/net/i40e/base/i40e_devids.h
i40e_device_ids = [0x1572, 0x1574, 0x1580, 0x1581, 0x1583, 0x1584, 0x1585, 0x1586, 0x1587, 0x1588, 0x1589, 0x158A, 0x158B]
iface_without_slash = set()
for iface in if_list:
iface_without_slash.add(self.split_pci_key(iface))
show_warning_devices = set()
unbind_devices = set()
for iface in iface_without_slash:
if iface == 'dummy':
continue
iface = self.split_pci_key(iface)
if self.m_devices[iface]['Device'] not in i40e_device_ids: # not i40e
return
iface_pci = iface.split('.')[0]
for device in self.m_devices.values():
if device['Slot'] in iface_without_slash: # we use it
continue
if iface_pci == device['Slot'].split('.')[0]:
if device.get('Driver_str') == 'i40e':
if pa() and pa().unbind_unused_ports:
# if --unbind-unused-ports is set we unbind ports that are not
# used by TRex
unbind_devices.add(device['Slot'])
else:
print('ERROR: i40e interface %s is under Linux and will interfere with TRex interface %s' % (device['Slot'], iface))
print('See following link for more information: https://trex-tgn.cisco.com/youtrack/issue/trex-528')
print('Unbind the interface from Linux with following command:')
print(' sudo ./dpdk_nic_bind.py -u %s' % device['Slot'])
print('')
sys.exit(-1)
if device.get('Driver_str') in dpdk_nic_bind.dpdk_drivers:
show_warning_devices.add(device['Slot'])
for dev in show_warning_devices:
print('WARNING: i40e interface %s is under DPDK driver and might interfere with current TRex interfaces.' % dev)
if unbind_devices:
print('Unbinding unused i40e interfaces: %s' % unbind_devices)
dpdk_nic_bind.unbind_all(unbind_devices, force=True)
def do_run (self, only_check_all_mlx=False):
""" returns code that specifies if interfaces are Mellanox/Napatech etc. """
self.load_config_file()
self.preprocess_astf_file_if_needed()
self.verify_stf_file()
if not pa() or pa().dump_interfaces is None or (pa().dump_interfaces == [] and pa().cfg):
if_list = if_list_remove_sub_if(self.m_cfg_dict[0]['interfaces'])
else:
if_list = pa().dump_interfaces
if not if_list:
self.run_dpdk_lspci()
for dev in self.m_devices.values():
if dev.get('Driver_str') in dpdk_nic_bind.dpdk_drivers + dpdk_nic_bind.dpdk_and_kernel:
if_list.append(dev['Slot'])
if self.check_vdev(if_list):
self.check_trex_running(if_list)
self.run_servers()
# no need to config hugepages
return
self.run_dpdk_lspci()
if_list = list(map(self.pci_name_to_full_name, if_list))
Broadcom_cnt=0;
# check how many mellanox cards we have
Mellanox_cnt=0;
dummy_cnt=0
for key in if_list:
if key == 'dummy':
dummy_cnt += 1
continue
key = self.split_pci_key(key)
if key not in self.m_devices:
err=" %s does not exist " %key;
raise DpdkSetup(err)
if 'Vendor_str' not in self.m_devices[key]:
err=" %s does not have Vendor_str " %key;
raise DpdkSetup(err)
if 'Mellanox' in self.m_devices[key]['Vendor_str']:
Mellanox_cnt += 1
if 'Broadcom' in self.m_devices[key]['Vendor_str']:
Broadcom_cnt += 1
if not (pa() and pa().dump_interfaces):
if (Mellanox_cnt > 0) and ((Mellanox_cnt + dummy_cnt) != len(if_list)):
err = "All driver should be from one vendor. You have at least one driver from Mellanox but not all."
raise DpdkSetup(err)
if Mellanox_cnt > 0:
self.set_only_mellanox_nics()
if self.get_only_mellanox_nics():
if not pa().no_ofed_check:
self.verify_ofed_os()
self.check_ofed_version()
for key in if_list:
if key == 'dummy':
continue
if pa().no_ofed_check: # in case of no-ofed don't optimized for Azure
continue
key = self.split_pci_key(key)
if 'Virtual' not in self.m_devices[key]['Device_str']:
pci_id = self.m_devices[key]['Slot_str']
self.tune_mlx_device(pci_id)
if 'Interface' in self.m_devices[key]:
dev_ids = self.m_devices[key]['Interface'].split(",")
for dev_id in dev_ids:
self.disable_flow_control_mlx_device (dev_id)
self.set_max_mtu_mlx_device(dev_id)
if only_check_all_mlx:
if Mellanox_cnt > 0:
sys.exit(MLX_EXIT_CODE);
else:
sys.exit(0);
self.check_i40e_binds(if_list)
self.check_trex_running(if_list)
self.config_hugepages() # should be after check of running TRex
self.run_servers()
Napatech_cnt=0;
to_bind_list = []
for key in if_list:
if key == 'dummy':
continue
key = self.split_pci_key(key)
if key not in self.m_devices:
err=" %s does not exist " %key;
raise DpdkSetup(err)
if (is_napatech(self.m_devices[key])):
# These adapters doesn't need binding
Napatech_cnt += 1
continue
if self.m_devices[key].get('Driver_str') not in (dpdk_nic_bind.dpdk_drivers + dpdk_nic_bind.dpdk_and_kernel):
to_bind_list.append(key)
if Napatech_cnt:
# This is currently a hack needed until the DPDK NTACC PMD can do proper