#! /bin/bash

"source" "find_python.sh" "--local"

"exec" "$PYTHON" "$0" "$@"

# hhaim

import sys

try:

xrange # Python 2

except NameError:

xrange = range # Python 3

import os

python_ver = 'python%s' % sys.version_info[0]

yaml_path = os.path.join('external_libs', 'pyyaml-3.11', python_ver)

if yaml_path not in sys.path:

sys.path.append(yaml_path)

import yaml

import dpdk_nic_bind

import re

import argparse

import copy

import shlex

import traceback

from collections import defaultdict, OrderedDict

import subprocess

import platform

import stat

import time

import shutil

import signal

import glob

from dpdk_nic_bind import is_napatech

march = os.uname()[4]

# exit code is Important should be

# -1 : don't continue

# 0 : no errors - no need to load mlx share object

# 16 : no errors - mlx4 share object should be loaded

# 32 : no errors - mlx5 share object should be loaded

# 48 : no errors - both mlx4/mlx5 share object should be loaded

# 64 : no errors - napatech 3GD should be running

MLX4_EXIT_CODE = 16

MLX5_EXIT_CODE = 32

NTACC_EXIT_CODE = 64

class VFIOBindErr(Exception): pass

PATH_ARR = os.getenv('PATH', '').split(':')

for path in ['/usr/local/sbin', '/usr/sbin', '/sbin']:

if path not in PATH_ARR:

PATH_ARR.append(path)

os.environ['PATH'] = ':'.join(PATH_ARR)

def if_list_remove_sub_if(if_list):

return if_list

class ConfigCreator(object):

mandatory_interface_fields = ['Slot_str', 'Device_str', 'NUMA']

_2hex_re = '[\da-fA-F]{2}'

mac_re = re.compile('^({0}:){{5}}{0}$'.format(_2hex_re))

if march == 'ppc64le':

MAX_LCORE_NUM = 159

else:

MAX_LCORE_NUM = 63

# cpu_topology - dict: physical processor -> physical core -> logical processing unit (thread)

# interfaces - array of dicts per interface, should include "mandatory_interface_fields" values

def __init__(self, cpu_topology, interfaces, include_lcores = [], exclude_lcores = [], only_first_thread = False, zmq_rpc_port = None, zmq_pub_port = None, prefix = None, ignore_numa = False, stack=None):

self.cpu_topology = copy.deepcopy(cpu_topology)

self.interfaces = copy.deepcopy(interfaces)

del cpu_topology

del interfaces

assert isinstance(self.cpu_topology, dict), 'Type of cpu_topology should be dict, got: %s' % type(self.cpu_topology)

assert len(self.cpu_topology.keys()) > 0, 'cpu_topology should contain at least one processor'

assert isinstance(self.interfaces, list), 'Type of interfaces should be list, got: %s' % type(list)

assert len(self.interfaces) % 2 == 0, 'Should be even number of interfaces, got: %s' % len(self.interfaces)

assert len(self.interfaces) >= 2, 'Should be at least two interfaces, got: %s' % len(self.interfaces)

assert isinstance(include_lcores, list), 'include_lcores should be list, got: %s' % type(include_lcores)

assert isinstance(exclude_lcores, list), 'exclude_lcores should be list, got: %s' % type(exclude_lcores)

assert len(self.interfaces) >= 2, 'Should be at least two interfaces, got: %s' % len(self.interfaces)

if only_first_thread:

for cores in self.cpu_topology.values():

for core in cores.keys():

cores[core] = cores[core][:1]

include_lcores = [int(x) for x in include_lcores]

exclude_lcores = [int(x) for x in exclude_lcores]

self.has_zero_lcore = False

self.lcores_per_numa = {}

total_lcores = 0

for numa, cores in self.cpu_topology.items():

self.lcores_per_numa[numa] = {'main': [], 'siblings': [], 'all': []}

for core, lcores in cores.items():

total_lcores += len(lcores)

for lcore in list(lcores):

if include_lcores and lcore not in include_lcores:

cores[core].remove(lcore)

elif exclude_lcores and lcore in exclude_lcores:

cores[core].remove(lcore)

elif lcore > self.MAX_LCORE_NUM:

cores[core].remove(lcore)

if 0 in lcores:

self.has_zero_lcore = True

lcores.remove(0)

self.lcores_per_numa[numa]['siblings'].extend(lcores)

else:

self.lcores_per_numa[numa]['main'].extend(lcores[:1])

self.lcores_per_numa[numa]['siblings'].extend(lcores[1:])

self.lcores_per_numa[numa]['all'].extend(lcores)

for interface in self.interfaces:

for mandatory_interface_field in ConfigCreator.mandatory_interface_fields:

if mandatory_interface_field not in interface:

raise DpdkSetup("Expected '%s' field in interface dictionary, got: %s" % (mandatory_interface_field, interface))

Device_str = self._verify_devices_same_type(self.interfaces)

if '200Gb' in Device_str:

self.speed = 200

elif '100Gb' in Device_str:

self.speed = 100

elif '50Gb' in Device_str:

self.speed = 50

elif '40Gb' in Device_str:

self.speed = 40

elif '25Gb' in Device_str:

self.speed = 25

elif '20Gb' in Device_str:

self.speed = 20

else:

self.speed = 10

minimum_required_lcores = len(self.interfaces) // 2 + 2

if total_lcores < minimum_required_lcores:

raise DpdkSetup('Your system should have at least %s cores for %s interfaces, and it has: %s.' %

(minimum_required_lcores, len(self.interfaces), total_lcores))

interfaces_per_numa = defaultdict(int)

for i in range(0, len(self.interfaces), 2):

if self.interfaces[i]['Slot_str'] == 'dummy':

numa = self.interfaces[i+1]['NUMA']

other_if_numa = self.interfaces[i]['NUMA']

else:

numa = self.interfaces[i]['NUMA']

other_if_numa = self.interfaces[i+1]['NUMA']

if numa != other_if_numa and not ignore_numa and self.interfaces[i]['Slot_str'] != 'dummy' and self.interfaces[i+1]['Slot_str'] != 'dummy':

raise DpdkSetup('NUMA of each pair of interfaces should be the same. Got NUMA %s for client interface %s, NUMA %s for server interface %s' %

(numa, self.interfaces[i]['Slot_str'], self.interfaces[i+1]['NUMA'], self.interfaces[i+1]['Slot_str']))

interfaces_per_numa[numa] += 2

self.interfaces_per_numa = interfaces_per_numa

self.prefix = prefix

self.zmq_pub_port = zmq_pub_port

self.zmq_rpc_port = zmq_rpc_port

self.ignore_numa = ignore_numa

self.stack = stack

@staticmethod

def verify_mac(mac_string):

if not ConfigCreator.mac_re.match(mac_string):

raise DpdkSetup('MAC address should be in format of 12:34:56:78:9a:bc, got: %s' % mac_string)

return mac_string.lower()

@staticmethod

def _exit_if_bad_ip(ip):

if not ConfigCreator._verify_ip(ip):

raise DpdkSetup("Got bad IP %s" % ip)

@staticmethod

def _verify_ip(ip):

a = ip.split('.')

if len(a) != 4:

return False

for x in a:

if not x.isdigit():

return False

i = int(x)

if i < 0 or i > 255:

return False

return True

@staticmethod

def _verify_devices_same_type(interfaces_list):

Device_str = interfaces_list[0]['Device_str']

TrexDrv = interfaces_list[0].get('TRex_Driver',"not_valid_drv")

if Device_str == 'dummy':

return Device_str

for interface in interfaces_list:

if interface['Device_str'] == 'dummy':

continue

odrv = interface.get('TRex_Driver',"not_valid_drv")

if TrexDrv != odrv:

raise DpdkSetup('Interfaces should be of same type, got:\n\t* %s\n\t* %s' % (TrexDrv, odrv))

return Device_str

def create_config(self, filename = None, print_config = False):

config_str = '### Config file generated by dpdk_setup_ports.py ###\n\n'

config_str += '- version: 2\n'

config_str += " interfaces: ['%s']\n" % "', '".join([interface['Slot_str'] + interface.get("sub_interface", "") for interface in self.interfaces])

if self.speed > 10:

config_str += ' port_bandwidth_gb: %s\n' % self.speed

if self.stack is not None:

config_str += ' stack: %s\n' % self.stack

if self.prefix:

config_str += ' prefix: %s\n' % self.prefix

if self.zmq_pub_port:

config_str += ' zmq_pub_port: %s\n' % self.zmq_pub_port

if self.zmq_rpc_port:

config_str += ' zmq_rpc_port: %s\n' % self.zmq_rpc_port

config_str += ' port_info:\n'

for index, interface in enumerate(self.interfaces):

if 'ip' in interface:

self._exit_if_bad_ip(interface['ip'])

self._exit_if_bad_ip(interface['def_gw'])

config_str += ' '*6 + '- ip: %s\n' % interface['ip']

config_str += ' '*8 + 'default_gw: %s\n' % interface['def_gw']

else:

config_str += ' '*6 + '- dest_mac: %s' % self.verify_mac(interface['dest_mac'])

if interface.get('loopback_dest'):

config_str += " # MAC OF LOOPBACK TO IT'S DUAL INTERFACE\n"

else:

config_str += '\n'

config_str += ' '*8 + 'src_mac: %s\n' % self.verify_mac(interface['src_mac'])

if index % 2:

config_str += '\n' # dual if barrier

if not self.ignore_numa:

config_str += ' platform:\n'

if len(self.interfaces_per_numa.keys()) == 1 and -1 in self.interfaces_per_numa: # VM, use any cores

lcores_pool = sorted([lcore for lcores in self.lcores_per_numa.values() for lcore in lcores['all']])

config_str += ' '*6 + 'master_thread_id: %s\n' % (0 if self.has_zero_lcore else lcores_pool.pop(0))

config_str += ' '*6 + 'latency_thread_id: %s\n' % lcores_pool.pop(0)

lcores_per_dual_if = int(len(lcores_pool) * 2 / len(self.interfaces))

config_str += ' '*6 + 'dual_if:\n'

for i in range(0, len(self.interfaces), 2):

lcores_for_this_dual_if = list(map(str, sorted(lcores_pool[:lcores_per_dual_if])))

lcores_pool = lcores_pool[lcores_per_dual_if:]

if not lcores_for_this_dual_if:

raise DpdkSetup('lcores_for_this_dual_if is empty (internal bug, please report with details of setup)')

config_str += ' '*8 + '- socket: 0\n'

config_str += ' '*10 + 'threads: [%s]\n\n' % ','.join(lcores_for_this_dual_if)

else:

# we will take common minimum among all NUMAs, to satisfy all

lcores_per_dual_if = 99

extra_lcores = 1 if self.has_zero_lcore else 2

# worst case 3 iterations, to ensure master and "rx" have cores left

while (lcores_per_dual_if * sum(self.interfaces_per_numa.values()) / 2) + extra_lcores > sum([len(lcores['all']) for lcores in self.lcores_per_numa.values()]):

lcores_per_dual_if -= 1

for numa, lcores_dict in self.lcores_per_numa.items():

if not self.interfaces_per_numa[numa]:

continue

lcores_per_dual_if = min(lcores_per_dual_if, int(2 * len(lcores_dict['all']) / self.interfaces_per_numa[numa]))

lcores_pool = copy.deepcopy(self.lcores_per_numa)

# first, allocate lcores for dual_if section

dual_if_section = ' '*6 + 'dual_if:\n'

for i in range(0, len(self.interfaces), 2):

if self.interfaces[i]['Device_str'] == 'dummy':

numa = self.interfaces[i+1]['NUMA']

else:

numa = self.interfaces[i]['NUMA']

dual_if_section += ' '*8 + '- socket: %s\n' % numa

lcores_for_this_dual_if = lcores_pool[numa]['all'][:lcores_per_dual_if]

lcores_pool[numa]['all'] = lcores_pool[numa]['all'][lcores_per_dual_if:]

for lcore in lcores_for_this_dual_if:

if lcore in lcores_pool[numa]['main']:

lcores_pool[numa]['main'].remove(lcore)

elif lcore in lcores_pool[numa]['siblings']:

lcores_pool[numa]['siblings'].remove(lcore)

else:

raise DpdkSetup('lcore not in main nor in siblings list (internal bug, please report with details of setup)')

if not lcores_for_this_dual_if:

raise DpdkSetup('Not enough cores at NUMA %s. This NUMA has %s processing units and %s interfaces.' % (numa, len(self.lcores_per_numa[numa]), self.interfaces_per_numa[numa]))

dual_if_section += ' '*10 + 'threads: [%s]\n\n' % ','.join(list(map(str, sorted(lcores_for_this_dual_if))))

# take the cores left to master and rx

mains_left = [lcore for lcores in lcores_pool.values() for lcore in lcores['main']]

siblings_left = [lcore for lcores in lcores_pool.values() for lcore in lcores['siblings']]

if mains_left:

rx_core = mains_left.pop(0)

else:

rx_core = siblings_left.pop(0)

if self.has_zero_lcore:

master_core = 0

elif mains_left:

master_core = mains_left.pop(0)

else:

master_core = siblings_left.pop(0)

config_str += ' '*6 + 'master_thread_id: %s\n' % master_core

config_str += ' '*6 + 'latency_thread_id: %s\n' % rx_core

# add the dual_if section

config_str += dual_if_section

# verify config is correct YAML format

try:

yaml.safe_load(config_str)

except Exception as e:

raise DpdkSetup('Could not create correct yaml config.\nGenerated YAML:\n%s\nEncountered error:\n%s' % (config_str, e))

if print_config:

print(config_str)

if filename:

if os.path.exists(filename) and not map_driver.args.no_prompt:

if not dpdk_nic_bind.confirm('File %s already exist, overwrite? (y/N)' % filename):

print('Skipping.')

return config_str

with open(filename, 'w') as f:

f.write(config_str)

print('Saved to %s.' % filename)

return config_str

# only load igb_uio if it's available

def load_igb_uio():

loaded_mods = dpdk_nic_bind.get_loaded_modules()

if 'igb_uio' in loaded_mods:

return True

if 'uio' not in loaded_mods:

ret = os.system('modprobe uio')

if ret:

return False

km = './ko/%s/igb_uio.ko' % dpdk_nic_bind.kernel_ver

if os.path.exists(km):

return os.system('insmod %s' % km) == 0

# try to compile igb_uio if it's missing

def compile_and_load_igb_uio():

loaded_mods = dpdk_nic_bind.get_loaded_modules()

if 'igb_uio' in loaded_mods:

return

if 'uio' not in loaded_mods:

ret = os.system('modprobe uio')

if ret:

print('Failed inserting uio module, please check if it is installed.')

sys.exit(-1)

km = './ko/%s/igb_uio.ko' % dpdk_nic_bind.kernel_ver

if not os.path.exists(km):

print("ERROR: We don't have precompiled igb_uio.ko module for your kernel version.")

print('Will try compiling automatically...')

build_path = '/tmp/trex-ko'

ret = os.system('mkdir -p %s' % build_path)

assert not ret, 'Makedirs failed'

ret = os.system('chmod -R 755 %s' % build_path)

assert not ret, 'chmod failed'

build_src_path = build_path + '/src'

shutil.rmtree(build_src_path, ignore_errors = True)

shutil.copytree('./ko/src', build_src_path)

try:

subprocess.check_output('make', cwd = build_src_path, stderr = subprocess.STDOUT, universal_newlines = True)

subprocess.check_output(['make', 'install'], cwd = build_src_path, stderr = subprocess.STDOUT, universal_newlines = True)

print('Success.\n')

except subprocess.CalledProcessError as e:

print('\n ERROR: Automatic compilation failed (return code: %s)' % e.returncode)

print(' Output:\n %s' % '\n '.join(e.output.splitlines()))

print('\nYou can try compiling yourself, using the following commands:')

print(' $mkdir -p /tmp/trex-ko')

print(' $cp -r ./ko/src /tmp/trex-ko')

print(' $cd /tmp/trex-ko/src')

print(' $make')

print(' $make install')

print(' $cd -')

print('Then, try to run TRex again.')

print('Note: you might need additional Linux packages for that:')

print(' * yum based (Fedora, CentOS, RedHat):')

print(' sudo yum install kernel-devel-`uname -r`')

print(' sudo yum group install "Development tools"')

print(' * apt based (Ubuntu):')

print(' sudo apt install linux-headers-`uname -r` build-essential')

sys.exit(-1)

km = os.path.join(build_path, dpdk_nic_bind.kernel_ver, 'igb_uio.ko')

ret = os.system('insmod %s' % km)

if ret:

print('Failed inserting igb_uio module.')

sys.exit(-1)

class map_driver(object):

args=None;

cfg_file='/etc/trex_cfg.yaml'

parent_args = None

def pa():

return map_driver.parent_args

class DpdkSetup(Exception):

pass

class CIfMap:

def __init__(self, cfg_file):

self.m_cfg_file =cfg_file;

self.m_cfg_dict={};

self.m_devices={};

self.m_is_mellanox_mode=False;

def dump_error (self,err):

s="""%s

From this TRex version a configuration file must exist in /etc/ folder "

The name of the configuration file should be /etc/trex_cfg.yaml "

The minimum configuration file should include something like this:

- version : 2 # version 2 of the configuration file

interfaces : ["03:00.0","03:00.1","13:00.1","13:00.0"] # list of the interfaces to bind run ./dpdk_nic_bind.py --status to see the list

port_limit : 2 # number of ports to use valid is 2,4,6,8,10,12

example of already bound devices

$ ./dpdk_nic_bind.py --status

Network devices using DPDK-compatible driver

============================================

0000:03:00.0 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv=igb_uio unused=

0000:03:00.1 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv=igb_uio unused=

0000:13:00.0 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv=igb_uio unused=

0000:13:00.1 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv=igb_uio unused=

Network devices using kernel driver

===================================

0000:02:00.0 '82545EM Gigabit Ethernet Controller (Copper)' if=eth2 drv=e1000 unused=igb_uio *Active*

Other network devices

=====================

""" % (err);

return s;

def raise_error (self,err):

s= self.dump_error (err)

raise DpdkSetup(s)

def set_only_mellanox_nics(self):

self.m_is_mellanox_mode=True;

def get_only_mellanox_nics(self):

return self.m_is_mellanox_mode

def read_pci (self,pci_id,reg_id):

out=subprocess.check_output(['setpci', '-s',pci_id, '%s.w' %(reg_id)])

out=out.decode(errors='replace');

return (out.strip());

def write_pci (self,pci_id,reg_id,val):

out=subprocess.check_output(['setpci','-s',pci_id, '%s.w=%s' %(reg_id,val)])

out=out.decode(errors='replace');

return (out.strip());

def tune_mlx_device (self,pci_id):

# set PCIe Read to 4K and not 512 ... need to add it to startup s

val=self.read_pci (pci_id,68)

if val[0]=='0':

#hypervisor does not give the right to write to this register

return;

if val[0]!='5':

val='5'+val[1:]

self.write_pci (pci_id,68,val)

assert(self.read_pci (pci_id,68)==val);

def get_mtu_mlx (self,dev_id):

if len(dev_id)>0:

try:

out=subprocess.check_output(['ifconfig', dev_id])

except Exception as e:

raise DpdkSetup(' "ifconfig %s" utility does not works, try to install it using "$yum install net-tools -y" on CentOS system' %(dev_id) )

out=out.decode(errors='replace');

obj=re.search(r'MTU:(\d+)',out,flags=re.MULTILINE|re.DOTALL);

if obj:

return int(obj.group(1));

else:

obj=re.search(r'mtu (\d+)',out,flags=re.MULTILINE|re.DOTALL);

if obj:

return int(obj.group(1));

else:

return -1

def set_mtu_mlx (self,dev_id,new_mtu):

if len(dev_id)>0:

out=subprocess.check_output(['ifconfig', dev_id,'mtu',str(new_mtu)])

out=out.decode(errors='replace');

def set_max_mtu_mlx_device(self,dev_id):

mtu=9*1024+22

dev_mtu=self.get_mtu_mlx (dev_id);

if (dev_mtu>0) and (dev_mtu!=mtu):

self.set_mtu_mlx(dev_id,mtu);

if self.get_mtu_mlx(dev_id) != mtu:

print("Could not set MTU to %d" % mtu)

sys.exit(-1);

def disable_flow_control_mlx_device (self,dev_id):

if len(dev_id)>0:

my_stderr = open("/dev/null","wb")

cmd ='ethtool -A '+dev_id + ' rx off tx off '

subprocess.call(cmd, stdout=my_stderr,stderr=my_stderr, shell=True)

my_stderr.close();

def check_ofed_version (self):

ofed_info='/usr/bin/ofed_info'

ofed_ver_re = re.compile('.*[-](\d)[.](\d)[-].*')

ofed_ver = 52

ofed_ver_show = '5.2'

if not os.path.isfile(ofed_info):

print("OFED %s is not installed on this setup." % ofed_info)

sys.exit(-1);

try:

out = subprocess.check_output([ofed_info])

except Exception as e:

print("OFED %s can't run " % (ofed_info))

sys.exit(-1);

lines=out.splitlines();

if len(lines)>1:

m= ofed_ver_re.match(str(lines[0]))

if m:

ver=int(m.group(1))*10+int(m.group(2))

if ver < ofed_ver:

print("Installed OFED version is '%s', should be at least '%s' and up." % (lines[0],ofed_ver_show))

sys.exit(-1);

else:

print("No valid OFED version '%s' found." % (lines[0]))

sys.exit(-1);

def verify_ofed_os(self):

err_msg = 'Warning: Mellanox NICs were tested only with RedHat/CentOS 7.9\n'

err_msg += 'Correct usage with other Linux distributions is not guaranteed.'

try:

dist = platform.dist()

if dist[0] not in ('redhat', 'centos') or not dist[1].startswith('7.9'):

print(err_msg)

except Exception as e:

print('Error while determining OS type: %s' % e)

def load_config_file (self):

fcfg=self.m_cfg_file

if not os.path.isfile(fcfg) :

self.raise_error ("There is no valid configuration file %s\n" % fcfg)

try:

stream = open(fcfg, 'r')

self.m_cfg_dict= yaml.safe_load(stream)

except Exception as e:

print(e);

raise e

stream.close();

cfg_dict = self.m_cfg_dict[0]

if 'version' not in cfg_dict:

raise DpdkSetup("Configuration file %s is old, it should include version field.\n" % fcfg )

if int(cfg_dict['version'])<2 :

raise DpdkSetup("Configuration file %s is old, expected version 2, got: %s.\n" % (fcfg, cfg_dict['version']))

if 'interfaces' not in self.m_cfg_dict[0]:

raise DpdkSetup("Configuration file %s is old, it should include interfaces field with even number of elements." % fcfg)

if_list= if_list_remove_sub_if(self.m_cfg_dict[0]['interfaces']);

l=len(if_list);

if l > 32:

raise DpdkSetup("Configuration file %s should include interfaces field with maximum 32 elements, got: %s." % (fcfg,l))

if l % 2:

raise DpdkSetup("Configuration file %s should include even number of interfaces, got: %s" % (fcfg,l))

if 'port_limit' in cfg_dict:

if cfg_dict['port_limit'] > len(if_list):

raise DpdkSetup('Error: port_limit should not be higher than the number of interfaces in config file: %s.\n' % fcfg)

if cfg_dict['port_limit'] % 2:

raise DpdkSetup('Error: port_limit in config file must be an even number, got: %s.\n' % cfg_dict['port_limit'])

if cfg_dict['port_limit'] <= 0:

raise DpdkSetup('Error: port_limit in config file must be a positive number, got: %s.\n' % cfg_dict['port_limit'])

if pa() and pa().limit_ports is not None and pa().limit_ports > len(if_list):

raise DpdkSetup('Error: --limit-ports CLI argument (%s) must not be higher than number of interfaces (%s) in config file: %s\n' % (pa().limit_ports, len(if_list), fcfg))

if cfg_dict.get('stack') == 'linux_based':

self.verify_ethtool()

def verify_ethtool(self):

try:

FNULL = open(os.devnull, 'w')

subprocess.check_call(['ethtool', '--version'], stdout=FNULL, stderr=subprocess.STDOUT)

except Exception as e:

raise DpdkSetup('Error: "linux_based" stack mode requires ethtool installed on your machine. Install it or do not use this mode.')

def do_bind_all(self, drv, pci, force = False):

assert type(pci) is list

cmd = '{ptn} dpdk_nic_bind.py --bind={drv} {pci} {frc}'.format(

ptn = sys.executable,

drv = drv,

pci = ' '.join(pci),

frc = '--force' if force else '')

print(cmd)

return os.system(cmd)

# pros: no need to compile .ko per Kernel version

# cons: need special config/hw (not always works)

def try_bind_to_vfio_pci(self, to_bind_list):

krnl_params_file = '/proc/cmdline'

if not os.path.exists(krnl_params_file):

raise VFIOBindErr('Could not find file with Kernel boot parameters: %s' % krnl_params_file)

with open(krnl_params_file) as f:

krnl_params = f.read()

# IOMMU is always enabled on Power systems

if march != 'ppc64le' and 'iommu=' not in krnl_params:

raise VFIOBindErr('vfio-pci is not an option here')

if 'vfio_pci' not in dpdk_nic_bind.get_loaded_modules():

ret = os.system('modprobe vfio_pci')

if ret:

raise VFIOBindErr('Could not load vfio_pci')

ret = self.do_bind_all('vfio-pci', to_bind_list)

if ret:

raise VFIOBindErr('Binding to vfio_pci failed')

def pci_name_to_full_name (self,pci_name):

if pci_name == 'dummy':

return pci_name

c='[0-9A-Fa-f]';

sp='[:]'

s_short=c+c+sp+c+c+'[.]'+c;

s_full=c+c+c+c+sp+s_short

re_full = re.compile(s_full)

re_short = re.compile(s_short)

if re_short.match(pci_name):

return '0000:'+pci_name

if re_full.match(pci_name):

return pci_name

err=" %s is not a valid pci address \n" %pci_name;

raise DpdkSetup(err)

def run_dpdk_lspci (self):

dpdk_nic_bind.get_nic_details()

self.m_devices= dpdk_nic_bind.devices

def get_prefix(self):

if pa().prefix:

return pa().prefix

return self.m_cfg_dict[0].get('prefix', '')

def preprocess_astf_file_if_needed(self):

""" check if we are in astf batch mode, in case we are convert the profile to json in tmp"""

if not pa() or not pa().astf or pa().interactive:

return

input_file = pa().file

if not input_file:

return

instance_name = ''

prefix = self.get_prefix()

if prefix:

instance_name = '-' + prefix

dst_json_file = "/tmp/astf{instance}.json".format(instance=instance_name)

extension = os.path.splitext(input_file)[1]

if extension == '.json':

shutil.copyfile(input_file, dst_json_file)

os.chmod(dst_json_file, 0o777)

return

elif extension != '.py':

raise DpdkSetup('ERROR when running with --astf mode, you need to have a new Python profile format (.py) and not YAML.')

print('Converting ASTF profile %s to json %s' % (input_file, dst_json_file))

# imports from trex.astf

cur_path = os.path.abspath(os.path.dirname(__file__))

trex_path = os.path.join(cur_path, 'automation', 'trex_control_plane', 'interactive')

if trex_path not in sys.path:

sys.path.insert(1, trex_path)

from trex.astf.trex_astf_profile import ASTFProfile

from trex.astf.sim import decode_tunables

tunables = {}

if pa().tunable:

tunables = decode_tunables(pa().tunable)

try:

profile = ASTFProfile.load(input_file, **tunables)

json_content = profile.to_json_str()

except Exception as e:

raise DpdkSetup('ERROR: Could not convert ASTF profile to JSON:\n%s' % e)

with open(dst_json_file, 'w') as f:

f.write(json_content)

os.chmod(dst_json_file, 0o777)

def verify_stf_file(self):

""" check the input file of STF """

if not pa() or not pa().file or pa().astf:

return

extension = os.path.splitext(pa().file)[1]

if extension == '.py':

raise DpdkSetup('ERROR: Python files can not be used with STF mode, did you forget "--astf" flag?')

elif extension != '.yaml':

pass # should we fail here?

def is_hugepage_file_exits(self,socket_id):

t = ['2048','1048576']

for obj in t:

if map_driver.args.ignore_numa:

filename = '/sys/kernel/mm/hugepages/hugepages-{}kB/nr_hugepages'.format(obj)

else:

filename = '/sys/devices/system/node/node{}/hugepages/hugepages-{}kB/nr_hugepages'.format(socket_id,obj)

if os.path.isfile(filename):

return (True,filename,int(obj))

return (False,None,None)

def config_hugepages(self, wanted_count = None):

mount_output = subprocess.check_output('mount', stderr = subprocess.STDOUT).decode(errors='replace')

nodes = 2

if 'hugetlbfs' not in mount_output:

huge_mnt_dir = '/mnt/huge'

if not os.path.isdir(huge_mnt_dir):

print("Creating huge node")

os.makedirs(huge_mnt_dir)

os.system('mount -t hugetlbfs nodev %s' % huge_mnt_dir)

if map_driver.args.ignore_numa:

nodes = 1

for socket_id in range(nodes):

r = self.is_hugepage_file_exits(socket_id)

if not r[0]:

if socket_id == 0:

print('WARNING: hugepages config file does not exist!')

continue

if wanted_count is None:

if self.m_cfg_dict[0].get('low_end', False):

if socket_id == 0:

if pa() and pa().limit_ports:

if_count = pa().limit_ports

else:

if_count = self.m_cfg_dict[0].get('port_limit', len(self.m_cfg_dict[0]['interfaces']))

wanted_count = 20 + 40 * if_count

else:

wanted_count = 1 # otherwise, DPDK will not be able to see the device

else:

wanted_count = 2048

if r[2] > 2048:

wanted_count = wanted_count / 1024

if wanted_count < 1 :

wanted_count = 1

filename = r[1]

with open(filename) as f:

configured_hugepages = int(f.read())

if configured_hugepages < wanted_count:

os.system('echo %d > %s' % (wanted_count, filename))

time.sleep(0.1)

with open(filename) as f: # verify

configured_hugepages = int(f.read())

if configured_hugepages < wanted_count:

print('WARNING: tried to configure %d hugepages for socket %d, but result is: %d' % (wanted_count, socket_id, configured_hugepages))

def get_core_for_services(self):

default_core = 0

cfg_dict = self.m_cfg_dict[0]

if cfg_dict.get('services_core', False):

# 1st priority from services_core

return cfg_dict['services_core']

elif cfg_dict.get('low_end', False):

# if low_end, use low_end_core or default if not supplied

return cfg_dict.get('low_end_core', default_core)

elif cfg_dict.get('platform', False):

# 2nd priority from master_core

return cfg_dict['platform'].get('master_thread_id', default_core)

else:

# 3rd priority is zero core

return default_core

def run_servers(self):

''' Run both scapy, bird & Emu servers according to pa'''

if not pa():

return

services_core = self.get_core_for_services()

if should_scapy_server_run():

ret = os.system('{sys_exe} general_daemon_server restart -n {name} -c {core} --py -e "{exe}" -r -d -i'.format(sys_exe = sys.executable,

core = services_core,

name = 'Scapy',

exe = '-m trex.scapy_server.scapy_zmq_server'))

if ret:

print("Could not start scapy daemon server, which is needed by GUI to create packets.\nIf you don't need it, use --no-scapy-server flag.")

sys.exit(-1)

if pa().bird_server:

ret = os.system('{sys_exe} general_daemon_server restart -n {name} -c {core} --py -e "{exe}" -i'.format(sys_exe = sys.executable,

name = 'PyBird',

core = services_core,

exe = '-m trex.pybird_server.pybird_zmq_server'))

if ret:

print("Could not start bird server.\nIf you don't need it, don't use --bird-server flag.")

sys.exit(-1)

if pa().emu:

emu_zmq_tcp_flag = '--emu-zmq-tcp' if pa().emu_zmq_tcp else ''

exe = './trex-emu {emu_zmq_tcp}'.format(emu_zmq_tcp = emu_zmq_tcp_flag)

ret = os.system('{sys_exe} general_daemon_server restart -n {name} -c {core} --sudo -e "{exe}"'.format(sys_exe = sys.executable,

core = services_core,

name = 'Emu',

exe = exe))

if ret:

print("Could not start EMU service.\nIf you don't need it, don't use -emu flag.")

sys.exit(-1)

# check vdev Linux interfaces status

# return True if interfaces are vdev

def check_vdev(self, if_list):

if not if_list:

return

af_names = []

ifname_re = re.compile('iface\s*=\s*([^\s,]+)')

found_vdev = False

found_pdev = False

for iface in if_list:

if iface == 'dummy':

continue

elif '--vdev' in iface:

found_vdev = True

if 'net_af_packet' in iface:

res = ifname_re.search(iface)

if res:

af_names.append(res.group(1))

elif ':' not in iface: # no PCI => assume af_packet

found_vdev = True

af_names.append(iface)

else:

found_pdev = True

if found_vdev:

if found_pdev:

raise DpdkSetup('You have mix of vdev and pdev interfaces in config file!')

for name in af_names:

if not os.path.exists('/sys/class/net/%s' % name):

raise DpdkSetup('ERROR: Could not find Linux interface %s.' % name)

oper_state = '/sys/class/net/%s/operstate' % name

if os.path.exists(oper_state):

with open(oper_state) as f:

f_cont = f.read().strip()

return found_vdev

def check_trex_running(self, if_list):

if if_list and map_driver.args.parent and self.m_cfg_dict[0].get('enable_zmq_pub', True):

publisher_port = self.m_cfg_dict[0].get('zmq_pub_port', 4500)

pid = dpdk_nic_bind.get_tcp_port_usage(publisher_port)

if pid:

cmdline = dpdk_nic_bind.read_pid_cmdline(pid)

print('ZMQ port is used by the following process:\npid: %s, cmd: %s' % (pid, cmdline))

sys.exit(-1)

# verify that all interfaces of i40e NIC are in use by current instance of TRex

def check_i40e_binds(self, if_list):

# i40e device IDs taked from dpdk/drivers/net/i40e/base/i40e_devids.h

i40e_device_ids = [0x1572, 0x1574, 0x1580, 0x1581, 0x1583, 0x1584, 0x1585, 0x1586, 0x1587, 0x1588, 0x1589, 0x158A, 0x158B]

iface_without_slash = set()

for iface in if_list:

iface_without_slash.add(self.split_pci_key(iface))

show_warning_devices = set()

unbind_devices = set()

for iface in iface_without_slash:

if iface == 'dummy':

continue

iface = self.split_pci_key(iface)

if self.m_devices[iface]['Device'] not in i40e_device_ids: # not i40e

return

iface_pci = iface.split('.')[0]

for device in self.m_devices.values():

if device['Slot'] in iface_without_slash: # we use it

continue

if iface_pci == device['Slot'].split('.')[0]:

if device.get('Driver_str') == 'i40e':

# Do we ignore the issue explained thereafter (experimental)?

if pa() and not pa().ignore_528_issue:

if pa() and pa().unbind_unused_ports:

# if --unbind-unused-ports is set we unbind ports that are not

# used by TRex

unbind_devices.add(device['Slot'])

else:

print('ERROR: i40e interface %s is under Linux and will interfere with TRex interface %s' % (device['Slot'], iface))

print('See the following link for more information: https://trex-tgn.cisco.com/youtrack/issue/trex-528')

print('Unbind the interface from Linux with following command:')

print(' sudo ./dpdk_nic_bind.py -u %s' % device['Slot'])

print('')

sys.exit(-1)

if device.get('Driver_str') in dpdk_nic_bind.dpdk_drivers:

show_warning_devices.add(device['Slot'])

for dev in show_warning_devices:

print('WARNING: i40e interface %s is under DPDK driver and might interfere with current TRex interfaces.' % dev)

if unbind_devices:

print('Unbinding unused i40e interfaces: %s' % unbind_devices)

dpdk_nic_bind.unbind_all(unbind_devices, force=True)

def do_run (self, only_check_all_mlx=False):

""" returns code that specifies if interfaces are Mellanox/Napatech etc. """

self.load_config_file()

self.preprocess_astf_file_if_needed()

self.verify_stf_file()

if not pa() or pa().dump_interfaces is None or (pa().dump_interfaces == [] and pa().cfg):

if_list = if_list_remove_sub_if(self.m_cfg_dict[0]['interfaces'])

else:

if_list = pa().dump_interfaces

if not if_list:

self.run_dpdk_lspci()

for dev in self.m_devices.values():

if dev.get('Driver_str') in dpdk_nic_bind.dpdk_drivers + dpdk_nic_bind.dpdk_and_kernel:

if_list.append(dev['Slot'])

if self.check_vdev(if_list):

self.check_trex_running(if_list)

self.run_servers()

# no need to config hugepages

return

self.run_dpdk_lspci()

if_list = list(map(self.pci_name_to_full_name, if_list))

Broadcom_cnt=0;

# check how many mellanox cards we have

Mellanox_cnt=0;

mlx5_present=0;

mlx4_present=0;

dummy_cnt=0

for key in if_list:

if key == 'dummy':

dummy_cnt += 1

continue

key = self.split_pci_key(key)

if key not in self.m_devices:

err=" %s does not exist " %key;

raise DpdkSetup(err)

if 'Vendor_str' not in self.m_devices[key]:

err=" %s does not have Vendor_str " %key;

raise DpdkSetup(err)

if 'Mellanox' in self.m_devices[key]['Vendor_str']:

Mellanox_cnt += 1

if 'mlx5' in self.m_devices[key]['Driver_str']:

mlx5_present = MLX5_EXIT_CODE

if 'mlx4' in self.m_devices[key]['Driver_str']:

mlx4_present = MLX4_EXIT_CODE

if 'Broadcom' in self.m_devices[key]['Vendor_str']:

Broadcom_cnt += 1

if not (pa() and pa().dump_interfaces):

if (Mellanox_cnt > 0) and ((Mellanox_cnt + dummy_cnt) != len(if_list)):

err = "All drivers should be from one vendor. You have at least one driver from Mellanox but not all."

raise DpdkSetup(err)

if Mellanox_cnt > 0:

self.set_only_mellanox_nics()

if self.get_only_mellanox_nics():

if not pa().no_ofed_check:

self.verify_ofed_os()

self.check_ofed_version()

for key in if_list:

if key == 'dummy':

continue

if pa().no_ofed_check: # in case of no-ofed don't optimized for Azure

continue

key = self.split_pci_key(key)

if 'Virtual' not in self.m_devices[key]['Device_str']:

pci_id = self.m_devices[key]['Slot_str']

self.tune_mlx_device(pci_id)

if 'Interface' in self.m_devices[key]:

dev_ids = self.m_devices[key]['Interface'].split(",")

for dev_id in dev_ids:

self.disable_flow_control_mlx_device (dev_id)

self.set_max_mtu_mlx_device(dev_id)

if only_check_all_mlx:

if Mellanox_cnt > 0:

sys.exit(mlx5_present + mlx4_present);

else:

sys.exit(0);

self.check_i40e_binds(if_list)