From fd52bd851e7c1c0f9aab359937f5fe72c49412bf Mon Sep 17 00:00:00 2001 From: Miruna Paun Date: Wed, 12 Jul 2017 13:12:22 +0200 Subject: Updating benchmarks with two new use cases LXCR-7881 SR-IOV for Docker and VMs Signed-off-by: Miruna Paun --- .../doc/benchmarks.xml | 381 ++++++++++++++++++++- doc/manifest_conf.mk | 2 +- 2 files changed, 381 insertions(+), 2 deletions(-) diff --git a/doc/book-enea-nfv-access-platform-guide/doc/benchmarks.xml b/doc/book-enea-nfv-access-platform-guide/doc/benchmarks.xml index fa461fa..daafadf 100644 --- a/doc/book-enea-nfv-access-platform-guide/doc/benchmarks.xml +++ b/doc/book-enea-nfv-access-platform-guide/doc/benchmarks.xml @@ -245,7 +245,7 @@ Pktgen is used to generate UDP traffic that will reach testpmd, running in a Docker image. It will then be forwarded back to source on - the return trip (Forwarding). + the return trip (Forwarding). This test measures: @@ -702,6 +702,129 @@ ovs-ofctl add-flow ovsbr0 in_port=2,action=output:1Import a + +
+ SR-IOV in in Docker + + PCI passthrough tests using pktgen and testpmd in Docker. + + pktgen[DPDK]Docker - PHY - Docker[DPDK] testpmd + + Measurements: + + + + RX packets per second in testpmd (with testpmd configured in + rxonly mode). + + + +
+ Test Setup + + Boot Enea NFV Access Platform from SSD:linux (hd0,gpt3)/boot/bzImage root=/dev/sda3 ip=dhcp nohz_full=1-7 / +isolcpus=1-7 rcu-nocbs=1-7 rcu_nocb_poll intel_pstate=disable clocksource=tsc / +tsc=reliable nohpet nosoftlockup intel_idle.max_cstate=0 processor.max_cstate=0 / +mce=ignore_ce audit=0 nmi_watchdog=0 iommu=pt intel_iommu=on hugepagesz=1GB / +hugepages=8 default_hugepagesz=1GB hugepagesz=2M hugepages=2048 / +vfio_iommu_type1.allow_unsafe_interrupts=1lAllow unsafe + interrupts:echo 1 > /sys/module/vfio_iommu_type1/parameters/allow_unsafe_interruptsConfigure + DPDK:mkdir -p /mnt/huge +mount -t hugetlbfs nodev /mnt/huge +dpdk-devbind.py --bind=ixgbe 0000:03:00.0 +ifconfig eno3 192.168.1.2 +echo 2 > /sys/class/net/eno3/device/sriov_numvfs +modprobe vfio-pci +dpdk-devbind.py --bind=vfio-pci 0000:03:10.0 +dpdk-devbind.py --bind=vfio-pci 0000:03:10.2Start two docker + containers:docker run --privileged -it --rm -v /mnt/huge:/mnt/huge/ / +--device /dev/vfio/vfio el7_guest /bin/bash +docker run --privileged -it --rm -v /mnt/huge:/mnt/huge/ / +--device /dev/vfio/vfio el7_guest /bin/bashIn the first + container start pktgen:cd /usr/share/apps/pktgen/ +./pktgen -c 0x1f -w 0000:03:10.0 -n 1 --file-prefix pg1 / +--socket-mem 1024 -- -P -m "[3:4].0"In the pktgen prompt set + the destination MAC address:set mac 0 XX:XX:XX:XX:XX:XX +strIn the second container start testpmd:testpmd -c 0x7 -n 1 -w 0000:03:10.2 -- -i --portmask=0x1 / +--txd=256 --rxd=256 --port-topology=chainedIn the testpmd + prompt set forwarding + rxonly:set fwd rxonly +start + Results + + + + + Bytes + + pktgen pps + TX + + testpmd pps + RX + + pktgen MBits/s + TX + + throughput + (%) + + + + 64 + + 14525286 + + 14190869 + + 9739 + + 97.7 + + + + 128 + + 8456960 + + 8412172 + + 10013 + + 99.4 + + + + 256 + + 4566624 + + 4526587 + + 10083 + + 99.1 + + + + 512 + + 2363744 + + 2348015 + + 10060 + + 99.3 + + + +
+
+
@@ -1253,6 +1376,262 @@ show port stats 0For VM1, we record the stats relevant for
+ +
+ SR-IOV in Virtual Machines + + PCI passthrough tests using pktgen and testpmd in virtual + machines. + + pktgen[DPDK]VM - PHY - VM[DPDK] testpmd. + + Measurements: + + + + pktgen to testpmd in forwarding mode. + + + + pktgen to testpmd in termination mode. + + + +
+ Test Setup + + SSD boot using the following grub.cfg + entry: linux (hd0,gpt3)/boot/bzImage root=/dev/sda3 ip=dhcp nohz_full=1-7 / +isolcpus=1-7 rcu-nocbs=1-7 rcu_nocb_poll intel_pstate=disable / +clocksource=tsc tsc=reliable nohpet nosoftlockup intel_idle.max_cstate=0 / +processor.max_cstate=0 mce=ignore_ce audit=0 nmi_watchdog=0 iommu=pt / +intel_iommu=on hugepagesz=1GB hugepages=8 default_hugepagesz=1GB / +hugepagesz=2M hugepages=2048 vfio_iommu_type1.allow_unsafe_interrupts=1Stop + other services and mount hugepages: systemctl stop openvswitch +mkdir -p /mnt/huge +mount -t hugetlbfs hugetlbfs /mnt/hugeConfigure SR-IOV + interfaces:/usr/share/usertools/dpdk-devbind.py --bind=ixgbe 0000:03:00.0 +echo 2 > /sys/class/net/eno3/device/sriov_numvfs +ifconfig eno3 10.0.0.1 +modprobe vfio_pci +/usr/share/usertools/dpdk-devbind.py --bind=vfio-pci 0000:03:10.0 +/usr/share/usertools/dpdk-devbind.py --bind=vfio-pci 0000:03:10.2 +ip link set eno3 vf 0 mac 0c:c4:7a:E5:0F:48 +ip link set eno3 vf 1 mac 0c:c4:7a:BF:52:E7Launch two QEMU + instances: taskset -c 4,5 qemu-system-x86_64 -cpu host,+invtsc,migratable=no -M / +q35 -smp cores=2,sockets=1 -vcpu 0,affinity=4 -vcpu 1,affinity=5 -enable-kvm / +-nographic -kernel /mnt/qemu/bzImage / +-drive file=/mnt/qemu/enea-image-virtualization-guest-qemux86-64.ext4,if=virtio,/ +format=raw -m 4096 -object memory-backend-file,id=mem,size=4096M,mem-path=/mnt/huge,/ +share=on -numa node,memdev=mem -mem-prealloc -device vfio-pci,host=03:10.0 / +-append 'root=/dev/vda console=ttyS0 hugepagesz=2M hugepages=1024 / +isolcpus=1 nohz_full=1 rcu_nocbs=1 irqaffinity=0 rcu_nocb_poll / +intel_pstate=disable intel_idle.max_cstate=0 / +processor.max_cstate=0 mce=ignore_ce audit=0' + + +taskset -c 2,3 qemu-system-x86_64 -cpu host,+invtsc,migratable=no -M / +q35 -smp cores=2,sockets=1 -vcpu 0,affinity=2 -vcpu 1,affinity=3 -enable-kvm / +-nographic -kernel /mnt/qemu/bzImage / +-drive file=/mnt/qemu/enea-image2-virtualization-guest-qemux86-64.ext4,if=virtio,/ +format=raw -m 4096 -object memory-backend-file,id=mem,size=4096M,mem-path=/mnt/huge,/ +share=on -numa node,memdev=mem -mem-prealloc -device vfio-pci,host=03:10.2 / +-append 'root=/dev/vda console=ttyS0 hugepagesz=2M hugepages=1024 / +isolcpus=1 nohz_full=1 rcu_nocbs=1 irqaffinity=0 rcu_nocb_poll / +intel_pstate=disable intel_idle.max_cstate=0 processor.max_cstate=0 / +mce=ignore_ce audit=0'In the first VM, mount hugepages and + start pktgen:mkdir -p /mnt/huge && \ +mount -t hugetlbfs hugetlbfs /mnt/huge +modprobe igb_uio +/usr/share/usertools/dpdk-devbind.py --bind=igb_uio 0000:00:03.0 +cd /usr/share/apps/pktgen +./pktgen -c 0x3 -- -P -m "1.0"In the pktgen console set the + MAC of the destination and start generating + packages:set mac 0 0C:C4:7A:BF:52:E7 +strIn the second VM, mount hugepages and start + testpmd:mkdir -p /mnt/huge && \ +mount -t hugetlbfs hugetlbfs /mnt/huge +modprobe igb_uio +/usr/share/usertools/dpdk-devbind.py --bind=igb_uio 0000:00:03.0 +testpmd -c 0x3 -n 2 -- -i --txd=512 --rxd=512 --port-topology=chained / +--eth-peer=0,0c:c4:7a:e5:0f:48In order to enable forwarding mode, in the testpmd console, + run:set fwd mac +startIn order to enable termination mode, in the testpmd console, + run:set fwd rxonly +start + Results in forwarding mode + + + + + Bytes + + VM1 pktgen pps + TX + + VM1 pktgen pps + RX + + VM2 testpmd + pps RX + + VM2 testpmd + pps RX + + + + 64 + + 7105645 + + 7103976 + + 7101487 + + 7101487 + + + + 128 + + 5722795 + + 5722252 + + 5704219 + + 5704219 + + + + 256 + + 3454075 + + 3455144 + + 3452020 + + 3452020 + + + + 512 + + 1847751 + + 1847751 + + 1847751 + + 1847751 + + + + 1024 + + 956214 + + 956214 + + 956214 + + 956214 + + + + 1500 + + 797174 + + 797174 + + 797174 + + 797174 + + + +
+ Results in termination mode + + + + + Bytes + + VM1 pktgen pps + TX + + VM2 testpmd + RX + + + + 64 + + 14204580 + + 14205063 + + + + 128 + + 8424611 + + 8424611 + + + + 256 + + 4529024 + + 4529024 + + + + 512 + + 2348640 + + 2348640 + + + + 1024 + + 1197101 + + 1197101 + + + + 1500 + + 822244 + + 822244 + + + +
+
+
\ No newline at end of file diff --git a/doc/manifest_conf.mk b/doc/manifest_conf.mk index 92e395c..9867951 100644 --- a/doc/manifest_conf.mk +++ b/doc/manifest_conf.mk @@ -5,4 +5,4 @@ #MANIFESTHASH ?= 0d0f06384afa65eaae4b170e234ee5a813edf44d #change the above value later to refs/tags/ELnnn (?) MANIFESTURL := git@git.enea.com:linux/manifests/el_manifests-virtualization.git -PROFILE_NAME := Enea NFV Access Platform +PROFILE_NAME := Enea NFV Access Platform.$(shell date +"%Y.%m.%d %H:%M:%S") -- cgit v1.2.3-54-g00ecf