Updating the benchmarks chapter with all the needed info

LXCR-7844

Signed-off-by: Miruna Paun <Miruna.Paun@enea.com>

1 files changed, 1034 insertions, 3 deletions

diff --git a/doc/book-enea-nfv-access-platform-guide/doc/benchmarks.xml b/doc/book-enea-nfv-access-platform-guide/doc/benchmarks.xml
index 5d6e268..7155e44 100644
--- a/doc/book-enea-nfv-access-platform-guide/doc/benchmarks.xml
+++ b/doc/book-enea-nfv-access-platform-guide/doc/benchmarks.xml
@@ -1,14 +1,1045 @@
 <?xml version="1.0" encoding="ISO-8859-1"?>
 <!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
 "http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd">
-<chapter condition="hidden" id="benchmarks">
+<chapter id="benchmarks">
   <title>Benchmarks</title>
 
   <para></para>
 
-  <section id="relinfo-getting-pre-built-images">
-    <title></title>
+  <section id="hw-setup">
+    <title>Hardware Setup</title>
 
     <para></para>
   </section>
+
+  <section id="bios">
+    <title>Bios</title>
+
+    <para></para>
+  </section>
+
+  <section id="use-cases">
+    <title>Use Cases</title>
+
+    <section id="docker-benchmarks">
+      <title>Docker related benchmarks</title>
+
+      <section>
+        <title>Use Case - Forward traffic in docker</title>
+
+        <para>Benchmarking traffic forwarding using testpmd in a Docker
+        container.</para>
+
+        <para>Pktgen is used to generate UDP traffic that will reach testpmd,
+        running in a docker image. It will then be forwarded back to source on
+        the return trip (Forwarding). </para>
+
+        <para>This test measures:</para>
+
+        <itemizedlist>
+          <listitem>
+            <para>pktgen TX, RX in packets per second (pps) and Mbps</para>
+          </listitem>
+
+          <listitem>
+            <para>testpmd TX, RX in packets per second (pps)</para>
+          </listitem>
+
+          <listitem>
+            <para>divide testpmd RX / pktgen TX in pps to obtain throughput in
+            percentages (%)</para>
+          </listitem>
+        </itemizedlist>
+
+        <section id="usecase-one">
+          <title>Test Setup for Target 1</title>
+
+          <para>Start by following the steps below:</para>
+
+          <para>SSD boot using the following <literal>grub.cfg</literal>
+          entry: <programlisting>linux (hd0,gpt3)/boot/bzImage root=/dev/sda3 ip=dhcp nohz_full=1-7 /
+isolcpus=1-7 rcu-nocbs=1-7 rcu_nocb_poll intel_pstate=disable /
+clocksource=tsc tsc=reliable nohpet nosoftlockup intel_idle.max_cstate=0 /
+processor.max_cstate=0 mce=ignore_ce audit=0 nmi_watchdog=0 iommu=pt /
+intel_iommu=on  hugepagesz=1GB hugepages=8 default_hugepagesz=1GB /
+hugepagesz=2M hugepages=2048 vfio_iommu_type1.allow_unsafe_interrupts=1</programlisting></para>
+
+          <para>Kill unnecessary services:<programlisting>killall ovsdb-server ovs-vswitchd
+rm -rf /etc/openvswitch/*
+mkdir -p /var/run/openvswitch</programlisting>Mount hugepages and configure
+          DPDK:<programlisting>mkdir -p /mnt/huge
+mount -t hugetlbfs nodev /mnt/huge
+modprobe igb_uio
+dpdk-devbind --bind=igb_uio 0000:03:00.0</programlisting>Run
+          pktgen:<programlisting>cd /usr/share/apps/pktgen/
+./pktgen -c 0xF -n 1 -- -P -m "[3:2].0"</programlisting>In the pktgen console
+          run:<programlisting>str</programlisting>To change framesize for
+          pktgen, from [64, 128, 256, 512]:<programlisting>set 0 size &amp;lt;number&amp;gt;</programlisting></para>
+        </section>
+
+        <section id="usecase-two">
+          <title>Test Setup for Target 2</title>
+
+          <para>Start by following the steps below:</para>
+
+          <para>SSD boot using the following <literal>grub.cfg</literal>
+          entry:</para>
+
+          <programlisting>linux (hd0,gpt3)/boot/bzImage root=/dev/sda3 ip=dhcp nohz_full=1-7 /
+isolcpus=1-7 rcu-nocbs=1-7 rcu_nocb_poll intel_pstate=disable /
+clocksource=tsc tsc=reliable nohpet nosoftlockup intel_idle.max_cstate=0 /
+processor.max_cstate=0 mce=ignore_ce audit=0 nmi_watchdog=0 iommu=pt /
+intel_iommu=on  hugepagesz=1GB hugepages=8 default_hugepagesz=1GB /
+hugepagesz=2M hugepages=2048 vfio_iommu_type1.allow_unsafe_interrupts=1</programlisting>
+
+          <para>It is expected to have docker/guest image on target. Configure
+          the OVS bridge:<programlisting># OVS old config clean-up
+killall ovsdb-server ovs-vswitchd
+rm -rf /etc/openvswitch/*
+mkdir -p /var/run/openvswitch
+
+# Mount hugepages and bind interfaces to dpdk
+mkdir -p /mnt/huge
+mount -t hugetlbfs nodev /mnt/huge
+modprobe igb_uio
+dpdk-devbind --bind=igb_uio 0000:03:00.0
+
+# configure openvswitch with DPDK
+export DB_SOCK=/var/run/openvswitch/db.sock
+ovsdb-tool create /etc/openvswitch/conf.db /
+/usr/share/openvswitch/vswitch.ovsschema
+ovsdb-server --remote=punix:$DB_SOCK /
+--remote=db:Open_vSwitch,Open_vSwitch,manager_options --pidfile --detach
+ovs-vsctl --no-wait init
+ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-lcore-mask=0x10
+ovs-vsctl --no-wait set Open_vSwitch . other_config:pmd-cpu-mask=0xc
+ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-socket-mem=2048
+ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true
+ovs-vswitchd unix:$DB_SOCK --pidfile --detach /
+--log-file=/var/log/openvswitch/ovs-vswitchd.log
+
+ovs-vsctl add-br ovsbr0 -- set bridge ovsbr0 datapath_type=netdev
+ovs-vsctl add-port ovsbr0 vhost-user1 /
+-- set Interface vhost-user1 type=dpdkvhostuser ofport_request=1
+ovs-vsctl add-port ovsbr0 dpdk0 -- set Interface /
+dpdk0 type=dpdk options:dpdk-devargs=0000:03:00.0 ofport_request=2
+
+# configure static flows
+ovs-ofctl del-flows ovsbr0
+ovs-ofctl add-flow ovsbr0 in_port=1,action=output:2
+ovs-ofctl add-flow ovsbr0 in_port=2,action=output:1</programlisting>Import a
+          Docker container:<programlisting>docker import enea-image-virtualization-guest-qemux86-64.tar.gz el7_guest</programlisting>Start
+          the Docker container:<programlisting>docker run -it --rm -v /var/run/openvswitch/:/var/run/openvswitch/ /
+-v /mnt/huge:/mnt/huge el7_guest /bin/bash</programlisting>Start the testpmd
+          application in Docker:<programlisting>testpmd -c 0x30 -n 2 --file-prefix prog1  --socket-mem 512 --no-pci /
+--vdev=virtio_user0,path=/var/run/openvswitch/vhost-user1 /
+-d /usr/lib/librte_pmd_virtio.so.1.1  -- --burst 64 --disable-hw-vlan /
+--disable-rss  -i --portmask=0x1 --coremask=0x20  --nb-cores=1 /
+--rxq=1 --txq=1 --txd=512 --rxd=512 --txqflags=0xf00 --port-topology=chained</programlisting>To
+          start traffic <emphasis role="bold">forwarding</emphasis>, run the
+          following command in testpmd CLI:<programlisting>start</programlisting>To
+          start traffic but in <emphasis role="bold">termination</emphasis>
+          mode (no traffic sent on TX), run following command in testpmd
+          CLI:<programlisting>set fwd rxonly
+start</programlisting><table>
+              <title>Results in forwarding mode</title>
+
+              <tgroup cols="8">
+                <tbody>
+                  <row>
+                    <entry align="center"><emphasis
+                    role="bold">Bytes</emphasis></entry>
+
+                    <entry align="center"><emphasis role="bold">pktgen pps
+                    TX</emphasis></entry>
+
+                    <entry align="center"><emphasis role="bold">pktgen MBits/s
+                    TX</emphasis></entry>
+
+                    <entry align="center"><emphasis role="bold">pktgen pps
+                    RX</emphasis></entry>
+
+                    <entry align="center"><emphasis role="bold">pktgen MBits/s
+                    RX</emphasis></entry>
+
+                    <entry align="center"><emphasis role="bold">testpmd pps
+                    RX</emphasis></entry>
+
+                    <entry align="center"><emphasis role="bold">testpmd pps
+                    TX</emphasis></entry>
+
+                    <entry align="center"><emphasis role="bold">throughput
+                    (%)</emphasis></entry>
+                  </row>
+
+                  <row>
+                    <entry role="bold"><emphasis
+                    role="bold">64</emphasis></entry>
+
+                    <entry>14890993</entry>
+
+                    <entry>10006</entry>
+
+                    <entry>7706039</entry>
+
+                    <entry>5178</entry>
+
+                    <entry>7692807</entry>
+
+                    <entry>7692864</entry>
+
+                    <entry>51.74%</entry>
+                  </row>
+
+                  <row>
+                    <entry><emphasis role="bold">128</emphasis></entry>
+
+                    <entry>8435104</entry>
+
+                    <entry>9999</entry>
+
+                    <entry>7689458</entry>
+
+                    <entry>9060</entry>
+
+                    <entry>7684962</entry>
+
+                    <entry>7684904</entry>
+
+                    <entry>90.6%</entry>
+                  </row>
+
+                  <row>
+                    <entry role="bold"><emphasis
+                    role="bold">256</emphasis></entry>
+
+                    <entry>4532384</entry>
+
+                    <entry>9999</entry>
+
+                    <entry>4532386</entry>
+
+                    <entry>9998</entry>
+
+                    <entry>4532403</entry>
+
+                    <entry>4532403</entry>
+
+                    <entry>99.9%</entry>
+                  </row>
+                </tbody>
+              </tgroup>
+            </table><table>
+              <title>Results in termination mode</title>
+
+              <tgroup cols="4">
+                <tbody>
+                  <row>
+                    <entry align="center"><emphasis
+                    role="bold">Bytes</emphasis></entry>
+
+                    <entry align="center"><emphasis role="bold">pktgen pps
+                    TX</emphasis></entry>
+
+                    <entry align="center"><emphasis role="bold">testpmd pps
+                    RX</emphasis></entry>
+
+                    <entry align="center"><emphasis role="bold">throughput
+                    (%)</emphasis></entry>
+                  </row>
+
+                  <row>
+                    <entry role="bold"><emphasis
+                    role="bold">64</emphasis></entry>
+
+                    <entry>14890993</entry>
+
+                    <entry>7330403</entry>
+
+                    <entry>49,2%</entry>
+                  </row>
+
+                  <row>
+                    <entry><emphasis role="bold">128</emphasis></entry>
+
+                    <entry>8435104</entry>
+
+                    <entry>7330379</entry>
+
+                    <entry>86,9%</entry>
+                  </row>
+
+                  <row>
+                    <entry role="bold"><emphasis
+                    role="bold">256</emphasis></entry>
+
+                    <entry>4532484</entry>
+
+                    <entry>4532407</entry>
+
+                    <entry>99,9%</entry>
+                  </row>
+                </tbody>
+              </tgroup>
+            </table></para>
+        </section>
+      </section>
+
+      <section id="usecase-three-four">
+        <title>Use Case - Forward traffic from Docker to another Docker on the
+        same host</title>
+
+        <para>Benchmark a combo test using testpmd running in two Docker
+        instances, one which Forwards traffic to the second one, which
+        Terminates it.</para>
+
+        <para>Packets are generated with pktgen and TX-d to the first testpmd,
+        which will RX and Forward them to the second testpmd, which will RX
+        and terminate them.</para>
+
+        <para>Measurements are made in:</para>
+
+        <itemizedlist>
+          <listitem>
+            <para>pktgen TX in pps and Mbits/s</para>
+          </listitem>
+
+          <listitem>
+            <para>testpmd TX and RX pps in Docker1</para>
+          </listitem>
+
+          <listitem>
+            <para>testpmd RX pps in Docker2</para>
+          </listitem>
+        </itemizedlist>
+
+        <para>Throughput found as a percent, by dividing Docker2 <emphasis
+        role="bold">testpmd RX pps</emphasis> by <emphasis role="bold">pktgen
+        TX pps</emphasis>.</para>
+
+        <section id="target-one-usecase-three">
+          <title>Test Setup for Target 1</title>
+
+          <para>Start by following the steps below:</para>
+
+          <para>SSD boot using the following <literal>grub.cfg</literal>
+          entry:</para>
+
+          <programlisting>linux (hd0,gpt3)/boot/bzImage root=/dev/sda3 ip=dhcp nohz_full=1-7 /
+isolcpus=1-7 rcu-nocbs=1-7 rcu_nocb_poll intel_pstate=disable /
+clocksource=tsc tsc=reliable nohpet nosoftlockup intel_idle.max_cstate=0 /
+processor.max_cstate=0 mce=ignore_ce audit=0 nmi_watchdog=0 iommu=pt /
+intel_iommu=on  hugepagesz=1GB hugepages=8 default_hugepagesz=1GB /
+hugepagesz=2M hugepages=2048 vfio_iommu_type1.allow_unsafe_interrupts=1</programlisting>
+
+          <para>Configure DPDK:<programlisting>mkdir -p /mnt/huge
+mount -t hugetlbfs nodev /mnt/huge
+modprobe igb_uio
+dpdk-devbind --bind=igb_uio 0000:03:00.0</programlisting>Run
+          pktgen:<programlisting>cd /usr/share/apps/pktgen/
+./pktgen -c 0xF -n 1 -- -P -m "[3:2].0"</programlisting>Choose one of the
+          values from [64, 128, 256, 512] to change the packet
+          size:<programlisting>set 0 size &lt;number&gt;</programlisting></para>
+        </section>
+
+        <section id="target-two-usecase-four">
+          <title>Test Setup for Target 2</title>
+
+          <para>Start by following the steps below:</para>
+
+          <para>SSD boot using the following <literal>grub.cfg</literal>
+          entry:</para>
+
+          <programlisting>linux (hd0,gpt3)/boot/bzImage root=/dev/sda3 ip=dhcp nohz_full=1-7 /
+isolcpus=1-7 rcu-nocbs=1-7 rcu_nocb_poll intel_pstate=disable /
+clocksource=tsc tsc=reliable nohpet nosoftlockup intel_idle.max_cstate=0 /
+processor.max_cstate=0 mce=ignore_ce audit=0 nmi_watchdog=0 /
+iommu=pt intel_iommu=on  hugepagesz=1GB hugepages=8 default_hugepagesz=1GB /
+hugepagesz=2M hugepages=2048 vfio_iommu_type1.allow_unsafe_interrupts=1</programlisting>
+
+          <para><programlisting>killall ovsdb-server ovs-vswitchd
+rm -rf /etc/openvswitch/*
+mkdir -p /var/run/openvswitch</programlisting>Configure DPDK:<programlisting>mkdir -p /mnt/huge
+mount -t hugetlbfs nodev /mnt/huge
+modprobe igb_uio
+dpdk-devbind --bind=igb_uio 0000:03:00.0</programlisting>Configure the OVS
+          bridge:<programlisting>export DB_SOCK=/var/run/openvswitch/db.sock
+ovsdb-tool create /etc/openvswitch/conf.db /
+/usr/share/openvswitch/vswitch.ovsschema
+ovsdb-server --remote=punix:$DB_SOCK /
+--remote=db:Open_vSwitch,Open_vSwitch,manager_options --pidfile --detach
+ovs-vsctl --no-wait init
+ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-lcore-mask=0x10
+ovs-vsctl --no-wait set Open_vSwitch . other_config:pmd-cpu-mask=0xcc
+ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-socket-mem=2048
+ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true
+ovs-vswitchd unix:$DB_SOCK --pidfile --detach /
+--log-file=/var/log/openvswitch/ovs-vswitchd.log
+ovs-vsctl add-br ovsbr0 -- set bridge ovsbr0 datapath_type=netdev
+ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface /
+vhost-user1 type=dpdkvhostuser ofport_request=1
+ovs-vsctl add-port ovsbr0 vhost-user2 -- set Interface /
+vhost-user2 type=dpdkvhostuser ofport_request=2
+ovs-vsctl add-port ovsbr0 dpdk0 -- set Interface dpdk0 /
+type=dpdk options:dpdk-devargs=0000:03:00.0 ofport_request=3
+ovs-ofctl del-flows ovsbr0
+ovs-ofctl add-flow ovsbr0 in_port=3,action=output:2
+ovs-ofctl add-flow ovsbr0 in_port=2,action=output:1</programlisting>Import a
+          Docker container:<programlisting>docker import enea-image-virtualization-guest-qemux86-64.tar.gz el7_guest</programlisting>Start
+          the first Docker:<programlisting>docker run -it --rm --cpuset-cpus=4,5 /
+-v /var/run/openvswitch/:/var/run/openvswitch/ /
+-v /mnt/huge:/mnt/huge el7_guest /bin/bash</programlisting>Start the testpmd
+          application in Docker1:<programlisting>testpmd -c 0x30 -n 2 --file-prefix prog1  --socket-mem 512 --no-pci /
+--vdev=virtio_user0,path=/var/run/openvswitch/vhost-user1 /
+-d /usr/lib/librte_pmd_virtio.so.1.1 -- --burst 64 --disable-hw-vlan /
+--disable-rss  -i --portmask=0x1 --coremask=0x20  --nb-cores=1 /
+--rxq=1 --txq=1 --txd=512 --rxd=512 --txqflags=0xf00 --port-topology=chained</programlisting>Configure
+          it in termination mode:<programlisting>set fwd rxonly</programlisting>Run
+          the testpmd application:<programlisting>start</programlisting>Open a
+          new console to the host and start the second docker
+          instance:<programlisting>docker run -it --rm --cpuset-cpus=0,1 -v /var/run/openvswitch/:/var/run/openvswitch/ /
+-v /mnt/huge:/mnt/huge el7_guest /bin/bash</programlisting>In the second
+          container start testpmd:<programlisting>testpmd -c 0x0F --file-prefix prog2 --socket-mem 512 --no-pci /
+--vdev=virtio_user0,path=/var/run/openvswitch/vhost-user2 /
+-d /usr/lib/librte_pmd_virtio.so.1.1 -- -i --disable-hw-vlan</programlisting>Run
+          the TestPmd application in the second docker:<programlisting>testpmd -c 0x3 -n 2 --file-prefix prog2  --socket-mem 512 --no-pci /
+--vdev=virtio_user0,path=/var/run/openvswitch/vhost-user2 /
+-d /usr/lib/librte_pmd_virtio.so.1.1 -- --burst 64 --disable-hw-vlan /
+--disable-rss  -i --portmask=0x1 --coremask=0x2  --nb-cores=1 --rxq=1 /
+--txq=1 --txd=512 --rxd=512 --txqflags=0xf00 --port-topology=chained</programlisting>In
+          the testpmd shell, run:<programlisting>start</programlisting>Start
+          pktgen traffic by running the following command in pktgen
+          CLI:<programlisting>start 0</programlisting>To record traffic
+          results:<programlisting>show port stats 0</programlisting>This
+          should be used in testpmd applications.</para>
+
+          <table>
+            <title>Results </title>
+
+            <tgroup cols="5">
+              <tbody>
+                <row>
+                  <entry align="center"><emphasis
+                  role="bold">Bytes</emphasis></entry>
+
+                  <entry align="center"><emphasis role="bold">Target 1 -
+                  pktgen pps TX</emphasis></entry>
+
+                  <entry align="center"><emphasis role="bold">Target 2 -
+                  (forwarding) testpmd pps RX</emphasis></entry>
+
+                  <entry align="center"><emphasis role="bold">Target 2 -
+                  (forwarding) testpmd pps TX</emphasis></entry>
+
+                  <entry align="center"><emphasis role="bold">Target 2 -
+                  (termination) testpmd pps RX</emphasis></entry>
+                </row>
+
+                <row>
+                  <entry role="bold"><emphasis
+                  role="bold">64</emphasis></entry>
+
+                  <entry>14844628</entry>
+
+                  <entry>5643565</entry>
+
+                  <entry>3459922</entry>
+
+                  <entry>3457326</entry>
+                </row>
+
+                <row>
+                  <entry><emphasis role="bold">128</emphasis></entry>
+
+                  <entry>8496962</entry>
+
+                  <entry>5667860</entry>
+
+                  <entry>3436811</entry>
+
+                  <entry>3438918</entry>
+                </row>
+
+                <row>
+                  <entry role="bold"><emphasis
+                  role="bold">256</emphasis></entry>
+
+                  <entry>4532372</entry>
+
+                  <entry>4532362</entry>
+
+                  <entry>3456623</entry>
+
+                  <entry>3457115</entry>
+                </row>
+
+                <row>
+                  <entry><emphasis role="bold">512</emphasis></entry>
+
+                  <entry>2367641</entry>
+
+                  <entry>2349450</entry>
+
+                  <entry>2349450</entry>
+
+                  <entry>2349446</entry>
+                </row>
+              </tbody>
+            </tgroup>
+          </table>
+        </section>
+      </section>
+    </section>
+
+    <section id="vm-benchmarks">
+      <title>VM related benchmarks</title>
+
+      <section id="usecase-four">
+        <title>Use Case - Forward/termination traffic in one VM</title>
+
+        <para>Benchmarking traffic (UDP) forwarding and termination using
+        testpmd in a virtual machine. </para>
+
+        <para>The Pktgen application is used to generate traffic that will
+        reach testpmd running on a virtual machine, and be forwarded back to
+        source on the return trip. With the same setup a second measurement
+        will be done with traffic termination in the virtual machine. </para>
+
+        <para>This test case measures:</para>
+
+        <itemizedlist>
+          <listitem>
+            <para>pktgen TX, RX in packets per second (pps) and Mbps</para>
+          </listitem>
+
+          <listitem>
+            <para>testpmd TX, RX in packets per second (pps)</para>
+          </listitem>
+
+          <listitem>
+            <para>divide <emphasis role="bold">testpmd RX</emphasis> by
+            <emphasis role="bold">pktgen TX</emphasis> in pps to obtain the
+            throughput in percentages (%)</para>
+          </listitem>
+        </itemizedlist>
+
+        <section id="targetone-usecasefour">
+          <title>Test Setup for Target 1</title>
+
+          <para>Start with the steps below:</para>
+
+          <para>SSD boot using the following <literal>grub.cfg
+          </literal>entry: <programlisting>linux (hd0,gpt3)/boot/bzImage root=/dev/sda3 ip=dhcp nohz_full=1-7 /
+isolcpus=1-7 rcu-nocbs=1-7 rcu_nocb_poll intel_pstate=disable /
+clocksource=tsc tsc=reliable nohpet nosoftlockup intel_idle.max_cstate=0 /
+processor.max_cstate=0 mce=ignore_ce audit=0 nmi_watchdog=0 iommu=pt /
+intel_iommu=on hugepagesz=1GB hugepages=8 default_hugepagesz=1GB /
+hugepagesz=2M hugepages=2048 vfio_iommu_type1.allow_unsafe_interrupts=1</programlisting></para>
+
+          <para>Kill unnecessary services: <programlisting>killall ovsdb-server ovs-vswitchd
+rm -rf /etc/openvswitch/*
+mkdir -p /var/run/openvswitch</programlisting>Configure DPDK:<programlisting>mkdir -p /mnt/huge
+mount -t hugetlbfs nodev /mnt/huge
+modprobe igb_uio
+dpdk-devbind --bind=igb_uio 0000:03:00.0</programlisting>Run
+          pktgen:<programlisting>cd /usr/share/apps/pktgen/
+./pktgen -c 0x7 -n 4 --proc-type auto --socket-mem 256 /
+-w 0000:03:00.0 -- -P -m "[1:2].0"</programlisting>Set pktgen frame size to
+          use from [64, 128, 256, 512]:<programlisting>set 0 size 64</programlisting></para>
+        </section>
+
+        <section id="targettwo-usecasefive">
+          <title>Test Setup for Target 2</title>
+
+          <para>Start by following the steps below:</para>
+
+          <para>SSD boot using the following <literal>grub.cfg</literal>
+          entry: <programlisting>linux (hd0,gpt3)/boot/bzImage root=/dev/sda3 ip=dhcp nohz_full=1-7 /
+isolcpus=1-7 rcu-nocbs=1-7 rcu_nocb_poll intel_pstate=disable /
+clocksource=tsc tsc=reliable nohpet nosoftlockup intel_idle.max_cstate=0 /
+processor.max_cstate=0 mce=ignore_ce audit=0 nmi_watchdog=0 iommu=pt /
+intel_iommu=on  hugepagesz=1GB hugepages=8 default_hugepagesz=1GB /
+hugepagesz=2M hugepages=2048 vfio_iommu_type1.allow_unsafe_interrupts=1</programlisting>Kill
+          unnecessary services: <programlisting>killall ovsdb-server ovs-vswitchd
+rm -rf /etc/openvswitch/*
+mkdir -p /var/run/openvswitch</programlisting>Configure DPDK:<programlisting>mkdir -p /mnt/huge
+mount -t hugetlbfs nodev /mnt/huge
+modprobe igb_uio
+dpdk-devbind --bind=igb_uio 0000:03:00.0</programlisting>Configure
+          OVS:<programlisting>export DB_SOCK=/var/run/openvswitch/db.sock
+ovsdb-tool create /etc/openvswitch/conf.db /
+/usr/share/openvswitch/vswitch.ovsschema
+ovsdb-server --remote=punix:$DB_SOCK /
+--remote=db:Open_vSwitch,Open_vSwitch,manager_options --pidfile --detach
+ovs-vsctl --no-wait init
+ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-lcore-mask=0x10
+ovs-vsctl --no-wait set Open_vSwitch . other_config:pmd-cpu-mask=0xc
+ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-socket-mem=2048
+ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true
+ovs-vswitchd unix:$DB_SOCK --pidfile --detach /
+--log-file=/var/log/openvswitch/ovs-vswitchd.log
+
+ovs-vsctl add-br ovsbr0 -- set bridge ovsbr0 datapath_type=netdev
+ovs-vsctl add-port ovsbr0 vhost-user1 /
+-- set Interface vhost-user1 type=dpdkvhostuser -- set Interface /
+vhost-user1 ofport_request=2
+ovs-vsctl add-port ovsbr0 dpdk0 -- set Interface dpdk0 /
+type=dpdk options:dpdk-devargs=0000:03:00.0 /
+-- set Interface dpdk0 ofport_request=1
+chmod 777 /var/run/openvswitch/vhost-user1
+
+ovs-ofctl del-flows ovsbr0
+ovs-ofctl add-flow ovsbr0 in_port=1,action=output:2
+ovs-ofctl add-flow ovsbr0 in_port=2,action=output:1</programlisting>Launch
+          QEMU:<programlisting>taskset -c 0,1 qemu-system-x86_64 -cpu host,+invtsc,migratable=no /
+-M q35 -smp cores=2,sockets=1 -vcpu 0,affinity=0 -vcpu 1,affinity=1 /
+-enable-kvm -nographic -realtime mlock=on -kernel /mnt/qemu/bzImage /
+-drive file=/mnt/qemu/enea-image-virtualization-guest-qemux86-64.ext4,/
+if=virtio,format=raw -m 4096 -object memory-backend-file,id=mem,/
+size=4096M,mem-path=/mnt/huge,share=on -numa node,memdev=mem /
+-mem-prealloc -chardev socket,id=char0,path=/var/run/openvswitch/vhost-user1 /
+-netdev type=vhost-user,id=mynet1,chardev=char0,vhostforce /
+-device virtio-net-pci,mac=52:54:00:00:00:01,netdev=mynet1,/
+mrg_rxbuf=on,rx_queue_size=1024,csum=off,gso=off,guest_tso4=off,/
+guest_tso6=off,guest_ecn=off -append 'root=/dev/vda console=ttyS0 /
+hugepagesz=2M hugepages=1024 isolcpus=1 nohz_full=1 rcu_nocbs=1 /
+irqaffinity=0 rcu_nocb_poll intel_pstate=disable intel_idle.max_cstate=0 /
+processor.max_cstate=0 mce=ignore_ce audit=0'</programlisting>Inside QEMU,
+          configure DPDK: <programlisting>mkdir -p /mnt/huge
+mount -t hugetlbfs nodev /mnt/huge
+modprobe igb_uio
+dpdk-devbind --bind=igb_uio 0000:00:02.0</programlisting>Inside QEMU, run
+          testpmd: <programlisting>testpmd -c 0x3 -n 2 -d librte_pmd_virtio.so.1.1  /
+-- --burst 64 --disable-hw-vlan --disable-rss  -i --portmask=0x1 /
+--coremask=0x2  --nb-cores=1 --rxq=1 --txq=1 --txd=512 --rxd=512 /
+--txqflags=0xf00 --port-topology=chained</programlisting>For the <emphasis
+          role="bold">Forwarding test</emphasis>, start testpmd
+          directly:<programlisting>start</programlisting>For the <emphasis
+          role="bold">Termination test</emphasis>, set testpmd to only
+          receive, then start it:<programlisting>set fwd rxonly
+start</programlisting>On target 1, you may start pktgen traffic
+          now:<programlisting>start 0</programlisting>On target 2, use this
+          command to refresh the testpmd display and note the highest
+          values:<programlisting>show port stats 0</programlisting>To stop
+          traffic from pktgen, in order to choose a different frame
+          size:<programlisting>stop 0</programlisting>To clear numbers in
+          testpmd:<programlisting>clear port stats
+show port stats 0</programlisting><table>
+              <title>Results in forwarding mode</title>
+
+              <tgroup cols="8">
+                <tbody>
+                  <row>
+                    <entry align="center"><emphasis
+                    role="bold">Bytes</emphasis></entry>
+
+                    <entry align="center"><emphasis role="bold">pktgen pps
+                    RX</emphasis></entry>
+
+                    <entry align="center"><emphasis role="bold">pktgen pps
+                    TX</emphasis></entry>
+
+                    <entry align="center"><emphasis role="bold">testpmd pps
+                    RX</emphasis></entry>
+
+                    <entry align="center"><emphasis role="bold">testpmd pps
+                    TX</emphasis></entry>
+
+                    <entry align="center"><emphasis role="bold">pktgen MBits/s
+                    RX</emphasis></entry>
+
+                    <entry align="center"><emphasis role="bold">pktgen MBits/s
+                    TX</emphasis></entry>
+
+                    <entry align="center"><emphasis role="bold">throughput
+                    (%)</emphasis></entry>
+                  </row>
+
+                  <row>
+                    <entry role="bold"><emphasis
+                    role="bold">64</emphasis></entry>
+
+                    <entry>7755769</entry>
+
+                    <entry>14858714</entry>
+
+                    <entry>7755447</entry>
+
+                    <entry>7755447</entry>
+
+                    <entry>5207</entry>
+
+                    <entry>9984</entry>
+
+                    <entry>52.2</entry>
+                  </row>
+
+                  <row>
+                    <entry><emphasis role="bold">128</emphasis></entry>
+
+                    <entry>7714626</entry>
+
+                    <entry>8435184</entry>
+
+                    <entry>7520349</entry>
+
+                    <entry>6932520</entry>
+
+                    <entry>8204</entry>
+
+                    <entry>9986</entry>
+
+                    <entry>82.1</entry>
+                  </row>
+
+                  <row>
+                    <entry role="bold"><emphasis
+                    role="bold">256</emphasis></entry>
+
+                    <entry>4528847</entry>
+
+                    <entry>4528854</entry>
+
+                    <entry>4529030</entry>
+
+                    <entry>4529034</entry>
+
+                    <entry>9999</entry>
+
+                    <entry>9999</entry>
+
+                    <entry>99.9</entry>
+                  </row>
+                </tbody>
+              </tgroup>
+            </table><table>
+              <title>Results in termination mode</title>
+
+              <tgroup cols="5">
+                <tbody>
+                  <row>
+                    <entry align="center"><emphasis
+                    role="bold">Bytes</emphasis></entry>
+
+                    <entry align="center"><emphasis role="bold">pktgen pps
+                    TX</emphasis></entry>
+
+                    <entry align="center"><emphasis role="bold">testpmd pps
+                    RX</emphasis></entry>
+
+                    <entry align="center"><emphasis role="bold">pktgen MBits/s
+                    TX</emphasis></entry>
+
+                    <entry align="center"><emphasis role="bold">throughput
+                    (%)</emphasis></entry>
+                  </row>
+
+                  <row>
+                    <entry role="bold"><emphasis
+                    role="bold">64</emphasis></entry>
+
+                    <entry>15138992</entry>
+
+                    <entry>7290663</entry>
+
+                    <entry>10063</entry>
+
+                    <entry>48.2</entry>
+                  </row>
+
+                  <row>
+                    <entry><emphasis role="bold">128</emphasis></entry>
+
+                    <entry>8426825</entry>
+
+                    <entry>6902646</entry>
+
+                    <entry>9977</entry>
+
+                    <entry>81.9</entry>
+                  </row>
+
+                  <row>
+                    <entry role="bold"><emphasis
+                    role="bold">256</emphasis></entry>
+
+                    <entry>4528957</entry>
+
+                    <entry>4528912</entry>
+
+                    <entry>9999</entry>
+
+                    <entry>100</entry>
+                  </row>
+                </tbody>
+              </tgroup>
+            </table></para>
+        </section>
+      </section>
+
+      <section id="usecase-six">
+        <title>Use Case - Forward traffic between two VMs</title>
+
+        <para>Benchmark a combo test using two virtual machines, the first
+        with traffic forwarding to the second, which terminates it.</para>
+
+        <para>Measurements are made in:</para>
+
+        <itemizedlist>
+          <listitem>
+            <para>pktgen TX in pps and Mbits/s</para>
+          </listitem>
+
+          <listitem>
+            <para>testpmd TX and RX pps in VM1</para>
+          </listitem>
+
+          <listitem>
+            <para>testpmd RX pps in VM2</para>
+          </listitem>
+
+          <listitem>
+            <para>throughput in percents, by dividing<emphasis role="bold">
+            VM2 testpmd RX pps</emphasis> by <emphasis role="bold">pktgen TX
+            pps</emphasis></para>
+          </listitem>
+        </itemizedlist>
+
+        <section id="targetone-usecase-five">
+          <title>Test Setup for Target 1</title>
+
+          <para>Start by doing the following:</para>
+
+          <para>SSD boot using the following <literal>grub.cfg</literal>
+          entry: <programlisting>linux (hd0,gpt3)/boot/bzImage root=/dev/sda3 ip=dhcp nohz_full=1-7 /
+isolcpus=1-7 rcu-nocbs=1-7 rcu_nocb_poll intel_pstate=disable /
+clocksource=tsc tsc=reliable nohpet nosoftlockup intel_idle.max_cstate=0 /
+processor.max_cstate=0 mce=ignore_ce audit=0 nmi_watchdog=0 iommu=pt /
+intel_iommu=on  hugepagesz=1GB hugepages=8 default_hugepagesz=1GB /
+hugepagesz=2M hugepages=2048 vfio_iommu_type1.allow_unsafe_interrupts=1</programlisting>Kill
+          Services:<programlisting>killall ovsdb-server ovs-vswitchd
+rm -rf /etc/openvswitch/*
+mkdir -p /var/run/openvswitch</programlisting>Configure DPDK:<programlisting>mkdir -p /mnt/huge
+mount -t hugetlbfs nodev /mnt/huge
+modprobe igb_uio
+dpdk-devbind --bind=igb_uio 0000:03:00.0</programlisting>Run
+          pktgen:<programlisting>cd /usr/share/apps/pktgen/
+./pktgen -c 0x7 -n 4 --proc-type auto --socket-mem 256 /
+-w 0000:03:00.0 -- -P -m "[1:2].0"</programlisting>Set pktgen frame size to
+          use from [64, 128, 256, 512]:<programlisting>set 0 size 64</programlisting></para>
+        </section>
+
+        <section id="targettwo-usecase-six">
+          <title>Test Setup for Target 2</title>
+
+          <para>Start by doing the following:</para>
+
+          <para>SSD boot using the following <literal>grub.cfg</literal>
+          entry: <programlisting>linux (hd0,gpt3)/boot/bzImage root=/dev/sda3 ip=dhcp nohz_full=1-7 /
+isolcpus=1-7 rcu-nocbs=1-7 rcu_nocb_poll intel_pstate=disable /
+clocksource=tsc tsc=reliable nohpet nosoftlockup intel_idle.max_cstate=0 /
+processor.max_cstate=0 mce=ignore_ce audit=0 nmi_watchdog=0 iommu=pt /
+intel_iommu=on  hugepagesz=1GB hugepages=8 default_hugepagesz=1GB /
+hugepagesz=2M hugepages=2048 vfio_iommu_type1.allow_unsafe_interrupts=1</programlisting>Kill
+          Services:<programlisting>killall ovsdb-server ovs-vswitchd
+rm -rf /etc/openvswitch/*
+mkdir -p /var/run/openvswitch</programlisting>Configure DPDK:<programlisting>mkdir -p /mnt/huge
+mount -t hugetlbfs nodev /mnt/huge
+modprobe igb_uio
+dpdk-devbind --bind=igb_uio 0000:03:00.0</programlisting>Configure
+          OVS:<programlisting>export DB_SOCK=/var/run/openvswitch/db.sock
+ovsdb-tool create /etc/openvswitch/conf.db /
+/usr/share/openvswitch/vswitch.ovsschema
+ovsdb-server --remote=punix:$DB_SOCK /
+--remote=db:Open_vSwitch,Open_vSwitch,manager_options --pidfile --detach
+ovs-vsctl --no-wait init
+ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-lcore-mask=0x10
+ovs-vsctl --no-wait set Open_vSwitch . other_config:pmd-cpu-mask=0xc
+ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-socket-mem=2048
+ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true
+ovs-vswitchd unix:$DB_SOCK --pidfile /
+--detach --log-file=/var/log/openvswitch/ovs-vswitchd.log
+
+
+ovs-vsctl add-br ovsbr0 -- set bridge ovsbr0 datapath_type=netdev
+ovs-vsctl add-port ovsbr0 dpdk0 /
+-- set Interface dpdk0 type=dpdk options:dpdk-devargs=0000:03:00.0 ofport_request=1
+ovs-vsctl add-port ovsbr0 vhost-user1 /
+-- set Interface vhost-user1 type=dpdkvhostuser ofport_request=2
+ovs-vsctl add-port ovsbr0 vhost-user2 /
+-- set Interface vhost-user2 type=dpdkvhostuser ofport_request=3
+
+
+ovs-ofctl del-flows ovsbr0
+ovs-ofctl add-flow ovsbr0 in_port=1,action=output:2
+ovs-ofctl add-flow ovsbr0 in_port=2,action=output:3</programlisting>Launch
+          first QEMU instance, VM1:<programlisting>taskset -c 0,1 qemu-system-x86_64 -cpu host,+invtsc,migratable=no -M q35 /
+-smp cores=2,sockets=1 -vcpu 0,affinity=0 -vcpu 1,affinity=1 -enable-kvm /
+-nographic -realtime mlock=on -kernel /home/root/qemu/bzImage /
+-drive file=/home/root/qemu/enea-image-virtualization-guest-qemux86-64.ext4,/
+if=virtio,format=raw -m 2048 -object memory-backend-file,id=mem,/
+size=2048M,mem-path=/mnt/huge,share=on -numa node,memdev=mem /
+-mem-prealloc -chardev socket,id=char0,path=/var/run/openvswitch/vhost-user1 /
+-netdev type=vhost-user,id=mynet1,chardev=char0,vhostforce /
+-device virtio-net-pci,mac=52:54:00:00:00:01,netdev=mynet1,/
+mrg_rxbuf=on,rx_queue_size=1024,csum=off,gso=off,guest_tso4=off,/
+guest_tso6=off,guest_ecn=off -append 'root=/dev/vda console=ttyS0 /
+hugepagesz=2M hugepages=512 isolcpus=1 nohz_full=1 rcu_nocbs=1 /
+irqaffinity=0 rcu_nocb_poll intel_pstate=disable intel_idle.max_cstate=0 /
+processor.max_cstate=0 mce=ignore_ce audit=0'</programlisting>Connect to
+          Target 2 through a new SSH session and run a second QEMU instance
+          (to get its own console, separate from instance VM1). We shall call
+          this VM2:<programlisting>taskset -c 4,5 qemu-system-x86_64 -cpu host,+invtsc,migratable=no /
+-M q35 -smp cores=2,sockets=1 -vcpu 0,affinity=4 -vcpu 1,affinity=5 /
+-enable-kvm -nographic -realtime mlock=on -kernel /home/root/qemu2/bzImage /
+-drive file=/home/root/qemu2/enea-image-virtualization-guest-qemux86-64.ext4,/
+if=virtio,format=raw -m 2048 -object memory-backend-file,id=mem,size=2048M,/
+mem-path=/mnt/huge,share=on -numa node,memdev=mem -mem-prealloc /
+-chardev socket,id=char1,path=/var/run/openvswitch/vhost-user2 /
+-netdev type=vhost-user,id=mynet1,chardev=char1,vhostforce /
+-device virtio-net-pci,mac=52:54:00:00:00:02,netdev=mynet1,/
+mrg_rxbuf=on,rx_queue_size=1024,csum=off,gso=off,guest_tso4=off,/
+guest_tso6=off,guest_ecn=off -append 'root=/dev/vda console=ttyS0 /
+hugepagesz=2M hugepages=512 isolcpus=1 nohz_full=1 rcu_nocbs=1 /
+irqaffinity=0 rcu_nocb_poll intel_pstate=disable intel_idle.max_cstate=0 /
+processor.max_cstate=0 mce=ignore_ce audit=0'</programlisting>Configure DPDK
+          inside VM1:<programlisting>mkdir -p /mnt/huge
+mount -t hugetlbfs nodev /mnt/huge
+modprobe igb_uio
+dpdk-devbind --bind=igb_uio 0000:00:02.0</programlisting>Run testpmd inside
+          VM1:<programlisting>testpmd -c 0x3 -n 2 -d librte_pmd_virtio.so.1.1  /
+-- --burst 64 --disable-hw-vlan --disable-rss  -i /
+--portmask=0x1 --coremask=0x2  --nb-cores=1 --rxq=1 /
+--txq=1 --txd=512 --rxd=512 --txqflags=0xf00 --port-topology=chained</programlisting>Start
+          testpmd inside VM1:<programlisting>start</programlisting>Configure
+          DPDK inside VM2:<programlisting>mkdir -p /mnt/huge
+mount -t hugetlbfs nodev /mnt/huge
+modprobe igb_uio
+dpdk-devbind --bind=igb_uio 0000:00:02.0</programlisting>Run testpmd inside
+          VM2:<programlisting>testpmd -c 0x3 -n 2 -d librte_pmd_virtio.so.1.1  /
+-- --burst 64 --disable-hw-vlan --disable-rss -i --portmask=0x1 /
+--coremask=0x2  --nb-cores=1 --rxq=1 --txq=1 --txd=512 /
+--rxd=512 --txqflags=0xf00 --port-topology=chained</programlisting>Set VM2 for
+          termination and start testpmd:<programlisting>set fwd rxonly
+start</programlisting>On target 1, start pktgen traffic:<programlisting>start 0</programlisting>Use
+          this command to refresh testpmd display in VM1 and VM2 and note the
+          highest values:<programlisting>show port stats 0</programlisting>To
+          stop traffic from pktgen, in order to choose a different frame
+          size:<programlisting>stop 0</programlisting>To clear numbers in
+          testpmd:<programlisting>clear port stats
+show port stats 0</programlisting>For VM1, we record the stats relevant for
+          <emphasis role="bold">forwarding</emphasis>:</para>
+
+          <itemizedlist>
+            <listitem>
+              <para>RX, TX in pps</para>
+            </listitem>
+          </itemizedlist>
+
+          <para>Only Rx-pps and Tx-pps numbers are important here, they change
+          every time stats are displayed as long as there is traffic. Run the
+          command a few times and pick the best (maximum) values seen.</para>
+
+          <para>For VM2, we record the stats relevant for <emphasis
+          role="bold">termination</emphasis>:</para>
+
+          <itemizedlist>
+            <listitem>
+              <para>RX in pps (TX will be 0)</para>
+            </listitem>
+          </itemizedlist>
+
+          <para>For pktgen, we record only the TX side, because flow is
+          terminated, with no RX traffic reaching pktgen:</para>
+
+          <itemizedlist>
+            <listitem>
+              <para>TX in pps and Mbit/s</para>
+            </listitem>
+          </itemizedlist>
+
+          <table>
+            <title>Results in forwarding mode</title>
+
+            <tgroup cols="7">
+              <tbody>
+                <row>
+                  <entry align="center"><emphasis
+                  role="bold">Bytes</emphasis></entry>
+
+                  <entry align="center"><emphasis role="bold">pktgen pps
+                  TX</emphasis></entry>
+
+                  <entry align="center"><emphasis role="bold">VM1 testpmd pps
+                  RX</emphasis></entry>
+
+                  <entry align="center"><emphasis role="bold">VM1 testpmd pps
+                  TX</emphasis></entry>
+
+                  <entry align="center"><emphasis role="bold">VM2 testpmd pps
+                  RX</emphasis></entry>
+
+                  <entry align="center"><emphasis role="bold">pktgen MBits/s
+                  TX</emphasis></entry>
+
+                  <entry align="center"><emphasis role="bold">throughput
+                  (%)</emphasis></entry>
+                </row>
+
+                <row>
+                  <entry role="bold"><emphasis
+                  role="bold">64</emphasis></entry>
+
+                  <entry>14845113</entry>
+
+                  <entry>6826540</entry>
+
+                  <entry>5389680</entry>
+
+                  <entry>5383577</entry>
+
+                  <entry>9975</entry>
+
+                  <entry>36.2</entry>
+                </row>
+
+                <row>
+                  <entry><emphasis role="bold">128</emphasis></entry>
+
+                  <entry>8426683</entry>
+
+                  <entry>6825857</entry>
+
+                  <entry>5386971</entry>
+
+                  <entry>5384530</entry>
+
+                  <entry>9976</entry>
+
+                  <entry>63.9</entry>
+                </row>
+
+                <row>
+                  <entry role="bold"><emphasis
+                  role="bold">256</emphasis></entry>
+
+                  <entry>4528894</entry>
+
+                  <entry>4507975</entry>
+
+                  <entry>4507958</entry>
+
+                  <entry>4532457</entry>
+
+                  <entry>9999</entry>
+
+                  <entry>100</entry>
+                </row>
+              </tbody>
+            </tgroup>
+          </table>
+        </section>
+      </section>
+    </section>
+  </section>
 </chapter>
\ No newline at end of file