path: root/doc/book-enea-nfv-access-platform-guide/doc/benchmarks.xml

<?xml version="1.0" encoding="ISO-8859-1"?>
<!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd">
<chapter id="benchmarks">
  <title>Benchmarks</title>

  <section id="hw-setup">
    <title>Hardware Setup</title>

    <para>The following table describes all the needed prequisites for an apt
    hardware setup:</para>

    <table>
      <title>Hardware Setup</title>

      <tgroup cols="2">
        <colspec align="left" />

        <thead>
          <row>
            <entry align="center">Item</entry>

            <entry align="center">Description</entry>
          </row>
        </thead>

        <tbody>
          <row>
            <entry align="left">Server Platform</entry>

            <entry align="left">Supermicro X10SDV-4C-TLN2F
            http://www.supermicro.com/products/motherboard/xeon/d/X10SDV-4C-TLN2F.cfm</entry>
          </row>

          <row>
            <entry align="left">ARCH</entry>

            <entry>x86-64</entry>
          </row>

          <row>
            <entry align="left">Processor</entry>

            <entry>1 x Intel Xeon D-1521 (Broadwell), 4 cores, 8
            hyper-threaded cores per processor</entry>
          </row>

          <row>
            <entry align="left">CPU freq</entry>

            <entry>2.40 GHz</entry>
          </row>

          <row>
            <entry align="left">RAM</entry>

            <entry>16GB</entry>
          </row>

          <row>
            <entry align="left">Network</entry>

            <entry>Dual integrated 10G ports</entry>
          </row>

          <row>
            <entry align="left">Storage</entry>

            <entry>Samsung 850 Pro 128GB SSD</entry>
          </row>
        </tbody>
      </tgroup>
    </table>

    <para>Generic tests configuration:</para>

    <itemizedlist>
      <listitem>
        <para>All tests use one port, one core and one Rx/TX queue for fast
        path traffic.</para>
      </listitem>
    </itemizedlist>
  </section>

  <section id="bios">
    <title>BIOS Settings</title>

    <para>The table below details the BIOS settings for which the default
    values were changed when doing performance measurements.</para>

    <table>
      <title>BIOS Settings</title>

      <tgroup cols="4">
        <colspec align="left" />

        <thead>
          <row>
            <entry align="center">Menu Path</entry>

            <entry align="center">Setting Name</entry>

            <entry align="center">Enea NFV Access value</entry>

            <entry align="center">BIOS Default value</entry>
          </row>
        </thead>

        <tbody>
          <row>
            <entry align="left">CPU Configuration</entry>

            <entry align="left">Direct Cache Access (DCA)</entry>

            <entry>Enable</entry>

            <entry>Auto</entry>
          </row>

          <row>
            <entry>CPU Configuration / Advanced Power Management
            Configuration</entry>

            <entry align="left">EIST (P-States)</entry>

            <entry>Disable</entry>

            <entry>Enable</entry>
          </row>

          <row>
            <entry>CPU Configuration / Advanced Power Management Configuration
            / CPU C State Control</entry>

            <entry align="left">CPU C State</entry>

            <entry>Disable</entry>

            <entry>Enable</entry>
          </row>

          <row>
            <entry>CPU Configuration / Advanced Power Management Configuration
            / CPU Advanced PM Turning / Energy Perf BIAS</entry>

            <entry align="left">Energy Performance Tuning</entry>

            <entry>Disable</entry>

            <entry>Enable</entry>
          </row>

          <row>
            <entry>CPU Configuration / Advanced Power Management Configuration
            / CPU Advanced PM Turning / Energy Perf BIAS</entry>

            <entry align="left">Energy Performance BIAS Setting</entry>

            <entry>Performance</entry>

            <entry>Balanced Performance</entry>
          </row>

          <row>
            <entry>CPU Configuration / Advanced Power Management Configuration
            / CPU Advanced PM Turning / Energy Perf BIAS</entry>

            <entry align="left">Power/Performance Switch</entry>

            <entry>Disable</entry>

            <entry>Enable</entry>
          </row>

          <row>
            <entry>CPU Configuration / Advanced Power Management Configuration
            / CPU Advanced PM Turning / Program PowerCTL _MSR</entry>

            <entry align="left">Energy Efficient Turbo</entry>

            <entry>Disable</entry>

            <entry>Enable</entry>
          </row>

          <row>
            <entry>Chipset Configuration / North Bridge / IIO
            Configuration</entry>

            <entry align="left">EV DFX Features</entry>

            <entry>Enable</entry>

            <entry>Disable</entry>
          </row>

          <row>
            <entry>Chipset Configuration / North Bridge / Memory
            Configuration</entry>

            <entry align="left">Enforce POR</entry>

            <entry>Disable</entry>

            <entry>Enable</entry>
          </row>

          <row>
            <entry>Chipset Configuration / North Bridge / Memory
            Configuration</entry>

            <entry align="left">Memory Frequency</entry>

            <entry>2400</entry>

            <entry>Auto</entry>
          </row>

          <row>
            <entry>Chipset Configuration / North Bridge / Memory
            Configuration</entry>

            <entry align="left">DRAM RAPL Baseline</entry>

            <entry>Disable</entry>

            <entry>DRAM RAPL Mode 1</entry>
          </row>
        </tbody>
      </tgroup>
    </table>
  </section>

  <section id="use-cases">
    <title>Use Cases</title>

    <section id="docker-benchmarks">
      <title>Docker related benchmarks</title>

      <section>
        <title>Forward traffic in Docker</title>

        <para>Benchmarking traffic forwarding using testpmd in a Docker
        container.</para>

        <para>Pktgen is used to generate UDP traffic that will reach testpmd,
        running in a Docker image. It will then be forwarded back to source on
        the return trip (Forwarding).</para>

        <para>This test measures:</para>

        <itemizedlist>
          <listitem>
            <para>pktgen TX, RX in packets per second (pps) and Mbps</para>
          </listitem>

          <listitem>
            <para>testpmd TX, RX in packets per second (pps)</para>
          </listitem>

          <listitem>
            <para>divide testpmd RX / pktgen TX in pps to obtain throughput in
            percentages (%)</para>
          </listitem>
        </itemizedlist>

        <section id="usecase-one">
          <title>Test Setup for Target 1</title>

          <para>Start by following the steps below:</para>

          <para>SSD boot using the following <literal>grub.cfg</literal>
          entry: <programlisting>linux (hd0,gpt3)/boot/bzImage root=/dev/sda3 ip=dhcp nohz_full=1-7 /
isolcpus=1-7 rcu-nocbs=1-7 rcu_nocb_poll intel_pstate=disable /
clocksource=tsc tsc=reliable nohpet nosoftlockup intel_idle.max_cstate=0 /
processor.max_cstate=0 mce=ignore_ce audit=0 nmi_watchdog=0 iommu=pt /
intel_iommu=on  hugepagesz=1GB hugepages=8 default_hugepagesz=1GB /
hugepagesz=2M hugepages=2048 vfio_iommu_type1.allow_unsafe_interrupts=1</programlisting></para>

          <para>Kill unnecessary services:<programlisting>killall ovsdb-server ovs-vswitchd
rm -rf /etc/openvswitch/*
mkdir -p /var/run/openvswitch</programlisting>Mount hugepages and configure
          DPDK:<programlisting>mkdir -p /mnt/huge
mount -t hugetlbfs nodev /mnt/huge
modprobe igb_uio
dpdk-devbind --bind=igb_uio 0000:03:00.0</programlisting>Run
          pktgen:<programlisting>cd /usr/share/apps/pktgen/
./pktgen -c 0xF -n 1 -- -P -m "[3:2].0"</programlisting>In the pktgen console
          run:<programlisting>str</programlisting>To change framesize for
          pktgen, from [64, 128, 256, 512]:<programlisting>set 0 size &amp;lt;number&amp;gt;</programlisting></para>
        </section>

        <section id="usecase-two">
          <title>Test Setup for Target 2</title>

          <para>Start by following the steps below:</para>

          <para>SSD boot using the following <literal>grub.cfg</literal>
          entry:</para>

          <programlisting>linux (hd0,gpt3)/boot/bzImage root=/dev/sda3 ip=dhcp nohz_full=1-7 /
isolcpus=1-7 rcu-nocbs=1-7 rcu_nocb_poll intel_pstate=disable /
clocksource=tsc tsc=reliable nohpet nosoftlockup intel_idle.max_cstate=0 /
processor.max_cstate=0 mce=ignore_ce audit=0 nmi_watchdog=0 iommu=pt /
intel_iommu=on  hugepagesz=1GB hugepages=8 default_hugepagesz=1GB /
hugepagesz=2M hugepages=2048 vfio_iommu_type1.allow_unsafe_interrupts=1</programlisting>

          <para>It is expected to have Docker/guest image on target. Configure
          the OVS bridge:<programlisting># OVS old config clean-up
killall ovsdb-server ovs-vswitchd
rm -rf /etc/openvswitch/*
mkdir -p /var/run/openvswitch

# Mount hugepages and bind interfaces to dpdk
mkdir -p /mnt/huge
mount -t hugetlbfs nodev /mnt/huge
modprobe igb_uio
dpdk-devbind --bind=igb_uio 0000:03:00.0

# configure openvswitch with DPDK
export DB_SOCK=/var/run/openvswitch/db.sock
ovsdb-tool create /etc/openvswitch/conf.db /
/usr/share/openvswitch/vswitch.ovsschema
ovsdb-server --remote=punix:$DB_SOCK /
--remote=db:Open_vSwitch,Open_vSwitch,manager_options --pidfile --detach
ovs-vsctl --no-wait init
ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-lcore-mask=0x10
ovs-vsctl --no-wait set Open_vSwitch . other_config:pmd-cpu-mask=0xc
ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-socket-mem=2048
ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true
ovs-vswitchd unix:$DB_SOCK --pidfile --detach /
--log-file=/var/log/openvswitch/ovs-vswitchd.log

ovs-vsctl add-br ovsbr0 -- set bridge ovsbr0 datapath_type=netdev
ovs-vsctl add-port ovsbr0 vhost-user1 /
-- set Interface vhost-user1 type=dpdkvhostuser ofport_request=1
ovs-vsctl add-port ovsbr0 dpdk0 -- set Interface /
dpdk0 type=dpdk options:dpdk-devargs=0000:03:00.0 ofport_request=2

# configure static flows
ovs-ofctl del-flows ovsbr0
ovs-ofctl add-flow ovsbr0 in_port=1,action=output:2
ovs-ofctl add-flow ovsbr0 in_port=2,action=output:1</programlisting>Import a
          Docker container:<programlisting>docker import enea-image-virtualization-guest-qemux86-64.tar.gz el7_guest</programlisting>Start
          the Docker container:<programlisting>docker run -it --rm -v /var/run/openvswitch/:/var/run/openvswitch/ /
-v /mnt/huge:/mnt/huge el7_guest /bin/bash</programlisting>Start the testpmd
          application in Docker:<programlisting>testpmd -c 0x30 -n 2 --file-prefix prog1  --socket-mem 512 --no-pci /
--vdev=virtio_user0,path=/var/run/openvswitch/vhost-user1 /
-d /usr/lib/librte_pmd_virtio.so.1.1  -- --burst 64 --disable-hw-vlan /
--disable-rss  -i --portmask=0x1 --coremask=0x20  --nb-cores=1 /
--rxq=1 --txq=1 --txd=512 --rxd=512 --txqflags=0xf00 --port-topology=chained</programlisting>To
          start traffic <emphasis role="bold">forwarding</emphasis>, run the
          following command in testpmd CLI:<programlisting>start</programlisting>To
          start traffic but in <emphasis role="bold">termination</emphasis>
          mode (no traffic sent on TX), run following command in testpmd
          CLI:<programlisting>set fwd rxonly
start</programlisting><table>
              <title>Results in forwarding mode</title>

              <tgroup cols="8">
                <tbody>
                  <row>
                    <entry align="center"><emphasis
                    role="bold">Bytes</emphasis></entry>

                    <entry align="center"><emphasis role="bold">pktgen pps
                    TX</emphasis></entry>

                    <entry align="center"><emphasis role="bold">pktgen MBits/s
                    TX</emphasis></entry>

                    <entry align="center"><emphasis role="bold">pktgen pps
                    RX</emphasis></entry>

                    <entry align="center"><emphasis role="bold">pktgen MBits/s
                    RX</emphasis></entry>

                    <entry align="center"><emphasis role="bold">testpmd pps
                    RX</emphasis></entry>

                    <entry align="center"><emphasis role="bold">testpmd pps
                    TX</emphasis></entry>

                    <entry align="center"><emphasis role="bold">throughput
                    (%)</emphasis></entry>
                  </row>

                  <row>
                    <entry role="bold"><emphasis
                    role="bold">64</emphasis></entry>

                    <entry>14890993</entry>

                    <entry>10006</entry>

                    <entry>7706039</entry>

                    <entry>5178</entry>

                    <entry>7692807</entry>

                    <entry>7692864</entry>

                    <entry>51.74%</entry>
                  </row>

                  <row>
                    <entry><emphasis role="bold">128</emphasis></entry>

                    <entry>8435104</entry>

                    <entry>9999</entry>

                    <entry>7689458</entry>

                    <entry>9060</entry>

                    <entry>7684962</entry>

                    <entry>7684904</entry>

                    <entry>90.6%</entry>
                  </row>

                  <row>
                    <entry role="bold"><emphasis
                    role="bold">256</emphasis></entry>

                    <entry>4532384</entry>

                    <entry>9999</entry>

                    <entry>4532386</entry>

                    <entry>9998</entry>

                    <entry>4532403</entry>

                    <entry>4532403</entry>

                    <entry>99.9%</entry>
                  </row>
                </tbody>
              </tgroup>
            </table><table>
              <title>Results in termination mode</title>

              <tgroup cols="4">
                <tbody>
                  <row>
                    <entry align="center"><emphasis
                    role="bold">Bytes</emphasis></entry>

                    <entry align="center"><emphasis role="bold">pktgen pps
                    TX</emphasis></entry>

                    <entry align="center"><emphasis role="bold">testpmd pps
                    RX</emphasis></entry>

                    <entry align="center"><emphasis role="bold">throughput
                    (%)</emphasis></entry>
                  </row>

                  <row>
                    <entry role="bold"><emphasis
                    role="bold">64</emphasis></entry>

                    <entry>14890993</entry>

                    <entry>7330403</entry>

                    <entry>49,2%</entry>
                  </row>

                  <row>
                    <entry><emphasis role="bold">128</emphasis></entry>

                    <entry>8435104</entry>

                    <entry>7330379</entry>

                    <entry>86,9%</entry>
                  </row>

                  <row>
                    <entry role="bold"><emphasis
                    role="bold">256</emphasis></entry>

                    <entry>4532484</entry>

                    <entry>4532407</entry>

                    <entry>99,9%</entry>
                  </row>
                </tbody>
              </tgroup>
            </table></para>
        </section>
      </section>

      <section id="usecase-three-four">
        <title>Forward traffic from Docker to another Docker on the same
        host</title>

        <para>Benchmark a combo test using testpmd running in two Docker
        instances, one which Forwards traffic to the second one, which
        Terminates it.</para>

        <para>Packets are generated with pktgen and TX-d to the first testpmd,
        which will RX and Forward them to the second testpmd, which will RX
        and terminate them.</para>

        <para>Measurements are made in:</para>

        <itemizedlist>
          <listitem>
            <para>pktgen TX in pps and Mbits/s</para>
          </listitem>

          <listitem>
            <para>testpmd TX and RX pps in Docker1</para>
          </listitem>

          <listitem>
            <para>testpmd RX pps in Docker2</para>
          </listitem>
        </itemizedlist>

        <para>Throughput found as a percent, by dividing Docker2 <emphasis
        role="bold">testpmd RX pps</emphasis> by <emphasis role="bold">pktgen
        TX pps</emphasis>.</para>

        <section id="target-one-usecase-three">
          <title>Test Setup for Target 1</title>

          <para>Start by following the steps below:</para>

          <para>SSD boot using the following <literal>grub.cfg</literal>
          entry:</para>

          <programlisting>linux (hd0,gpt3)/boot/bzImage root=/dev/sda3 ip=dhcp nohz_full=1-7 /
isolcpus=1-7 rcu-nocbs=1-7 rcu_nocb_poll intel_pstate=disable /
clocksource=tsc tsc=reliable nohpet nosoftlockup intel_idle.max_cstate=0 /
processor.max_cstate=0 mce=ignore_ce audit=0 nmi_watchdog=0 iommu=pt /
intel_iommu=on  hugepagesz=1GB hugepages=8 default_hugepagesz=1GB /
hugepagesz=2M hugepages=2048 vfio_iommu_type1.allow_unsafe_interrupts=1</programlisting>

          <para>Configure DPDK:<programlisting>mkdir -p /mnt/huge
mount -t hugetlbfs nodev /mnt/huge
modprobe igb_uio
dpdk-devbind --bind=igb_uio 0000:03:00.0</programlisting>Run
          pktgen:<programlisting>cd /usr/share/apps/pktgen/
./pktgen -c 0xF -n 1 -- -P -m "[3:2].0"</programlisting>Choose one of the
          values from [64, 128, 256, 512] to change the packet
          size:<programlisting>set 0 size &lt;number&gt;</programlisting></para>
        </section>

        <section id="target-two-usecase-four">
          <title>Test Setup for Target 2</title>

          <para>Start by following the steps below:</para>

          <para>SSD boot using the following <literal>grub.cfg</literal>
          entry:</para>

          <programlisting>linux (hd0,gpt3)/boot/bzImage root=/dev/sda3 ip=dhcp nohz_full=1-7 /
isolcpus=1-7 rcu-nocbs=1-7 rcu_nocb_poll intel_pstate=disable /
clocksource=tsc tsc=reliable nohpet nosoftlockup intel_idle.max_cstate=0 /
processor.max_cstate=0 mce=ignore_ce audit=0 nmi_watchdog=0 /
iommu=pt intel_iommu=on  hugepagesz=1GB hugepages=8 default_hugepagesz=1GB /
hugepagesz=2M hugepages=2048 vfio_iommu_type1.allow_unsafe_interrupts=1</programlisting>

          <para><programlisting>killall ovsdb-server ovs-vswitchd
rm -rf /etc/openvswitch/*
mkdir -p /var/run/openvswitch</programlisting>Configure DPDK:<programlisting>mkdir -p /mnt/huge
mount -t hugetlbfs nodev /mnt/huge
modprobe igb_uio
dpdk-devbind --bind=igb_uio 0000:03:00.0</programlisting>Configure the OVS
          bridge:<programlisting>export DB_SOCK=/var/run/openvswitch/db.sock
ovsdb-tool create /etc/openvswitch/conf.db /
/usr/share/openvswitch/vswitch.ovsschema
ovsdb-server --remote=punix:$DB_SOCK /
--remote=db:Open_vSwitch,Open_vSwitch,manager_options --pidfile --detach
ovs-vsctl --no-wait init
ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-lcore-mask=0x10
ovs-vsctl --no-wait set Open_vSwitch . other_config:pmd-cpu-mask=0xcc
ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-socket-mem=2048
ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true
ovs-vswitchd unix:$DB_SOCK --pidfile --detach /
--log-file=/var/log/openvswitch/ovs-vswitchd.log
ovs-vsctl add-br ovsbr0 -- set bridge ovsbr0 datapath_type=netdev
ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface /
vhost-user1 type=dpdkvhostuser ofport_request=1
ovs-vsctl add-port ovsbr0 vhost-user2 -- set Interface /
vhost-user2 type=dpdkvhostuser ofport_request=2
ovs-vsctl add-port ovsbr0 dpdk0 -- set Interface dpdk0 /
type=dpdk options:dpdk-devargs=0000:03:00.0 ofport_request=3
ovs-ofctl del-flows ovsbr0
ovs-ofctl add-flow ovsbr0 in_port=3,action=output:2
ovs-ofctl add-flow ovsbr0 in_port=2,action=output:1</programlisting>Import a
          Docker container:<programlisting>docker import enea-image-virtualization-guest-qemux86-64.tar.gz el7_guest</programlisting>Start
          the first Docker:<programlisting>docker run -it --rm --cpuset-cpus=4,5 /
-v /var/run/openvswitch/:/var/run/openvswitch/ /
-v /mnt/huge:/mnt/huge el7_guest /bin/bash</programlisting>Start the testpmd
          application in Docker1:<programlisting>testpmd -c 0x30 -n 2 --file-prefix prog1  --socket-mem 512 --no-pci /
--vdev=virtio_user0,path=/var/run/openvswitch/vhost-user1 /
-d /usr/lib/librte_pmd_virtio.so.1.1 -- --burst 64 --disable-hw-vlan /
--disable-rss  -i --portmask=0x1 --coremask=0x20  --nb-cores=1 /
--rxq=1 --txq=1 --txd=512 --rxd=512 --txqflags=0xf00 --port-topology=chained</programlisting>Configure
          it in termination mode:<programlisting>set fwd rxonly</programlisting>Run
          the testpmd application:<programlisting>start</programlisting>Open a
          new console to the host and start the second Docker
          instance:<programlisting>docker run -it --rm --cpuset-cpus=0,1 -v /var/run/openvswitch/:/var/run/openvswitch/ /
-v /mnt/huge:/mnt/huge el7_guest /bin/bash</programlisting>In the second
          container start testpmd:<programlisting>testpmd -c 0x0F --file-prefix prog2 --socket-mem 512 --no-pci /
--vdev=virtio_user0,path=/var/run/openvswitch/vhost-user2 /
-d /usr/lib/librte_pmd_virtio.so.1.1 -- -i --disable-hw-vlan</programlisting>Run
          the TestPmd application in the second Docker:<programlisting>testpmd -c 0x3 -n 2 --file-prefix prog2  --socket-mem 512 --no-pci /
--vdev=virtio_user0,path=/var/run/openvswitch/vhost-user2 /
-d /usr/lib/librte_pmd_virtio.so.1.1 -- --burst 64 --disable-hw-vlan /
--disable-rss  -i --portmask=0x1 --coremask=0x2  --nb-cores=1 --rxq=1 /
--txq=1 --txd=512 --rxd=512 --txqflags=0xf00 --port-topology=chained</programlisting>In
          the testpmd shell, run:<programlisting>start</programlisting>Start
          pktgen traffic by running the following command in pktgen
          CLI:<programlisting>start 0</programlisting>To record traffic
          results:<programlisting>show port stats 0</programlisting>This
          should be used in testpmd applications.</para>

          <table>
            <title>Results</title>

            <tgroup cols="5">
              <tbody>
                <row>
                  <entry align="center"><emphasis
                  role="bold">Bytes</emphasis></entry>

                  <entry align="center"><emphasis role="bold">Target 1 -
                  pktgen pps TX</emphasis></entry>

                  <entry align="center"><emphasis role="bold">Target 2 -
                  (forwarding) testpmd pps RX</emphasis></entry>

                  <entry align="center"><emphasis role="bold">Target 2 -
                  (forwarding) testpmd pps TX</emphasis></entry>

                  <entry align="center"><emphasis role="bold">Target 2 -
                  (termination) testpmd pps RX</emphasis></entry>
                </row>

                <row>
                  <entry role="bold"><emphasis
                  role="bold">64</emphasis></entry>

                  <entry>14844628</entry>

                  <entry>5643565</entry>

                  <entry>3459922</entry>

                  <entry>3457326</entry>
                </row>

                <row>
                  <entry><emphasis role="bold">128</emphasis></entry>

                  <entry>8496962</entry>

                  <entry>5667860</entry>

                  <entry>3436811</entry>

                  <entry>3438918</entry>
                </row>

                <row>
                  <entry role="bold"><emphasis
                  role="bold">256</emphasis></entry>

                  <entry>4532372</entry>

                  <entry>4532362</entry>

                  <entry>3456623</entry>

                  <entry>3457115</entry>
                </row>

                <row>
                  <entry><emphasis role="bold">512</emphasis></entry>

                  <entry>2367641</entry>

                  <entry>2349450</entry>

                  <entry>2349450</entry>

                  <entry>2349446</entry>
                </row>
              </tbody>
            </tgroup>
          </table>
        </section>
      </section>
    </section>

    <section id="vm-benchmarks">
      <title>VM related benchmarks</title>

      <section id="usecase-four">
        <title>Forward/termination traffic in one VM</title>

        <para>Benchmarking traffic (UDP) forwarding and termination using
        testpmd in a virtual machine.</para>

        <para>The Pktgen application is used to generate traffic that will
        reach testpmd running on a virtual machine, and be forwarded back to
        source on the return trip. With the same setup a second measurement
        will be done with traffic termination in the virtual machine.</para>

        <para>This test case measures:</para>

        <itemizedlist>
          <listitem>
            <para>pktgen TX, RX in packets per second (pps) and Mbps</para>
          </listitem>

          <listitem>
            <para>testpmd TX, RX in packets per second (pps)</para>
          </listitem>

          <listitem>
            <para>divide <emphasis role="bold">testpmd RX</emphasis> by
            <emphasis role="bold">pktgen TX</emphasis> in pps to obtain the
            throughput in percentages (%)</para>
          </listitem>
        </itemizedlist>

        <section id="targetone-usecasefour">
          <title>Test Setup for Target 1</title>

          <para>Start with the steps below:</para>

          <para>SSD boot using the following <literal>grub.cfg
          </literal>entry: <programlisting>linux (hd0,gpt3)/boot/bzImage root=/dev/sda3 ip=dhcp nohz_full=1-7 /
isolcpus=1-7 rcu-nocbs=1-7 rcu_nocb_poll intel_pstate=disable /
clocksource=tsc tsc=reliable nohpet nosoftlockup intel_idle.max_cstate=0 /
processor.max_cstate=0 mce=ignore_ce audit=0 nmi_watchdog=0 iommu=pt /
intel_iommu=on hugepagesz=1GB hugepages=8 default_hugepagesz=1GB /
hugepagesz=2M hugepages=2048 vfio_iommu_type1.allow_unsafe_interrupts=1</programlisting></para>

          <para>Kill unnecessary services: <programlisting>killall ovsdb-server ovs-vswitchd
rm -rf /etc/openvswitch/*
mkdir -p /var/run/openvswitch</programlisting>Configure DPDK:<programlisting>mkdir -p /mnt/huge
mount -t hugetlbfs nodev /mnt/huge
modprobe igb_uio
dpdk-devbind --bind=igb_uio 0000:03:00.0</programlisting>Run
          pktgen:<programlisting>cd /usr/share/apps/pktgen/
./pktgen -c 0x7 -n 4 --proc-type auto --socket-mem 256 /
-w 0000:03:00.0 -- -P -m "[1:2].0"</programlisting>Set pktgen frame size to
          use from [64, 128, 256, 512]:<programlisting>set 0 size 64</programlisting></para>
        </section>

        <section id="targettwo-usecasefive">
          <title>Test Setup for Target 2</title>

          <para>Start by following the steps below:</para>

          <para>SSD boot using the following <literal>grub.cfg</literal>
          entry: <programlisting>linux (hd0,gpt3)/boot/bzImage root=/dev/sda3 ip=dhcp nohz_full=1-7 /
isolcpus=1-7 rcu-nocbs=1-7 rcu_nocb_poll intel_pstate=disable /
clocksource=tsc tsc=reliable nohpet nosoftlockup intel_idle.max_cstate=0 /
processor.max_cstate=0 mce=ignore_ce audit=0 nmi_watchdog=0 iommu=pt /
intel_iommu=on  hugepagesz=1GB hugepages=8 default_hugepagesz=1GB /
hugepagesz=2M hugepages=2048 vfio_iommu_type1.allow_unsafe_interrupts=1</programlisting>Kill
          unnecessary services: <programlisting>killall ovsdb-server ovs-vswitchd
rm -rf /etc/openvswitch/*
mkdir -p /var/run/openvswitch</programlisting>Configure DPDK:<programlisting>mkdir -p /mnt/huge
mount -t hugetlbfs nodev /mnt/huge
modprobe igb_uio
dpdk-devbind --bind=igb_uio 0000:03:00.0</programlisting>Configure
          OVS:<programlisting>export DB_SOCK=/var/run/openvswitch/db.sock
ovsdb-tool create /etc/openvswitch/conf.db /
/usr/share/openvswitch/vswitch.ovsschema
ovsdb-server --remote=punix:$DB_SOCK /
--remote=db:Open_vSwitch,Open_vSwitch,manager_options --pidfile --detach
ovs-vsctl --no-wait init
ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-lcore-mask=0x10
ovs-vsctl --no-wait set Open_vSwitch . other_config:pmd-cpu-mask=0xc
ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-socket-mem=2048
ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true
ovs-vswitchd unix:$DB_SOCK --pidfile --detach /
--log-file=/var/log/openvswitch/ovs-vswitchd.log

ovs-vsctl add-br ovsbr0 -- set bridge ovsbr0 datapath_type=netdev
ovs-vsctl add-port ovsbr0 vhost-user1 /
-- set Interface vhost-user1 type=dpdkvhostuser -- set Interface /
vhost-user1 ofport_request=2
ovs-vsctl add-port ovsbr0 dpdk0 -- set Interface dpdk0 /
type=dpdk options:dpdk-devargs=0000:03:00.0 /
-- set Interface dpdk0 ofport_request=1
chmod 777 /var/run/openvswitch/vhost-user1

ovs-ofctl del-flows ovsbr0
ovs-ofctl add-flow ovsbr0 in_port=1,action=output:2
ovs-ofctl add-flow ovsbr0 in_port=2,action=output:1</programlisting>Launch
          QEMU:<programlisting>taskset -c 0,1 qemu-system-x86_64 -cpu host,+invtsc,migratable=no /
-M q35 -smp cores=2,sockets=1 -vcpu 0,affinity=0 -vcpu 1,affinity=1 /
-enable-kvm -nographic -realtime mlock=on -kernel /mnt/qemu/bzImage /
-drive file=/mnt/qemu/enea-image-virtualization-guest-qemux86-64.ext4,/
if=virtio,format=raw -m 4096 -object memory-backend-file,id=mem,/
size=4096M,mem-path=/mnt/huge,share=on -numa node,memdev=mem /
-mem-prealloc -chardev socket,id=char0,path=/var/run/openvswitch/vhost-user1 /
-netdev type=vhost-user,id=mynet1,chardev=char0,vhostforce /
-device virtio-net-pci,mac=52:54:00:00:00:01,netdev=mynet1,/
mrg_rxbuf=on,rx_queue_size=1024,csum=off,gso=off,guest_tso4=off,/
guest_tso6=off,guest_ecn=off -append 'root=/dev/vda console=ttyS0 /
hugepagesz=2M hugepages=1024 isolcpus=1 nohz_full=1 rcu_nocbs=1 /
irqaffinity=0 rcu_nocb_poll intel_pstate=disable intel_idle.max_cstate=0 /
processor.max_cstate=0 mce=ignore_ce audit=0'</programlisting>Inside QEMU,
          configure DPDK: <programlisting>mkdir -p /mnt/huge
mount -t hugetlbfs nodev /mnt/huge
modprobe igb_uio
dpdk-devbind --bind=igb_uio 0000:00:02.0</programlisting>Inside QEMU, run
          testpmd: <programlisting>testpmd -c 0x3 -n 2 -d librte_pmd_virtio.so.1.1  /
-- --burst 64 --disable-hw-vlan --disable-rss  -i --portmask=0x1 /
--coremask=0x2  --nb-cores=1 --rxq=1 --txq=1 --txd=512 --rxd=512 /
--txqflags=0xf00 --port-topology=chained</programlisting>For the <emphasis
          role="bold">Forwarding test</emphasis>, start testpmd
          directly:<programlisting>start</programlisting>For the <emphasis
          role="bold">Termination test</emphasis>, set testpmd to only
          receive, then start it:<programlisting>set fwd rxonly
start</programlisting>On target 1, you may start pktgen traffic
          now:<programlisting>start 0</programlisting>On target 2, use this
          command to refresh the testpmd display and note the highest
          values:<programlisting>show port stats 0</programlisting>To stop
          traffic from pktgen, in order to choose a different frame
          size:<programlisting>stop 0</programlisting>To clear numbers in
          testpmd:<programlisting>clear port stats
show port stats 0</programlisting><table>
              <title>Results in forwarding mode</title>

              <tgroup cols="8">
                <tbody>
                  <row>
                    <entry align="center"><emphasis
                    role="bold">Bytes</emphasis></entry>

                    <entry align="center"><emphasis role="bold">pktgen pps
                    RX</emphasis></entry>

                    <entry align="center"><emphasis role="bold">pktgen pps
                    TX</emphasis></entry>

                    <entry align="center"><emphasis role="bold">testpmd pps
                    RX</emphasis></entry>

                    <entry align="center"><emphasis role="bold">testpmd pps
                    TX</emphasis></entry>

                    <entry align="center"><emphasis role="bold">pktgen MBits/s
                    RX</emphasis></entry>

                    <entry align="center"><emphasis role="bold">pktgen MBits/s
                    TX</emphasis></entry>

                    <entry align="center"><emphasis role="bold">throughput
                    (%)</emphasis></entry>
                  </row>

                  <row>
                    <entry role="bold"><emphasis
                    role="bold">64</emphasis></entry>

                    <entry>7755769</entry>

                    <entry>14858714</entry>

                    <entry>7755447</entry>

                    <entry>7755447</entry>

                    <entry>5207</entry>

                    <entry>9984</entry>

                    <entry>52.2</entry>
                  </row>

                  <row>
                    <entry><emphasis role="bold">128</emphasis></entry>

                    <entry>7714626</entry>

                    <entry>8435184</entry>

                    <entry>7520349</entry>

                    <entry>6932520</entry>

                    <entry>8204</entry>

                    <entry>9986</entry>

                    <entry>82.1</entry>
                  </row>

                  <row>
                    <entry role="bold"><emphasis
                    role="bold">256</emphasis></entry>

                    <entry>4528847</entry>

                    <entry>4528854</entry>

                    <entry>4529030</entry>

                    <entry>4529034</entry>

                    <entry>9999</entry>

                    <entry>9999</entry>

                    <entry>99.9</entry>
                  </row>
                </tbody>
              </tgroup>
            </table><table>
              <title>Results in termination mode</title>

              <tgroup cols="5">
                <tbody>
                  <row>
                    <entry align="center"><emphasis
                    role="bold">Bytes</emphasis></entry>

                    <entry align="center"><emphasis role="bold">pktgen pps
                    TX</emphasis></entry>

                    <entry align="center"><emphasis role="bold">testpmd pps
                    RX</emphasis></entry>

                    <entry align="center"><emphasis role="bold">pktgen MBits/s
                    TX</emphasis></entry>

                    <entry align="center"><emphasis role="bold">throughput
                    (%)</emphasis></entry>
                  </row>

                  <row>
                    <entry role="bold"><emphasis
                    role="bold">64</emphasis></entry>

                    <entry>15138992</entry>

                    <entry>7290663</entry>

                    <entry>10063</entry>

                    <entry>48.2</entry>
                  </row>

                  <row>
                    <entry><emphasis role="bold">128</emphasis></entry>

                    <entry>8426825</entry>

                    <entry>6902646</entry>

                    <entry>9977</entry>

                    <entry>81.9</entry>
                  </row>

                  <row>
                    <entry role="bold"><emphasis
                    role="bold">256</emphasis></entry>

                    <entry>4528957</entry>

                    <entry>4528912</entry>

                    <entry>9999</entry>

                    <entry>100</entry>
                  </row>
                </tbody>
              </tgroup>
            </table></para>
        </section>
      </section>

      <section id="usecase-six">
        <title>Forward traffic between two VMs</title>

        <para>Benchmark a combo test using two virtual machines, the first
        with traffic forwarding to the second, which terminates it.</para>

        <para>Measurements are made in:</para>

        <itemizedlist>
          <listitem>
            <para>pktgen TX in pps and Mbits/s</para>
          </listitem>

          <listitem>
            <para>testpmd TX and RX pps in VM1</para>
          </listitem>

          <listitem>
            <para>testpmd RX pps in VM2</para>
          </listitem>

          <listitem>
            <para>throughput in percents, by dividing<emphasis role="bold">
            VM2 testpmd RX pps</emphasis> by <emphasis role="bold">pktgen TX
            pps</emphasis></para>
          </listitem>
        </itemizedlist>

        <section id="targetone-usecase-five">
          <title>Test Setup for Target 1</title>

          <para>Start by doing the following:</para>

          <para>SSD boot using the following <literal>grub.cfg</literal>
          entry: <programlisting>linux (hd0,gpt3)/boot/bzImage root=/dev/sda3 ip=dhcp nohz_full=1-7 /
isolcpus=1-7 rcu-nocbs=1-7 rcu_nocb_poll intel_pstate=disable /
clocksource=tsc tsc=reliable nohpet nosoftlockup intel_idle.max_cstate=0 /
processor.max_cstate=0 mce=ignore_ce audit=0 nmi_watchdog=0 iommu=pt /
intel_iommu=on  hugepagesz=1GB hugepages=8 default_hugepagesz=1GB /
hugepagesz=2M hugepages=2048 vfio_iommu_type1.allow_unsafe_interrupts=1</programlisting>Kill
          Services:<programlisting>killall ovsdb-server ovs-vswitchd
rm -rf /etc/openvswitch/*
mkdir -p /var/run/openvswitch</programlisting>Configure DPDK:<programlisting>mkdir -p /mnt/huge
mount -t hugetlbfs nodev /mnt/huge
modprobe igb_uio
dpdk-devbind --bind=igb_uio 0000:03:00.0</programlisting>Run
          pktgen:<programlisting>cd /usr/share/apps/pktgen/
./pktgen -c 0x7 -n 4 --proc-type auto --socket-mem 256 /
-w 0000:03:00.0 -- -P -m "[1:2].0"</programlisting>Set pktgen frame size to
          use from [64, 128, 256, 512]:<programlisting>set 0 size 64</programlisting></para>
        </section>

        <section id="targettwo-usecase-six">
          <title>Test Setup for Target 2</title>

          <para>Start by doing the following:</para>

          <para>SSD boot using the following <literal>grub.cfg</literal>
          entry: <programlisting>linux (hd0,gpt3)/boot/bzImage root=/dev/sda3 ip=dhcp nohz_full=1-7 /
isolcpus=1-7 rcu-nocbs=1-7 rcu_nocb_poll intel_pstate=disable /
clocksource=tsc tsc=reliable nohpet nosoftlockup intel_idle.max_cstate=0 /
processor.max_cstate=0 mce=ignore_ce audit=0 nmi_watchdog=0 iommu=pt /
intel_iommu=on  hugepagesz=1GB hugepages=8 default_hugepagesz=1GB /
hugepagesz=2M hugepages=2048 vfio_iommu_type1.allow_unsafe_interrupts=1</programlisting>Kill
          Services:<programlisting>killall ovsdb-server ovs-vswitchd
rm -rf /etc/openvswitch/*
mkdir -p /var/run/openvswitch</programlisting>Configure DPDK:<programlisting>mkdir -p /mnt/huge
mount -t hugetlbfs nodev /mnt/huge
modprobe igb_uio
dpdk-devbind --bind=igb_uio 0000:03:00.0</programlisting>Configure
          OVS:<programlisting>export DB_SOCK=/var/run/openvswitch/db.sock
ovsdb-tool create /etc/openvswitch/conf.db /
/usr/share/openvswitch/vswitch.ovsschema
ovsdb-server --remote=punix:$DB_SOCK /
--remote=db:Open_vSwitch,Open_vSwitch,manager_options --pidfile --detach
ovs-vsctl --no-wait init
ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-lcore-mask=0x10
ovs-vsctl --no-wait set Open_vSwitch . other_config:pmd-cpu-mask=0xc
ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-socket-mem=2048
ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true
ovs-vswitchd unix:$DB_SOCK --pidfile /
--detach --log-file=/var/log/openvswitch/ovs-vswitchd.log


ovs-vsctl add-br ovsbr0 -- set bridge ovsbr0 datapath_type=netdev
ovs-vsctl add-port ovsbr0 dpdk0 /
-- set Interface dpdk0 type=dpdk options:dpdk-devargs=0000:03:00.0 ofport_request=1
ovs-vsctl add-port ovsbr0 vhost-user1 /
-- set Interface vhost-user1 type=dpdkvhostuser ofport_request=2
ovs-vsctl add-port ovsbr0 vhost-user2 /
-- set Interface vhost-user2 type=dpdkvhostuser ofport_request=3


ovs-ofctl del-flows ovsbr0
ovs-ofctl add-flow ovsbr0 in_port=1,action=output:2
ovs-ofctl add-flow ovsbr0 in_port=2,action=output:3</programlisting>Launch
          first QEMU instance, VM1:<programlisting>taskset -c 0,1 qemu-system-x86_64 -cpu host,+invtsc,migratable=no -M q35 /
-smp cores=2,sockets=1 -vcpu 0,affinity=0 -vcpu 1,affinity=1 -enable-kvm /
-nographic -realtime mlock=on -kernel /home/root/qemu/bzImage /
-drive file=/home/root/qemu/enea-image-virtualization-guest-qemux86-64.ext4,/
if=virtio,format=raw -m 2048 -object memory-backend-file,id=mem,/
size=2048M,mem-path=/mnt/huge,share=on -numa node,memdev=mem /
-mem-prealloc -chardev socket,id=char0,path=/var/run/openvswitch/vhost-user1 /
-netdev type=vhost-user,id=mynet1,chardev=char0,vhostforce /
-device virtio-net-pci,mac=52:54:00:00:00:01,netdev=mynet1,/
mrg_rxbuf=on,rx_queue_size=1024,csum=off,gso=off,guest_tso4=off,/
guest_tso6=off,guest_ecn=off -append 'root=/dev/vda console=ttyS0 /
hugepagesz=2M hugepages=512 isolcpus=1 nohz_full=1 rcu_nocbs=1 /
irqaffinity=0 rcu_nocb_poll intel_pstate=disable intel_idle.max_cstate=0 /
processor.max_cstate=0 mce=ignore_ce audit=0'</programlisting>Connect to
          Target 2 through a new SSH session and run a second QEMU instance
          (to get its own console, separate from instance VM1). We shall call
          this VM2:<programlisting>taskset -c 4,5 qemu-system-x86_64 -cpu host,+invtsc,migratable=no /
-M q35 -smp cores=2,sockets=1 -vcpu 0,affinity=4 -vcpu 1,affinity=5 /
-enable-kvm -nographic -realtime mlock=on -kernel /home/root/qemu2/bzImage /
-drive file=/home/root/qemu2/enea-image-virtualization-guest-qemux86-64.ext4,/
if=virtio,format=raw -m 2048 -object memory-backend-file,id=mem,size=2048M,/
mem-path=/mnt/huge,share=on -numa node,memdev=mem -mem-prealloc /
-chardev socket,id=char1,path=/var/run/openvswitch/vhost-user2 /
-netdev type=vhost-user,id=mynet1,chardev=char1,vhostforce /
-device virtio-net-pci,mac=52:54:00:00:00:02,netdev=mynet1,/
mrg_rxbuf=on,rx_queue_size=1024,csum=off,gso=off,guest_tso4=off,/
guest_tso6=off,guest_ecn=off -append 'root=/dev/vda console=ttyS0 /
hugepagesz=2M hugepages=512 isolcpus=1 nohz_full=1 rcu_nocbs=1 /
irqaffinity=0 rcu_nocb_poll intel_pstate=disable intel_idle.max_cstate=0 /
processor.max_cstate=0 mce=ignore_ce audit=0'</programlisting>Configure DPDK
          inside VM1:<programlisting>mkdir -p /mnt/huge
mount -t hugetlbfs nodev /mnt/huge
modprobe igb_uio
dpdk-devbind --bind=igb_uio 0000:00:02.0</programlisting>Run testpmd inside
          VM1:<programlisting>testpmd -c 0x3 -n 2 -d librte_pmd_virtio.so.1.1  /
-- --burst 64 --disable-hw-vlan --disable-rss  -i /
--portmask=0x1 --coremask=0x2  --nb-cores=1 --rxq=1 /
--txq=1 --txd=512 --rxd=512 --txqflags=0xf00 --port-topology=chained</programlisting>Start
          testpmd inside VM1:<programlisting>start</programlisting>Configure
          DPDK inside VM2:<programlisting>mkdir -p /mnt/huge
mount -t hugetlbfs nodev /mnt/huge
modprobe igb_uio
dpdk-devbind --bind=igb_uio 0000:00:02.0</programlisting>Run testpmd inside
          VM2:<programlisting>testpmd -c 0x3 -n 2 -d librte_pmd_virtio.so.1.1  /
-- --burst 64 --disable-hw-vlan --disable-rss -i --portmask=0x1 /
--coremask=0x2  --nb-cores=1 --rxq=1 --txq=1 --txd=512 /
--rxd=512 --txqflags=0xf00 --port-topology=chained</programlisting>Set VM2 for
          termination and start testpmd:<programlisting>set fwd rxonly
start</programlisting>On target 1, start pktgen traffic:<programlisting>start 0</programlisting>Use
          this command to refresh testpmd display in VM1 and VM2 and note the
          highest values:<programlisting>show port stats 0</programlisting>To
          stop traffic from pktgen, in order to choose a different frame
          size:<programlisting>stop 0</programlisting>To clear numbers in
          testpmd:<programlisting>clear port stats
show port stats 0</programlisting>For VM1, we record the stats relevant for
          <emphasis role="bold">forwarding</emphasis>:</para>

          <itemizedlist>
            <listitem>
              <para>RX, TX in pps</para>
            </listitem>
          </itemizedlist>

          <para>Only Rx-pps and Tx-pps numbers are important here, they change
          every time stats are displayed as long as there is traffic. Run the
          command a few times and pick the best (maximum) values seen.</para>

          <para>For VM2, we record the stats relevant for <emphasis
          role="bold">termination</emphasis>:</para>

          <itemizedlist>
            <listitem>
              <para>RX in pps (TX will be 0)</para>
            </listitem>
          </itemizedlist>

          <para>For pktgen, we record only the TX side, because flow is
          terminated, with no RX traffic reaching pktgen:</para>

          <itemizedlist>
            <listitem>
              <para>TX in pps and Mbit/s</para>
            </listitem>
          </itemizedlist>

          <table>
            <title>Results in forwarding mode</title>

            <tgroup cols="7">
              <tbody>
                <row>
                  <entry align="center"><emphasis
                  role="bold">Bytes</emphasis></entry>

                  <entry align="center"><emphasis role="bold">pktgen pps
                  TX</emphasis></entry>

                  <entry align="center"><emphasis role="bold">VM1 testpmd pps
                  RX</emphasis></entry>

                  <entry align="center"><emphasis role="bold">VM1 testpmd pps
                  TX</emphasis></entry>

                  <entry align="center"><emphasis role="bold">VM2 testpmd pps
                  RX</emphasis></entry>

                  <entry align="center"><emphasis role="bold">pktgen MBits/s
                  TX</emphasis></entry>

                  <entry align="center"><emphasis role="bold">throughput
                  (%)</emphasis></entry>
                </row>

                <row>
                  <entry role="bold"><emphasis
                  role="bold">64</emphasis></entry>

                  <entry>14845113</entry>

                  <entry>6826540</entry>

                  <entry>5389680</entry>

                  <entry>5383577</entry>

                  <entry>9975</entry>

                  <entry>36.2</entry>
                </row>

                <row>
                  <entry><emphasis role="bold">128</emphasis></entry>

                  <entry>8426683</entry>

                  <entry>6825857</entry>

                  <entry>5386971</entry>

                  <entry>5384530</entry>

                  <entry>9976</entry>

                  <entry>63.9</entry>
                </row>

                <row>
                  <entry role="bold"><emphasis
                  role="bold">256</emphasis></entry>

                  <entry>4528894</entry>

                  <entry>4507975</entry>

                  <entry>4507958</entry>

                  <entry>4532457</entry>

                  <entry>9999</entry>

                  <entry>100</entry>
                </row>
              </tbody>
            </tgroup>
          </table>
        </section>
      </section>
    </section>
  </section>
</chapter>