Cross compile lm benchmark for ARM

$ wget http://www.bitmover.com/lmbench/lmbench3.tar.gz
$ tar -zxvf lmbench3.tar.gz
$ cd lmbench3/src
$ cp Makefile Makefile.bak
$ vim Makefile
$ diff Makefile.bak Makefile
114c114
<  @env CFLAGS=-O MAKE="$(MAKE)" MAKEFLAGS="$(MAKEFLAGS)" CC="$(CC)" OS="$(OS)" ../scripts/build all
---
>  @env CFLAGS="-O -static" MAKE="$(MAKE)" MAKEFLAGS="$(MAKEFLAGS)" CC="$(CC)" OS="$(OS)" ../scripts/build all
231c231
< $O/lmbench : ../scripts/lmbench bk.ver
---
> $O/lmbench : ../scripts/lmbench #bk.ver
$ make CC=/opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc OS=arm-linux TARGET=linux 

An example script to run memory test only

#!/bin/bash -x

exec 2>&1

target=linux
sizes="1k 4k 16k 32k 128k 256k 512k 1m 2m 4m 8m" 
stride="32 128 1024"

date

LOGFILE="benchmark_run_raw.log"
LOGFILE2="benchmark_run.log"
echo "Creating / cleaning out file.";
cat /dev/null > $LOGFILE;
cat /dev/null > $LOGFILE2;

TITLE=`basename ${PWD}`
echo ">>>>>>   Benchmarks is $TITLE" >> $LOGFILE

echo "\"------mhz \"">> $LOGFILE
${RUNPREFIX} ./mhz  >> $LOGFILE

echo "\"------stream -v 1 -M 8k -W5 -N 10\"">>  $LOGFILE
${RUNPREFIX} ./stream -v 1 -M 8k -W5 -N 10 2>> $LOGFILE
echo "\"------stream -v 1 -M 128k -W5 -N 10\"">>  $LOGFILE
${RUNPREFIX} ./stream -v 1 -M 128k -W5 -N 10 2>> $LOGFILE
echo "\"------stream -v 1 -M 3m -W5 -N 10\"">>  $LOGFILE
${RUNPREFIX} ./stream -v 1 -M 3m -W5 -N 10 2>> $LOGFILE


for ds in ${stride}; do
 echo "\"------lat_mem_rd \"">>  $LOGFILE
    ${RUNPREFIX} ./lat_mem_rd $(echo ${sizes} | awk '{print $NF}') ${ds}  2>> $LOGFILE
done

echo "\"------bw_mem frd \"">>  $LOGFILE
for ds in ${sizes}; do
    ${RUNPREFIX} ./bw_mem ${ds} frd  2>> $LOGFILE
done

echo "\"------bw_mem rd \"">>  $LOGFILE
for ds in ${sizes}; do
    ${RUNPREFIX} ./bw_mem ${ds} rd  2>> $LOGFILE
done

    echo "\"------bw_mem wr \"">>  $LOGFILE
for ds in ${sizes}; do
    ${RUNPREFIX} ./bw_mem ${ds} wr  2>> $LOGFILE
done

echo "\"------bw_mem fwr \"">>  $LOGFILE
for ds in ${sizes}; do
    ${RUNPREFIX} ./bw_mem ${ds} fwr  2>> $LOGFILE
done

echo "\"------bw_mem fcp \"">>  $LOGFILE
for ds in ${sizes}; do
    ${RUNPREFIX} ./bw_mem ${ds} fcp  2>> $LOGFILE
done

echo "\"------bw_mem cp \"">>  $LOGFILE
for ds in ${sizes}; do
    ${RUNPREFIX} ./bw_mem ${ds} cp  2>> $LOGFILE
done

echo "\"------bw_mem rdrw \"">>  $LOGFILE
for ds in ${sizes}; do
    ${RUNPREFIX} ./bw_mem ${ds} rdwr  2>> $LOGFILE
done

echo "<<<<<<   End of $TITLE" >> $LOGFILE

cat $LOGFILE | sed -r -e '/stream -v 1 -M 8k/,/------lat_mem_rd/s/(.*): (.*) (.*)/\1(\3),\2/' -e '/stride=32/,/------bw_mem frd/s/^0.00977.*|^0.01367.*|^0.01758.*|^0.01953.*|^0.02148.*|^0.02539.*|^0.02734.*|^0.0293.*|^0.03516.*|^0.03906.*|^0.04297.*|^0.05078.*|^0.05469.*|^0.05859.*|^0.07031.*|^0.07812.*|^0.08594.*|^0.10156.*|^0.10938.*|^0.11719.*|^0.14062.*|^0.15625.*|^0.17188.*|^0.20312.*|^0.21875.*|^0.23438.*|^0.28125.*|^0.3125.*|^0.34375.*|^0.40625.*|^0.4375.*|^0.46875.*|^0.5625.*|^0.625.*|^0.6875.*|^0.8125.*|^0.875.*|^0.9375.*|^1.125.*|^1.25.*|^1.375.*|^1.625.*|^1.75.*|^1.875.*|^2.25.*|^2.5.*|^2.75.*|^3.25.*|^3.5.*|^3.75.*|^4.5.*|^5.*|^5.5.*|^6.5.*|^7.0.*|^7.5.*//' -e '/stride=1024/,/------bw_mem frd/s/^0.00049.*|^0.00098.*|^0.00195.*|^0.00293.*|^0.00391.*|^0.00586.*|^0.00781.*|^0.01172.*|^0.01562.*|^0.02344.*|^0.03125.*|^0.04688.*|^0.0625.*|^0.09375.*|^0.125.*|^0.1875.*|^0.25.*|^0.375.*|^0.5.*|^6.0000.*|^0.75.*|^3.0.*//' -e '/^$/d' -e 's/"stride/stride/' -e 's/([0-9]) ([0-9])/\1,\2/' > $LOGFILE2

date

here is a link that describe how to use lmbenchmark in detail: http://www.hexiongjun.com/?p=174

Linux kernel CRC checksum error

To remove the possible performance impact caused by NFS, I tried to add several test programs into my rootfs. First I added a 3M bytes file, everything is fine, then I added a lot of files with 18Mbytes in total, then suddenly my kernel can not be loaded.

U-Boot code: 00000000 -> 00047568  BSS: -> 00065340
DRAM:  Monitor len: 00065340
Ram size: 40000000
Ram top: 40000000
TLB table from 3fff0000 to 40000000
Reserving 404k for U-Boot at: 3ff8a000
Reserving 5184k for malloc() at: 3fa7a000
Reserving 176 Bytes for Board Info at: 3fa79f50
Reserving 408 Bytes for Global Data at: 3fa79db8
Reserving 11264 Bytes for FDT at: 3fa771b8
DRAM:  1 GiB
New Stack Pointer is: 3fa771a0
Relocation Offset is: 3ff8a000
Relocating to 3ff8a000, new gd at 3fa79db8, sp at 3fa771a0
Initializing board
Board: DB-88F3720-DDR3-Modular
       CPU     @ 600    [MHz]
       DDR     @ 600    [MHz]
Now running in RAM - U-Boot at: 3ff8a000
U-Boot DT blob at : 000000003fa771b8

Hit any key to stop autoboot:  0
## Booting kernel from Legacy Image at 0017ffc0 ...
   Image Name:   linux-4.1.6
   Image Type:   AArch64 Linux Kernel Image (uncompressed)
   Data Size:    15304304 Bytes = 14.6 MiB
   Load Address: 00180000
   Entry Point:  00180000
   Verifying Checksum ... Bad Data CRC
ERROR: can't get kernel image!

Weird thing is, if I load the kernel after u-boot already reach the prompt, everything is good. It looks U-boot write something into the kernel area during the init phase. How come? My first thought is to compare. U-boot has the command "cmp" to compare 2 chunk of data, what I need to do is to load an extra copy of my kernel into memory

LOAD /BINARY /ADDRESS=0x1017FFC0 OF "C:\incoming\bin\kernel_image"

Note my kernel was loaded to 0x0017FFC0 together with u-boot. Now do the compare with u-boot

U-boot>> cmp.b 0x0017ffc0 0x1017ffc0 15304308
byte at 0x0000000000fef4e0 (0x30) != byte at 0x0000000010fef4e0 (0x0)
Total of 15136032 byte(s) were the same
U-boot>> cmp.l 0xfef4e0 0x10fef4e0 0x10000
word at 0x0000000000fef4e0 (0xfef530) != word at 0x0000000010fef4e0 (0x100)
Total of 0 word(s) were the same

Let's dump the memory

U-boot>> md.l 0xfef4e0 0x100
00fef4e0: 00fef530 00000000 00020b44 00000000    0.......D.......
00fef4f0: ffffffff 00000000 00fef720 00000000    ........ .......
00fef500: 00000000 00000000 ffffffff 00000000    ................
00fef510: 0000000a 00000000 00000000 00000000    ................
00fef520: ffffffff 00000000 00000001 00000000    ................
00fef530: 00fef5e0 00000000 00021460 00000000    ........`.......
00fef540: 00fefc00 00000000 0003b527 00000000    ........'.......
00fef550: 00fef720 00000000 000313c8 00000000     ...............
00fef560: ffffffd0 00000000 00fef650 00000000    ........P.......
00fef570: 10004001 00004101 00fefc00 00000000    .@...A..........
00fef580: 00040000 000a4204 000313c8 00000000    .....B..........
00fef590: 00fef640 00000000 00021460 00000000    @.......`.......
00fef5a0: 00fefc60 00000000 0003494b 00000000    `.......KI......
00fef5b0: 00fef780 00000000 000313c8 00000000    ................
00fef5c0: ffffffe0 00000000 00fef6b0 00000000    ................
00fef5d0: 10004001 00004101 00fefc60 00000000    .@...A..`.......
00fef5e0: 00040000 000a4204 00021034 00000000    .....B..4.......
00fef5f0: 00fefc30 00000000 37316130 33666137    0.......0a177af3
00fef600: 00fef750 00000000 000313c8 00000000    P...............
00fef610: 00fef6b0 00000000 00021720 00000000    ........ .......
00fef620: 00fef750 00000000 0003b52e 00000000    P...............
00fef630: 40000000 00000000 00034776 00000000    ...@....vG......
00fef640: 00fef6e0 00000000 00021720 00000000    ........ .......
00fef650: 00fef780 00000000 0003491b 00000000    .........I......
00fef660: 00000000 00000000 40076d90 00000000    .........m.@....
00fef670: 00000000 00000000 00000000 00000000    ................
00fef680: 00000000 00800020 00000400 08010000    .... ...........
00fef690: 00040000 000a4204 00000d80 02008480    .....B..........
00fef6a0: 00fef6c0 00000000 000101e8 00000000    ................
00fef6b0: 00fef6c0 00000000 0001684c 00000000    ........Lh......
00fef6c0: 00fef6e0 00000000 0001694c 00000000    ........Li......
00fef6d0: 0000000a 00000000 000101e8 00000000    ................
00fef6e0: 00fef700 00000000 000168b4 00000000    .........h......
00fef6f0: 00fef7bb 00000000 0001689c 00000000    .........h......
00fef700: 00fef720 00000000 000101e8 00000000     ...............
00fef710: 0000003b 00000000 0003e198 00000000    ;...............
00fef720: 00fefc60 00000000 00004864 00000000    `.......dH......
00fef730: 0003e2a8 00000000 00fefc30 00000000    ........0.......
00fef740: 00fefc60 00000000 00fefc60 00000000    `.......`.......
00fef750: 00fefc20 00000000 ffffffc8 ffffff80     ...............
00fef760: 00fefc60 00000000 00fefc60 00000000    `.......`.......
00fef770: 00fefc20 00000000 ffffffc8 ffffff80     ...............
00fef780: 6f6c6552 69746163 7420676e 6633206f    Relocating to 3f
00fef790: 30613866 202c3030 2077656e 61206467    f8a000, new gd a
00fef7a0: 66332074 64393761 202c3862 61207073    t 3fa79db8, sp a
00fef7b0: 66332074 31373761 000a3061 36313032    t 3fa771a0..2016
00fef7c0: 2e31545f 6e652e30 72645f67 765f706f    _T1.0.eng_drop_v
00fef7d0: 000a0a32 00000000 00254e30 ffffffc0    2.......0N%.....
00fef7e0: 002554c8 ffffffc0 00000002 00000000    .T%.............
00fef7f0: 00000000 00000000 00000000 00000000    ................
00fef800: 00000000 00000000 00000000 00000000    ................
00fef810: 00000000 00000000 00fef878 ffffffc0    ........x.......
00fef820: 00fef858 ffffffc0 00fef8d8 ffffffc0    X...............
00fef830: 00fef8b8 ffffffc0 00fef898 ffffffc0    ................
00fef840: 00fef918 ffffffc0 00fef8f8 ffffffc0    ................
00fef850: 00000000 00000000 00699768 ffffffc0    ........h.i.....
00fef860: 000001a4 00000000 002550d0 ffffffc0    .........P%.....
00fef870: 00254ea0 ffffffc0 00699778 ffffffc0    .N%.....x.i.....
00fef880: 000001a4 00000000 00255100 ffffffc0    .........Q%.....
00fef890: 00255618 ffffffc0 00698958 ffffffc0    .V%.....X.i.....
00fef8a0: 00000124 00000000 00255040 ffffffc0    $.......@P%.....
00fef8b0: 00000000 00000000 00699780 ffffffc0    ..........i.....
00fef8c0: 00000124 00000000 00255070 ffffffc0    $.......pP%.....
00fef8d0: 00000000 00000000 006989b0 ffffffc0    ..........i.....

Look at the string "Relocating to"----it looks like this area is being used as a stack area by u-boot. What was the stack pointer when u-boot print "Relocating to"?  It was in function "setup_reloc()" but my debugger do not like the idea to set a break point on this function so I have to add an endless while loop



The stack pointer is 0xFEFC60, which is close to the mis-compare area. Of course it won't work because this stack conflict with my kernel image.  When and where is this stack pointer set? Set a break point on _main and I got



The initial stack pointer was defined by CONFIG_SYS_INIT_SP_ADDR. Refresh the definition and now the kernel can be loaded with large rootfs.

Debugging ARMv8 system clock issue

I noticed one issue with my board - the iperf server and client shows different performance data. Since the client is Windows PC and that has the correct clock, the clock on my embedded linux must be wrong. Here is how I debugged the issue

First figure out the source that is being used as clock

# cat /sys/devices/system/clocksource/clocksource0/available_clocksource
arch_sys_counter
# cat /sys/devices/system/clocksource/clocksource0/current_clocksource
arch_sys_counter
# cat /proc/interrupts
           CPU0       CPU1
  1:          0          0     GICv3  29 Edge      arch_timer
  2:       1117         93     GICv3  30 Edge      arch_timer
  5:       4558          0     GICv3  44 Level     serial_tx
  6:        515          0     GICv3  45 Level     serial_rx
  8:          0          0     GICv3 192 Level     xhci-hcd:usb1
  9:          0          0     GICv3 194 Level     a0020000.sata
 10:          0          0     GICv3 196 Level     mvri-irq
IPI0:       278        554       Rescheduling interrupts
IPI1:         4          4       Function call interrupts
IPI2:         0          0       CPU stop interrupts
IPI3:         0          0       Timer broadcast interrupts
IPI4:         0          0       IRQ work interrupts

It is the ARMv-8 arch counter, which is in the IP from ARM and every ARMv8 system should have one. Let's check if there is anything else related to clock from kernel message

# dmesg | grep clock
[    0.000000] clocksource arch_sys_counter: mask: 0xffffffffffffff max_cycles:
0x49cd42e20, max_idle_ns: 440795202120 ns
[    0.000003] sched_clock: 56 bits at 20MHz, resolution 50ns, wraps every 43980
46511100ns
[    0.005003] Registered arch_counter_get_cntvct+0x0/0x10 as sched_clock source

[    0.306657] clocksource jiffies: mask: 0xffffffff max_cycles: 0xffffffff, max
_idle_ns: 95563022313750000 ns
[    0.510202] Switched to clocksource arch_sys_counter

It said 56bits counter at 20MHz. Where was this value come from? Let's dig a little bit more. Grep "arch_sys_counter" and follow the calling stack finally I found the following code detect the clock frequency

static void
arch_timer_detect_rate(void __iomem *cntbase, struct device_node *np)
{
        /* Who has more than one independent system counter? */
        if (arch_timer_rate)
                return;

        /*
         * Try to determine the frequency from the device tree or CNTFRQ,
         * if ACPI is enabled, get the frequency from CNTFRQ ONLY.
         */
        if (!acpi_disabled ||
            of_property_read_u32(np, "clock-frequency", &arch_timer_rate)) {
                if (cntbase)
                        arch_timer_rate = readl_relaxed(cntbase + CNTFRQ);
                else
                        arch_timer_rate = arch_timer_get_cntfrq();
        }

        printk(KERN_INFO "arch_timer_rate is %08x , cntbase is %p. \n", 
            arch_timer_rate, cntbase);

        /* Check the timer frequency. */
        if (arch_timer_rate == 0)
                pr_warn("Architected timer frequency not available\n");
}

The code above first try to detect the clock frequency from device tree (in which I did not setup the timer entry), then try to get it from arch_timer_get_cntfrq().

vim arch/arm64/include/asm/arch_timer.h
static inline u32 arch_timer_get_cntfrq(void)
{
        u32 val;
        asm volatile("mrs %0,   cntfrq_el0" : "=r" (val));
        return val;
}

The clock frequency is in co-processor cntfrq_el0. Let's grep if the value of arch_timer_rate was printed out
# dmesg | grep arch_timer_rate
[    0.000000] arch_timer_rate is 01312d00 , cntbase is           (null).

Note 0x01312d00 is 20M. Is this value a default value on reset or was it preset by a piece of code? I grep "cntfrq_el0" in my kernel source codes and found nothing, and I tried to grep in u-boot's code to find this

 vim u-boot/arch/arm/cpu/armv8/nap/psci.S
 .global _armadalp_cpu_entry
_armadalp_cpu_entry:

        bl      enable_affinity

        isb

        /*
         * Could be EL3/EL2/EL1, Initial State:
         * Little Endian, MMU Disabled, i/dCache Disabled
         */
        adr     x0, vectors
        switch_el x1, 3f, 2f, 1f
3:      msr     vbar_el3, x0
        mrs     x0, scr_el3
        orr     x0, x0, #0xf                    /* SCR_EL3.NS|IRQ|FIQ|EA */
        msr     scr_el3, x0
        msr     cptr_el3, xzr                   /* Enable FP/SIMD */
        ldr     x0, =COUNTER_FREQUENCY
        msr     cntfrq_el0, x0                  /* Initialize CNTFRQ */
        b       0f
2:      msr     vbar_el2, x0
        mov     x0, #0x33ff
        msr     cptr_el2, x0                    /* Enable FP/SIMD */
        b       0f
1:      msr     vbar_el1, x0
        mov     x0, #3 << 20
        msr     cpacr_el1, x0                   /* Enable FP/SIMD */
0:

Register "cntfrq_el0" was initialized as COUNTER_FREQUENCY which was defined as 20M in a header file.  This is the value we used on another chip and I ported the u-boot based on that chip but I never got chance to modify this value. I talked with chip designer and figured out it shoud be  12.5M on our system.  Issue fixed by update the constant marco COUNTER_FREQUENCY.

Using iperf to measure the ethernet performance

It is simple to run iperf. On my embedded Linux I run iperf as server mode (by -s)

# ./iperf3 -s
warning: this system does not seem to support IPv6 - trying IPv4
-----------------------------------------------------------
Server listening on 5201


The go to Windows or Ubuntu, run iperf in client mode. To do TCP test: (the speed is limited to 100Mbps due to the speed of hub, the hardware is GbE though).


C:\Users\ttan\Downloads\iperf-3.1.3-win64\iperf-3.1.3-win64>iperf3.exe -c 10.85.151.106
Connecting to host 10.85.151.106, port 5201
[  4] local 10.85.151.105 port 58679 connected to 10.85.151.106 port 5201
[ ID] Interval           Transfer     Bandwidth
[  4]   0.00-1.00   sec  11.5 MBytes  96.4 Mbits/sec                 
[  4]   1.00-2.00   sec  11.2 MBytes  94.4 Mbits/sec                 
[  4]   2.00-3.00   sec  11.2 MBytes  94.4 Mbits/sec                 
[  4]   3.00-4.00   sec  11.4 MBytes  95.4 Mbits/sec                 
[  4]   4.00-5.00   sec  11.2 MBytes  94.3 Mbits/sec                 
[  4]   5.00-6.00   sec  11.2 MBytes  94.4 Mbits/sec                 
[  4]   6.00-7.00   sec  11.2 MBytes  94.4 Mbits/sec                 
[  4]   7.00-8.00   sec  11.4 MBytes  95.4 Mbits/sec                 
[  4]   8.00-9.00   sec  11.2 MBytes  94.3 Mbits/sec                 
[  4]   9.00-10.00  sec  11.2 MBytes  94.4 Mbits/sec                 
- - - - - - - - - - - - - - - - - - - - - - - - -
[ ID] Interval           Transfer     Bandwidth
[  4]   0.00-10.00  sec   113 MBytes  94.8 Mbits/sec                  sender
[  4]   0.00-10.00  sec   113 MBytes  94.8 Mbits/sec                  receiver

iperf Done.

Run UDP mode(by -u) and specify data of 100M bytes:
C:\Users\tiantan\Downloads\iperf-3.1.3-win64\iperf-3.1.3-win64>iperf3.exe -c 10.85.151.106 -u -b 100m
Connecting to host 10.85.151.106, port 5201
[  4] local 10.85.151.105 port 54749 connected to 10.85.151.106 port 5201
[ ID] Interval           Transfer     Bandwidth       Total Datagrams
[  4]   0.00-1.00   sec  10.5 MBytes  87.9 Mbits/sec  1342 
[  4]   1.00-2.00   sec  11.4 MBytes  95.6 Mbits/sec  1459 
[  4]   2.00-3.00   sec  11.4 MBytes  95.6 Mbits/sec  1459 
[  4]   3.00-4.00   sec  11.4 MBytes  95.6 Mbits/sec  1459 
[  4]   4.00-5.00   sec  11.4 MBytes  95.6 Mbits/sec  1458 
[  4]   5.00-6.00   sec  11.4 MBytes  95.6 Mbits/sec  1460 
[  4]   6.00-7.00   sec  11.4 MBytes  95.6 Mbits/sec  1459 
[  4]   7.00-8.00   sec  11.4 MBytes  95.6 Mbits/sec  1458 
[  4]   8.00-9.00   sec  11.4 MBytes  95.6 Mbits/sec  1459 
[  4]   9.00-10.00  sec  11.4 MBytes  95.6 Mbits/sec  1459 
- - - - - - - - - - - - - - - - - - - - - - - - -
[ ID] Interval           Transfer     Bandwidth       Jitter    Lost/Total Datagrams
[  4]   0.00-10.00  sec   113 MBytes  94.8 Mbits/sec  0.253 ms  0/14471 (0%) 
[  4] Sent 14471 datagrams

iperf Done.

Run TCP again with 20 threads:
C:\Users\ttan\Downloads\iperf-3.1.3-win64\iperf-3.1.3-win64>iperf3.exe -c 10.85.151.106 -P 20
Connecting to host 10.85.151.106, port 5201
[  4] local 10.85.151.105 port 58687 connected to 10.85.151.106 port 5201
[  6] local 10.85.151.105 port 58688 connected to 10.85.151.106 port 5201
[  8] local 10.85.151.105 port 58689 connected to 10.85.151.106 port 5201
[ 10] local 10.85.151.105 port 58690 connected to 10.85.151.106 port 5201
[ 12] local 10.85.151.105 port 58691 connected to 10.85.151.106 port 5201
[ 14] local 10.85.151.105 port 58692 connected to 10.85.151.106 port 5201
[ 16] local 10.85.151.105 port 58693 connected to 10.85.151.106 port 5201
[ 18] local 10.85.151.105 port 58694 connected to 10.85.151.106 port 5201
[ 20] local 10.85.151.105 port 58695 connected to 10.85.151.106 port 5201
[ 22] local 10.85.151.105 port 58696 connected to 10.85.151.106 port 5201
[ 24] local 10.85.151.105 port 58697 connected to 10.85.151.106 port 5201
[ 26] local 10.85.151.105 port 58698 connected to 10.85.151.106 port 5201
[ 28] local 10.85.151.105 port 58699 connected to 10.85.151.106 port 5201
[ 30] local 10.85.151.105 port 58700 connected to 10.85.151.106 port 5201
[ 32] local 10.85.151.105 port 58701 connected to 10.85.151.106 port 5201
[ 34] local 10.85.151.105 port 58702 connected to 10.85.151.106 port 5201
[ 36] local 10.85.151.105 port 58703 connected to 10.85.151.106 port 5201
[ 38] local 10.85.151.105 port 58704 connected to 10.85.151.106 port 5201
[ 40] local 10.85.151.105 port 58705 connected to 10.85.151.106 port 5201
[ 42] local 10.85.151.105 port 58706 connected to 10.85.151.106 port 5201
[ ID] Interval           Transfer     Bandwidth
[  4]   0.00-1.00   sec  1.25 MBytes  10.5 Mbits/sec                 
[  6]   0.00-1.00   sec  1.25 MBytes  10.5 Mbits/sec                 
[  8]   0.00-1.00   sec  1.00 MBytes  8.38 Mbits/sec                 
[ 10]   0.00-1.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 12]   0.00-1.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 14]   0.00-1.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 16]   0.00-1.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 18]   0.00-1.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 20]   0.00-1.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 22]   0.00-1.00   sec   256 KBytes  2.10 Mbits/sec                 
[ 24]   0.00-1.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 26]   0.00-1.00   sec   256 KBytes  2.10 Mbits/sec                 
[ 28]   0.00-1.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 30]   0.00-1.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 32]   0.00-1.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 34]   0.00-1.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 36]   0.00-1.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 38]   0.00-1.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 40]   0.00-1.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 42]   0.00-1.00   sec   512 KBytes  4.19 Mbits/sec                 
[SUM]   0.00-1.00   sec  13.0 MBytes   109 Mbits/sec                 
- - - - - - - - - - - - - - - - - - - - - - - - -
[  4]   1.00-2.00   sec   768 KBytes  6.29 Mbits/sec                 
[  6]   1.00-2.00   sec   640 KBytes  5.24 Mbits/sec                 
[  8]   1.00-2.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 10]   1.00-2.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 12]   1.00-2.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 14]   1.00-2.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 16]   1.00-2.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 18]   1.00-2.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 20]   1.00-2.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 22]   1.00-2.00   sec  0.00 Bytes  0.00 bits/sec                 
[ 24]   1.00-2.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 26]   1.00-2.00   sec  0.00 Bytes  0.00 bits/sec                 
[ 28]   1.00-2.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 30]   1.00-2.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 32]   1.00-2.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 34]   1.00-2.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 36]   1.00-2.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 38]   1.00-2.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 40]   1.00-2.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 42]   1.00-2.00   sec   512 KBytes  4.19 Mbits/sec                 
[SUM]   1.00-2.00   sec  12.1 MBytes   102 Mbits/sec                 
- - - - - - - - - - - - - - - - - - - - - - - - -
[  4]   2.00-3.00   sec   768 KBytes  6.30 Mbits/sec                 
[  6]   2.00-3.00   sec   640 KBytes  5.25 Mbits/sec                 
[  8]   2.00-3.00   sec   512 KBytes  4.20 Mbits/sec                 
[ 10]   2.00-3.00   sec   768 KBytes  6.30 Mbits/sec                 
[ 12]   2.00-3.00   sec   512 KBytes  4.20 Mbits/sec                 
[ 14]   2.00-3.00   sec   512 KBytes  4.20 Mbits/sec                 
[ 16]   2.00-3.00   sec   512 KBytes  4.20 Mbits/sec                 
[ 18]   2.00-3.00   sec   768 KBytes  6.30 Mbits/sec                 
[ 20]   2.00-3.00   sec   768 KBytes  6.30 Mbits/sec                 
[ 22]   2.00-3.00   sec  0.00 Bytes  0.00 bits/sec                 
[ 24]   2.00-3.00   sec   768 KBytes  6.30 Mbits/sec                 
[ 26]   2.00-3.00   sec  0.00 Bytes  0.00 bits/sec                 
[ 28]   2.00-3.00   sec   512 KBytes  4.20 Mbits/sec                 
[ 30]   2.00-3.00   sec   512 KBytes  4.20 Mbits/sec                 
[ 32]   2.00-3.00   sec   512 KBytes  4.20 Mbits/sec                 
[ 34]   2.00-3.00   sec   512 KBytes  4.20 Mbits/sec                 
[ 36]   2.00-3.00   sec   512 KBytes  4.20 Mbits/sec                 
[ 38]   2.00-3.00   sec   768 KBytes  6.30 Mbits/sec                 
[ 40]   2.00-3.00   sec   768 KBytes  6.30 Mbits/sec                 
[ 42]   2.00-3.00   sec   768 KBytes  6.30 Mbits/sec                 
[SUM]   2.00-3.00   sec  11.1 MBytes  93.4 Mbits/sec                 
- - - - - - - - - - - - - - - - - - - - - - - - -
[  4]   3.00-4.00   sec   512 KBytes  4.19 Mbits/sec                 
[  6]   3.00-4.00   sec   640 KBytes  5.24 Mbits/sec                 
[  8]   3.00-4.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 10]   3.00-4.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 12]   3.00-4.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 14]   3.00-4.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 16]   3.00-4.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 18]   3.00-4.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 20]   3.00-4.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 22]   3.00-4.00   sec   128 KBytes  1.05 Mbits/sec                 
[ 24]   3.00-4.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 26]   3.00-4.00   sec   128 KBytes  1.05 Mbits/sec                 
[ 28]   3.00-4.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 30]   3.00-4.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 32]   3.00-4.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 34]   3.00-4.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 36]   3.00-4.00   sec   640 KBytes  5.24 Mbits/sec                 
[ 38]   3.00-4.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 40]   3.00-4.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 42]   3.00-4.00   sec   512 KBytes  4.19 Mbits/sec                 
[SUM]   3.00-4.00   sec  11.5 MBytes  96.4 Mbits/sec                 
- - - - - - - - - - - - - - - - - - - - - - - - -
[  4]   4.00-5.00   sec   768 KBytes  6.29 Mbits/sec                 
[  6]   4.00-5.00   sec   640 KBytes  5.24 Mbits/sec                 
[  8]   4.00-5.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 10]   4.00-5.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 12]   4.00-5.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 14]   4.00-5.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 16]   4.00-5.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 18]   4.00-5.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 20]   4.00-5.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 22]   4.00-5.00   sec  0.00 Bytes  0.00 bits/sec                 
[ 24]   4.00-5.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 26]   4.00-5.00   sec  0.00 Bytes  0.00 bits/sec                 
[ 28]   4.00-5.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 30]   4.00-5.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 32]   4.00-5.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 34]   4.00-5.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 36]   4.00-5.00   sec   640 KBytes  5.24 Mbits/sec                 
[ 38]   4.00-5.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 40]   4.00-5.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 42]   4.00-5.00   sec   768 KBytes  6.29 Mbits/sec                 
[SUM]   4.00-5.00   sec  11.2 MBytes  94.4 Mbits/sec                 
- - - - - - - - - - - - - - - - - - - - - - - - -
[  4]   5.00-6.00   sec   512 KBytes  4.20 Mbits/sec                 
[  6]   5.00-6.00   sec   512 KBytes  4.20 Mbits/sec                 
[  8]   5.00-6.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 10]   5.00-6.00   sec   512 KBytes  4.20 Mbits/sec                 
[ 12]   5.00-6.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 14]   5.00-6.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 16]   5.00-6.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 18]   5.00-6.00   sec   512 KBytes  4.20 Mbits/sec                 
[ 20]   5.00-6.00   sec   512 KBytes  4.20 Mbits/sec                 
[ 22]   5.00-6.00   sec   128 KBytes  1.05 Mbits/sec                 
[ 24]   5.00-6.00   sec   512 KBytes  4.20 Mbits/sec                 
[ 26]   5.00-6.00   sec   128 KBytes  1.05 Mbits/sec                 
[ 28]   5.00-6.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 30]   5.00-6.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 32]   5.00-6.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 34]   5.00-6.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 36]   5.00-6.00   sec   640 KBytes  5.24 Mbits/sec                 
[ 38]   5.00-6.00   sec   512 KBytes  4.20 Mbits/sec                 
[ 40]   5.00-6.00   sec   512 KBytes  4.20 Mbits/sec                 
[ 42]   5.00-6.00   sec   512 KBytes  4.20 Mbits/sec                 
[SUM]   5.00-6.00   sec  11.4 MBytes  95.4 Mbits/sec                 
- - - - - - - - - - - - - - - - - - - - - - - - -
[  4]   6.00-7.00   sec   640 KBytes  5.24 Mbits/sec                 
[  6]   6.00-7.00   sec   768 KBytes  6.29 Mbits/sec                 
[  8]   6.00-7.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 10]   6.00-7.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 12]   6.00-7.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 14]   6.00-7.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 16]   6.00-7.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 18]   6.00-7.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 20]   6.00-7.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 22]   6.00-7.00   sec  0.00 Bytes  0.00 bits/sec                 
[ 24]   6.00-7.00   sec   640 KBytes  5.24 Mbits/sec                 
[ 26]   6.00-7.00   sec  0.00 Bytes  0.00 bits/sec                 
[ 28]   6.00-7.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 30]   6.00-7.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 32]   6.00-7.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 34]   6.00-7.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 36]   6.00-7.00   sec   640 KBytes  5.24 Mbits/sec                 
[ 38]   6.00-7.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 40]   6.00-7.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 42]   6.00-7.00   sec   768 KBytes  6.29 Mbits/sec                 
[SUM]   6.00-7.00   sec  11.1 MBytes  93.3 Mbits/sec                 
- - - - - - - - - - - - - - - - - - - - - - - - -
[  4]   7.00-8.00   sec   640 KBytes  5.25 Mbits/sec                 
[  6]   7.00-8.00   sec   512 KBytes  4.20 Mbits/sec                 
[  8]   7.00-8.00   sec   768 KBytes  6.30 Mbits/sec                 
[ 10]   7.00-8.00   sec   512 KBytes  4.20 Mbits/sec                 
[ 12]   7.00-8.00   sec   768 KBytes  6.30 Mbits/sec                 
[ 14]   7.00-8.00   sec   768 KBytes  6.30 Mbits/sec                 
[ 16]   7.00-8.00   sec   768 KBytes  6.30 Mbits/sec                 
[ 18]   7.00-8.00   sec   512 KBytes  4.20 Mbits/sec                 
[ 20]   7.00-8.00   sec   512 KBytes  4.20 Mbits/sec                 
[ 22]   7.00-8.00   sec   128 KBytes  1.05 Mbits/sec                 
[ 24]   7.00-8.00   sec   640 KBytes  5.25 Mbits/sec                 
[ 26]   7.00-8.00   sec   128 KBytes  1.05 Mbits/sec                 
[ 28]   7.00-8.00   sec   768 KBytes  6.30 Mbits/sec                 
[ 30]   7.00-8.00   sec   768 KBytes  6.30 Mbits/sec                 
[ 32]   7.00-8.00   sec   768 KBytes  6.30 Mbits/sec                 
[ 34]   7.00-8.00   sec   640 KBytes  5.25 Mbits/sec                 
[ 36]   7.00-8.00   sec   512 KBytes  4.20 Mbits/sec                 
[ 38]   7.00-8.00   sec   512 KBytes  4.20 Mbits/sec                 
[ 40]   7.00-8.00   sec   512 KBytes  4.20 Mbits/sec                 
[ 42]   7.00-8.00   sec   512 KBytes  4.20 Mbits/sec                 
[SUM]   7.00-8.00   sec  11.4 MBytes  95.5 Mbits/sec                 
- - - - - - - - - - - - - - - - - - - - - - - - -
[  4]   8.00-9.00   sec   512 KBytes  4.19 Mbits/sec                 
[  6]   8.00-9.00   sec   768 KBytes  6.29 Mbits/sec                 
[  8]   8.00-9.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 10]   8.00-9.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 12]   8.00-9.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 14]   8.00-9.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 16]   8.00-9.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 18]   8.00-9.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 20]   8.00-9.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 22]   8.00-9.00   sec  0.00 Bytes  0.00 bits/sec                 
[ 24]   8.00-9.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 26]   8.00-9.00   sec  0.00 Bytes  0.00 bits/sec                 
[ 28]   8.00-9.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 30]   8.00-9.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 32]   8.00-9.00   sec   512 KBytes  4.19 Mbits/sec                 
[ 34]   8.00-9.00   sec   640 KBytes  5.24 Mbits/sec                 
[ 36]   8.00-9.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 38]   8.00-9.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 40]   8.00-9.00   sec   768 KBytes  6.29 Mbits/sec                 
[ 42]   8.00-9.00   sec   768 KBytes  6.29 Mbits/sec                 
[SUM]   8.00-9.00   sec  11.1 MBytes  93.3 Mbits/sec                 
- - - - - - - - - - - - - - - - - - - - - - - - -
[  4]   9.00-10.00  sec   768 KBytes  6.29 Mbits/sec                 
[  6]   9.00-10.00  sec   512 KBytes  4.20 Mbits/sec                 
[  8]   9.00-10.00  sec   640 KBytes  5.24 Mbits/sec                 
[ 10]   9.00-10.00  sec   512 KBytes  4.20 Mbits/sec                 
[ 12]   9.00-10.00  sec   768 KBytes  6.29 Mbits/sec                 
[ 14]   9.00-10.00  sec   768 KBytes  6.29 Mbits/sec                 
[ 16]   9.00-10.00  sec   768 KBytes  6.29 Mbits/sec                 
[ 18]   9.00-10.00  sec   512 KBytes  4.20 Mbits/sec                 
[ 20]   9.00-10.00  sec   512 KBytes  4.20 Mbits/sec                 
[ 22]   9.00-10.00  sec   128 KBytes  1.05 Mbits/sec                 
[ 24]   9.00-10.00  sec   768 KBytes  6.29 Mbits/sec                 
[ 26]   9.00-10.00  sec   128 KBytes  1.05 Mbits/sec                 
[ 28]   9.00-10.00  sec   768 KBytes  6.29 Mbits/sec                 
[ 30]   9.00-10.00  sec   768 KBytes  6.29 Mbits/sec                 
[ 32]   9.00-10.00  sec   768 KBytes  6.29 Mbits/sec                 
[ 34]   9.00-10.00  sec   512 KBytes  4.20 Mbits/sec                 
[ 36]   9.00-10.00  sec   512 KBytes  4.20 Mbits/sec                 
[ 38]   9.00-10.00  sec   512 KBytes  4.20 Mbits/sec                 
[ 40]   9.00-10.00  sec   512 KBytes  4.20 Mbits/sec                 
[ 42]   9.00-10.00  sec   512 KBytes  4.20 Mbits/sec                 
[SUM]   9.00-10.00  sec  11.4 MBytes  95.4 Mbits/sec                 
- - - - - - - - - - - - - - - - - - - - - - - - -
[ ID] Interval           Transfer     Bandwidth
[  4]   0.00-10.00  sec  7.00 MBytes  5.87 Mbits/sec                  sender
[  4]   0.00-10.00  sec  7.00 MBytes  5.87 Mbits/sec                  receiver
[  6]   0.00-10.00  sec  6.75 MBytes  5.66 Mbits/sec                  sender
[  6]   0.00-10.00  sec  6.75 MBytes  5.66 Mbits/sec                  receiver
[  8]   0.00-10.00  sec  6.62 MBytes  5.56 Mbits/sec                  sender
[  8]   0.00-10.00  sec  6.62 MBytes  5.56 Mbits/sec                  receiver
[ 10]   0.00-10.00  sec  6.50 MBytes  5.45 Mbits/sec                  sender
[ 10]   0.00-10.00  sec  6.50 MBytes  5.45 Mbits/sec                  receiver
[ 12]   0.00-10.00  sec  6.50 MBytes  5.45 Mbits/sec                  sender
[ 12]   0.00-10.00  sec  6.50 MBytes  5.45 Mbits/sec                  receiver
[ 14]   0.00-10.00  sec  6.50 MBytes  5.45 Mbits/sec                  sender
[ 14]   0.00-10.00  sec  6.50 MBytes  5.45 Mbits/sec                  receiver
[ 16]   0.00-10.00  sec  6.50 MBytes  5.45 Mbits/sec                  sender
[ 16]   0.00-10.00  sec  6.50 MBytes  5.45 Mbits/sec                  receiver
[ 18]   0.00-10.00  sec  6.25 MBytes  5.24 Mbits/sec                  sender
[ 18]   0.00-10.00  sec  6.25 MBytes  5.24 Mbits/sec                  receiver
[ 20]   0.00-10.00  sec  6.25 MBytes  5.24 Mbits/sec                  sender
[ 20]   0.00-10.00  sec  6.25 MBytes  5.24 Mbits/sec                  receiver
[ 22]   0.00-10.00  sec   768 KBytes   629 Kbits/sec                  sender
[ 22]   0.00-10.00  sec  6.25 MBytes  5.24 Mbits/sec                  receiver
[ 24]   0.00-10.00  sec  6.25 MBytes  5.24 Mbits/sec                  sender
[ 24]   0.00-10.00  sec   653 KBytes   535 Kbits/sec                  receiver
[ 26]   0.00-10.00  sec   768 KBytes   629 Kbits/sec                  sender
[ 26]   0.00-10.00  sec   633 KBytes   519 Kbits/sec                  receiver
[ 28]   0.00-10.00  sec  6.25 MBytes  5.24 Mbits/sec                  sender
[ 28]   0.00-10.00  sec  6.25 MBytes  5.24 Mbits/sec                  receiver
[ 30]   0.00-10.00  sec  6.25 MBytes  5.24 Mbits/sec                  sender
[ 30]   0.00-10.00  sec  6.25 MBytes  5.24 Mbits/sec                  receiver
[ 32]   0.00-10.00  sec  6.25 MBytes  5.24 Mbits/sec                  sender
[ 32]   0.00-10.00  sec  6.25 MBytes  5.24 Mbits/sec                  receiver
[ 34]   0.00-10.00  sec  6.00 MBytes  5.03 Mbits/sec                  sender
[ 34]   0.00-10.00  sec  6.00 MBytes  5.03 Mbits/sec                  receiver
[ 36]   0.00-10.00  sec  6.00 MBytes  5.03 Mbits/sec                  sender
[ 36]   0.00-10.00  sec  6.00 MBytes  5.03 Mbits/sec                  receiver
[ 38]   0.00-10.00  sec  6.00 MBytes  5.03 Mbits/sec                  sender
[ 38]   0.00-10.00  sec  6.00 MBytes  5.03 Mbits/sec                  receiver
[ 40]   0.00-10.00  sec  6.00 MBytes  5.03 Mbits/sec                  sender
[ 40]   0.00-10.00  sec  6.00 MBytes  5.03 Mbits/sec                  receiver
[ 42]   0.00-10.00  sec  6.00 MBytes  5.03 Mbits/sec                  sender
[ 42]   0.00-10.00  sec  6.00 MBytes  5.03 Mbits/sec                  receiver
[SUM]   0.00-10.00  sec   115 MBytes  96.8 Mbits/sec                  sender
[SUM]   0.00-10.00  sec   115 MBytes  96.6 Mbits/sec                  receiver

iperf Done.

design a recommendation system

Reference:

• Hulu’s Recommendation System
• Recommender Systems
•  

http://blog.gainlo.co/index.php/2016/05/24/design-a-recommendation-system/
https://www.youtube.com/watch?v=NSscbT7JwxY

Cross compile iperf3 for Linux on ARMv8

iperf is a nice ethernet performance measurement tool. If you want try it on Microsoft Windows or Ubuntu Linux, chances are people already build the read-to-run binaries and you may simply download it from the iperf official website.

There is  a little bit tricky to run iperf on a home made embedded Linux system. In my case I ported Linux kernel to a dual ARM Cortex-A72 cores processor, and here is how to get iperf cross compiled.

Working on Linux Ubuntu v14 64bits,

lab-ubuntu64:~/iperf$ wget https://iperf.fr/download/source/iperf-3.1.3-source.tar.gz

lab-ubuntu64:~/iperf$ tar -xvf iperf-3.1.3-source.tar.gz

lab-ubuntu64:~/iperf$ cd iperf-3.1.3/

lab-ubuntu64:~/iperf/iperf-3.1.3$export PATH=/opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/:$PATH

lab-ubuntu64:~/iperf/iperf-3.1.3$echo $PATH
/opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/:/home/labuser/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin

lab-ubuntu64:~/iperf/iperf-3.1.3$export ARCH=arm64

lab-ubuntu64:~/iperf/iperf-3.1.3$export CROSS_COMPILE=/opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-

lab-ubuntu64:~/iperf/iperf-3.1.3$./configure --host=arm CC="/opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc"  CFLAGS=-static CXX="/opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-g++" CPPFLAGS=-static
checking for a BSD-compatible install... /usr/bin/install -c
checking whether build environment is sane... yes
checking for arm-strip... no
checking for strip... strip
checking for a thread-safe mkdir -p... /bin/mkdir -p
checking for gawk... no
checking for mawk... mawk
checking whether make sets $(MAKE)... yes
checking whether make supports nested variables... yes
checking whether to enable maintainer-specific portions of Makefiles... no
checking build system type... x86_64-unknown-linux-gnu
checking host system type... arm-unknown-none
checking for arm-gcc... /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc
checking whether the C compiler works... yes
checking for C compiler default output file name... a.out
checking for suffix of executables...
checking whether we are cross compiling... yes
checking for suffix of object files... o
checking whether we are using the GNU C compiler... yes
checking whether /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc accepts -g... yes
checking for /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc option to accept ISO C89... none needed
checking whether /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc understands -c and -o together... yes
checking for style of include used by make... GNU
checking dependency style of /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc... gcc3
checking for arm-ranlib... no
checking for ranlib... ranlib
checking whether ln -s works... yes
checking how to print strings... printf
checking for a sed that does not truncate output... /bin/sed
checking for grep that handles long lines and -e... /bin/grep
checking for egrep... /bin/grep -E
checking for fgrep... /bin/grep -F
checking for ld used by /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc... /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/aarch64-linux-gnu/bin/ld
checking if the linker (/opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/aarch64-linux-gnu/bin/ld) is GNU ld... yes
checking for BSD- or MS-compatible name lister (nm)... no
checking for arm-dumpbin... no
checking for arm-link... no
checking for dumpbin... no
checking for link... link -dump
checking the name lister (nm) interface... BSD nm
checking the maximum length of command line arguments... 1572864
checking how to convert x86_64-unknown-linux-gnu file names to arm-unknown-none format... func_convert_file_noop
checking how to convert x86_64-unknown-linux-gnu file names to toolchain format... func_convert_file_noop
checking for /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/aarch64-linux-gnu/bin/ld option to reload object files... -r
checking for arm-objdump... no
checking for objdump... objdump
checking how to recognize dependent libraries... unknown
checking for arm-dlltool... no
checking for dlltool... no
checking how to associate runtime and link libraries... printf %s\n
checking for arm-ar... no
checking for ar... ar
checking for archiver @FILE support... @
checking for arm-strip... strip
checking for arm-ranlib... ranlib
checking command to parse nm output from /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc object... ok
checking for sysroot... no
checking for a working dd... /bin/dd
checking how to truncate binary pipes... /bin/dd bs=4096 count=1
checking for arm-mt... no
checking for mt... mt
checking if mt is a manifest tool... no
checking how to run the C preprocessor... /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -E
checking for ANSI C header files... yes
checking for sys/types.h... yes
checking for sys/stat.h... yes
checking for stdlib.h... yes
checking for string.h... yes
checking for memory.h... yes
checking for strings.h... yes
checking for inttypes.h... yes
checking for stdint.h... yes
checking for unistd.h... yes
checking for dlfcn.h... yes
checking for objdir... .libs
checking if /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc supports -fno-rtti -fno-exceptions... no
checking for /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc option to produce PIC... -fPIC -DPIC
checking if /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc PIC flag -fPIC -DPIC works... yes
checking if /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc static flag -static works... yes
checking if /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc supports -c -o file.o... yes
checking if /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc supports -c -o file.o... (cached) yes
checking whether the /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc linker (/opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/aarch64-linux-gnu/bin/ld) supports shared libraries... yes
checking whether -lc should be explicitly linked in... no
checking dynamic linker characteristics... no
checking how to hardcode library paths into programs... immediate
checking whether stripping libraries is possible... yes
checking if libtool supports shared libraries... no
checking whether to build shared libraries... no
checking whether to build static libraries... yes
checking for ANSI C header files... (cached) yes
checking for library containing floor... -lm
checking for library containing nanosleep... none required
checking for library containing hstrerror... none required
checking for library containing socket... none required
checking for library containing inet_ntop... none required
checking for an ANSI C-conforming const... yes
checking sys/socket.h usability... yes
checking sys/socket.h presence... yes
checking for sys/socket.h... yes
checking for netinet/sctp.h... no
checking TCP_CONGESTION socket option... yes
checking IPv6 flowlabel support... yes
checking for cpuset_setaffinity... no
checking for sched_setaffinity... yes
checking for sendfile... yes
checking SO_MAX_PACING_RATE socket option... yes
checking that generated files are newer than configure... done
configure: creating ./config.status
config.status: creating Makefile
config.status: creating src/Makefile
config.status: creating src/version.h
config.status: creating examples/Makefile
config.status: creating iperf3.spec
config.status: creating src/iperf_config.h
config.status: src/iperf_config.h is unchanged
config.status: executing depfiles commands
config.status: executing libtool commands

lab-ubuntu64:~/iperf/iperf-3.1.3$make clean

lab-ubuntu64:~/iperf/iperf-3.1.3$make
Making all in src
make[1]: Entering directory `/home/labuser/iperf/iperf-3.1.3/src'
make  all-am
make[2]: Entering directory `/home/labuser/iperf/iperf-3.1.3/src'
/bin/bash ../libtool  --tag=CC   --mode=compile /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I.   -static  -static -Wall -MT cjson.lo -MD -MP -MF .deps/cjson.Tpo -c -o cjson.lo cjson.c
libtool: compile:  /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I. -Wall -MT cjson.lo -MD -MP -MF .deps/cjson.Tpo -c cjson.c -o cjson.o
mv -f .deps/cjson.Tpo .deps/cjson.Plo
/bin/bash ../libtool  --tag=CC   --mode=compile /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I.   -static  -static -Wall -MT iperf_api.lo -MD -MP -MF .deps/iperf_api.Tpo -c -o iperf_api.lo iperf_api.c
libtool: compile:  /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I. -Wall -MT iperf_api.lo -MD -MP -MF .deps/iperf_api.Tpo -c iperf_api.c -o iperf_api.o
mv -f .deps/iperf_api.Tpo .deps/iperf_api.Plo
/bin/bash ../libtool  --tag=CC   --mode=compile /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I.   -static  -static -Wall -MT iperf_error.lo -MD -MP -MF .deps/iperf_error.Tpo -c -o iperf_error.lo iperf_error.c
libtool: compile:  /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I. -Wall -MT iperf_error.lo -MD -MP -MF .deps/iperf_error.Tpo -c iperf_error.c -o iperf_error.o
mv -f .deps/iperf_error.Tpo .deps/iperf_error.Plo
/bin/bash ../libtool  --tag=CC   --mode=compile /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I.   -static  -static -Wall -MT iperf_client_api.lo -MD -MP -MF .deps/iperf_client_api.Tpo -c -o iperf_client_api.lo iperf_client_api.c
libtool: compile:  /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I. -Wall -MT iperf_client_api.lo -MD -MP -MF .deps/iperf_client_api.Tpo -c iperf_client_api.c -o iperf_client_api.o
mv -f .deps/iperf_client_api.Tpo .deps/iperf_client_api.Plo
/bin/bash ../libtool  --tag=CC   --mode=compile /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I.   -static  -static -Wall -MT iperf_locale.lo -MD -MP -MF .deps/iperf_locale.Tpo -c -o iperf_locale.lo iperf_locale.c
libtool: compile:  /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I. -Wall -MT iperf_locale.lo -MD -MP -MF .deps/iperf_locale.Tpo -c iperf_locale.c -o iperf_locale.o
mv -f .deps/iperf_locale.Tpo .deps/iperf_locale.Plo
/bin/bash ../libtool  --tag=CC   --mode=compile /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I.   -static  -static -Wall -MT iperf_server_api.lo -MD -MP -MF .deps/iperf_server_api.Tpo -c -o iperf_server_api.lo iperf_server_api.c
libtool: compile:  /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I. -Wall -MT iperf_server_api.lo -MD -MP -MF .deps/iperf_server_api.Tpo -c iperf_server_api.c -o iperf_server_api.o
mv -f .deps/iperf_server_api.Tpo .deps/iperf_server_api.Plo
/bin/bash ../libtool  --tag=CC   --mode=compile /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I.   -static  -static -Wall -MT iperf_tcp.lo -MD -MP -MF .deps/iperf_tcp.Tpo -c -o iperf_tcp.lo iperf_tcp.c
libtool: compile:  /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I. -Wall -MT iperf_tcp.lo -MD -MP -MF .deps/iperf_tcp.Tpo -c iperf_tcp.c -o iperf_tcp.o
mv -f .deps/iperf_tcp.Tpo .deps/iperf_tcp.Plo
/bin/bash ../libtool  --tag=CC   --mode=compile /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I.   -static  -static -Wall -MT iperf_udp.lo -MD -MP -MF .deps/iperf_udp.Tpo -c -o iperf_udp.lo iperf_udp.c
libtool: compile:  /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I. -Wall -MT iperf_udp.lo -MD -MP -MF .deps/iperf_udp.Tpo -c iperf_udp.c -o iperf_udp.o
mv -f .deps/iperf_udp.Tpo .deps/iperf_udp.Plo
/bin/bash ../libtool  --tag=CC   --mode=compile /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I.   -static  -static -Wall -MT iperf_sctp.lo -MD -MP -MF .deps/iperf_sctp.Tpo -c -o iperf_sctp.lo iperf_sctp.c
libtool: compile:  /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I. -Wall -MT iperf_sctp.lo -MD -MP -MF .deps/iperf_sctp.Tpo -c iperf_sctp.c -o iperf_sctp.o
mv -f .deps/iperf_sctp.Tpo .deps/iperf_sctp.Plo
/bin/bash ../libtool  --tag=CC   --mode=compile /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I.   -static  -static -Wall -MT iperf_util.lo -MD -MP -MF .deps/iperf_util.Tpo -c -o iperf_util.lo iperf_util.c
libtool: compile:  /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I. -Wall -MT iperf_util.lo -MD -MP -MF .deps/iperf_util.Tpo -c iperf_util.c -o iperf_util.o
mv -f .deps/iperf_util.Tpo .deps/iperf_util.Plo
/bin/bash ../libtool  --tag=CC   --mode=compile /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I.   -static  -static -Wall -MT net.lo -MD -MP -MF .deps/net.Tpo -c -o net.lo net.c
libtool: compile:  /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I. -Wall -MT net.lo -MD -MP -MF .deps/net.Tpo -c net.c -o net.o
mv -f .deps/net.Tpo .deps/net.Plo
/bin/bash ../libtool  --tag=CC   --mode=compile /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I.   -static  -static -Wall -MT tcp_info.lo -MD -MP -MF .deps/tcp_info.Tpo -c -o tcp_info.lo tcp_info.c
libtool: compile:  /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I. -Wall -MT tcp_info.lo -MD -MP -MF .deps/tcp_info.Tpo -c tcp_info.c -o tcp_info.o
mv -f .deps/tcp_info.Tpo .deps/tcp_info.Plo
/bin/bash ../libtool  --tag=CC   --mode=compile /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I.   -static  -static -Wall -MT tcp_window_size.lo -MD -MP -MF .deps/tcp_window_size.Tpo -c -o tcp_window_size.lo tcp_window_size.c
libtool: compile:  /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I. -Wall -MT tcp_window_size.lo -MD -MP -MF .deps/tcp_window_size.Tpo -c tcp_window_size.c -o tcp_window_size.o
mv -f .deps/tcp_window_size.Tpo .deps/tcp_window_size.Plo
/bin/bash ../libtool  --tag=CC   --mode=compile /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I.   -static  -static -Wall -MT timer.lo -MD -MP -MF .deps/timer.Tpo -c -o timer.lo timer.c
libtool: compile:  /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I. -Wall -MT timer.lo -MD -MP -MF .deps/timer.Tpo -c timer.c -o timer.o
mv -f .deps/timer.Tpo .deps/timer.Plo
/bin/bash ../libtool  --tag=CC   --mode=compile /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I.   -static  -static -Wall -MT units.lo -MD -MP -MF .deps/units.Tpo -c -o units.lo units.c
libtool: compile:  /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I. -Wall -MT units.lo -MD -MP -MF .deps/units.Tpo -c units.c -o units.o
mv -f .deps/units.Tpo .deps/units.Plo
/bin/bash ../libtool  --tag=CC   --mode=link /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc  -static -Wall   -o libiperf.la -rpath /usr/local/lib cjson.lo iperf_api.lo iperf_error.lo iperf_client_api.lo iperf_locale.lo iperf_server_api.lo iperf_tcp.lo iperf_udp.lo iperf_sctp.lo iperf_util.lo net.lo tcp_info.lo tcp_window_size.lo timer.lo units.lo  -lm
libtool: link: ar cru .libs/libiperf.a  cjson.o iperf_api.o iperf_error.o iperf_client_api.o iperf_locale.o iperf_server_api.o iperf_tcp.o iperf_udp.o iperf_sctp.o iperf_util.o net.o tcp_info.o tcp_window_size.o timer.o units.o
libtool: link: ranlib .libs/libiperf.a
libtool: link: ( cd ".libs" && rm -f "libiperf.la" && ln -s "../libiperf.la" "libiperf.la" )
/opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I.   -static -g -static -Wall -MT iperf3-main.o -MD -MP -MF .deps/iperf3-main.Tpo -c -o iperf3-main.o `test -f 'main.c' || echo './'`main.c
mv -f .deps/iperf3-main.Tpo .deps/iperf3-main.Po
/bin/bash ../libtool  --tag=CC   --mode=link /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -g -static -Wall -g  -o iperf3 iperf3-main.o libiperf.la -lm
libtool: link: /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -g -Wall -g -o iperf3 iperf3-main.o  ./.libs/libiperf.a -lm
/opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I.   -static -g -static -Wall -MT t_timer-t_timer.o -MD -MP -MF .deps/t_timer-t_timer.Tpo -c -o t_timer-t_timer.o `test -f 't_timer.c' || echo './'`t_timer.c
mv -f .deps/t_timer-t_timer.Tpo .deps/t_timer-t_timer.Po
/bin/bash ../libtool  --tag=CC   --mode=link /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -g -static -Wall   -o t_timer t_timer-t_timer.o libiperf.la -lm
libtool: link: /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -g -Wall -o t_timer t_timer-t_timer.o  ./.libs/libiperf.a -lm
/opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I.   -static -g -static -Wall -MT t_units-t_units.o -MD -MP -MF .deps/t_units-t_units.Tpo -c -o t_units-t_units.o `test -f 't_units.c' || echo './'`t_units.c
mv -f .deps/t_units-t_units.Tpo .deps/t_units-t_units.Po
/bin/bash ../libtool  --tag=CC   --mode=link /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -g -static -Wall   -o t_units t_units-t_units.o libiperf.la -lm
libtool: link: /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -g -Wall -o t_units t_units-t_units.o  ./.libs/libiperf.a -lm
/opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I.   -static -g -static -Wall -MT t_uuid-t_uuid.o -MD -MP -MF .deps/t_uuid-t_uuid.Tpo -c -o t_uuid-t_uuid.o `test -f 't_uuid.c' || echo './'`t_uuid.c
mv -f .deps/t_uuid-t_uuid.Tpo .deps/t_uuid-t_uuid.Po
/bin/bash ../libtool  --tag=CC   --mode=link /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -g -static -Wall   -o t_uuid t_uuid-t_uuid.o libiperf.la -lm
libtool: link: /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -g -Wall -o t_uuid t_uuid-t_uuid.o  ./.libs/libiperf.a -lm
/opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I.   -static -pg -g -static -Wall -MT iperf3_profile-main.o -MD -MP -MF .deps/iperf3_profile-main.Tpo -c -o iperf3_profile-main.o `test -f 'main.c' || echo './'`main.c
mv -f .deps/iperf3_profile-main.Tpo .deps/iperf3_profile-main.Po
/opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I.   -static -pg -g -static -Wall -MT iperf3_profile-cjson.o -MD -MP -MF .deps/iperf3_profile-cjson.Tpo -c -o iperf3_profile-cjson.o `test -f 'cjson.c' || echo './'`cjson.c
mv -f .deps/iperf3_profile-cjson.Tpo .deps/iperf3_profile-cjson.Po
/opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I.   -static -pg -g -static -Wall -MT iperf3_profile-iperf_api.o -MD -MP -MF .deps/iperf3_profile-iperf_api.Tpo -c -o iperf3_profile-iperf_api.o `test -f 'iperf_api.c' || echo './'`iperf_api.c
mv -f .deps/iperf3_profile-iperf_api.Tpo .deps/iperf3_profile-iperf_api.Po
/opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I.   -static -pg -g -static -Wall -MT iperf3_profile-iperf_error.o -MD -MP -MF .deps/iperf3_profile-iperf_error.Tpo -c -o iperf3_profile-iperf_error.o `test -f 'iperf_error.c' || echo './'`iperf_error.c
mv -f .deps/iperf3_profile-iperf_error.Tpo .deps/iperf3_profile-iperf_error.Po
/opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I.   -static -pg -g -static -Wall -MT iperf3_profile-iperf_client_api.o -MD -MP -MF .deps/iperf3_profile-iperf_client_api.Tpo -c -o iperf3_profile-iperf_client_api.o `test -f 'iperf_client_api.c' || echo './'`iperf_client_api.c
mv -f .deps/iperf3_profile-iperf_client_api.Tpo .deps/iperf3_profile-iperf_client_api.Po
/opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I.   -static -pg -g -static -Wall -MT iperf3_profile-iperf_locale.o -MD -MP -MF .deps/iperf3_profile-iperf_locale.Tpo -c -o iperf3_profile-iperf_locale.o `test -f 'iperf_locale.c' || echo './'`iperf_locale.c
mv -f .deps/iperf3_profile-iperf_locale.Tpo .deps/iperf3_profile-iperf_locale.Po
/opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I.   -static -pg -g -static -Wall -MT iperf3_profile-iperf_server_api.o -MD -MP -MF .deps/iperf3_profile-iperf_server_api.Tpo -c -o iperf3_profile-iperf_server_api.o `test -f 'iperf_server_api.c' || echo './'`iperf_server_api.c
mv -f .deps/iperf3_profile-iperf_server_api.Tpo .deps/iperf3_profile-iperf_server_api.Po
/opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I.   -static -pg -g -static -Wall -MT iperf3_profile-iperf_tcp.o -MD -MP -MF .deps/iperf3_profile-iperf_tcp.Tpo -c -o iperf3_profile-iperf_tcp.o `test -f 'iperf_tcp.c' || echo './'`iperf_tcp.c
mv -f .deps/iperf3_profile-iperf_tcp.Tpo .deps/iperf3_profile-iperf_tcp.Po
/opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I.   -static -pg -g -static -Wall -MT iperf3_profile-iperf_udp.o -MD -MP -MF .deps/iperf3_profile-iperf_udp.Tpo -c -o iperf3_profile-iperf_udp.o `test -f 'iperf_udp.c' || echo './'`iperf_udp.c
mv -f .deps/iperf3_profile-iperf_udp.Tpo .deps/iperf3_profile-iperf_udp.Po
/opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I.   -static -pg -g -static -Wall -MT iperf3_profile-iperf_sctp.o -MD -MP -MF .deps/iperf3_profile-iperf_sctp.Tpo -c -o iperf3_profile-iperf_sctp.o `test -f 'iperf_sctp.c' || echo './'`iperf_sctp.c
mv -f .deps/iperf3_profile-iperf_sctp.Tpo .deps/iperf3_profile-iperf_sctp.Po
/opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I.   -static -pg -g -static -Wall -MT iperf3_profile-iperf_util.o -MD -MP -MF .deps/iperf3_profile-iperf_util.Tpo -c -o iperf3_profile-iperf_util.o `test -f 'iperf_util.c' || echo './'`iperf_util.c
mv -f .deps/iperf3_profile-iperf_util.Tpo .deps/iperf3_profile-iperf_util.Po
/opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I.   -static -pg -g -static -Wall -MT iperf3_profile-net.o -MD -MP -MF .deps/iperf3_profile-net.Tpo -c -o iperf3_profile-net.o `test -f 'net.c' || echo './'`net.c
mv -f .deps/iperf3_profile-net.Tpo .deps/iperf3_profile-net.Po
/opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I.   -static -pg -g -static -Wall -MT iperf3_profile-tcp_info.o -MD -MP -MF .deps/iperf3_profile-tcp_info.Tpo -c -o iperf3_profile-tcp_info.o `test -f 'tcp_info.c' || echo './'`tcp_info.c
mv -f .deps/iperf3_profile-tcp_info.Tpo .deps/iperf3_profile-tcp_info.Po
/opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I.   -static -pg -g -static -Wall -MT iperf3_profile-tcp_window_size.o -MD -MP -MF .deps/iperf3_profile-tcp_window_size.Tpo -c -o iperf3_profile-tcp_window_size.o `test -f 'tcp_window_size.c' || echo './'`tcp_window_size.c
mv -f .deps/iperf3_profile-tcp_window_size.Tpo .deps/iperf3_profile-tcp_window_size.Po
/opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I.   -static -pg -g -static -Wall -MT iperf3_profile-timer.o -MD -MP -MF .deps/iperf3_profile-timer.Tpo -c -o iperf3_profile-timer.o `test -f 'timer.c' || echo './'`timer.c
mv -f .deps/iperf3_profile-timer.Tpo .deps/iperf3_profile-timer.Po
/opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I.   -static -pg -g -static -Wall -MT iperf3_profile-units.o -MD -MP -MF .deps/iperf3_profile-units.Tpo -c -o iperf3_profile-units.o `test -f 'units.c' || echo './'`units.c
mv -f .deps/iperf3_profile-units.Tpo .deps/iperf3_profile-units.Po
/bin/bash ../libtool  --tag=CC   --mode=link /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -pg -g -static -Wall -pg -g  -o iperf3_profile iperf3_profile-main.o iperf3_profile-cjson.o iperf3_profile-iperf_api.o iperf3_profile-iperf_error.o iperf3_profile-iperf_client_api.o iperf3_profile-iperf_locale.o iperf3_profile-iperf_server_api.o iperf3_profile-iperf_tcp.o iperf3_profile-iperf_udp.o iperf3_profile-iperf_sctp.o iperf3_profile-iperf_util.o iperf3_profile-net.o iperf3_profile-tcp_info.o iperf3_profile-tcp_window_size.o iperf3_profile-timer.o iperf3_profile-units.o libiperf.la -lm
libtool: link: /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -pg -g -Wall -pg -g -o iperf3_profile iperf3_profile-main.o iperf3_profile-cjson.o iperf3_profile-iperf_api.o iperf3_profile-iperf_error.o iperf3_profile-iperf_client_api.o iperf3_profile-iperf_locale.o iperf3_profile-iperf_server_api.o iperf3_profile-iperf_tcp.o iperf3_profile-iperf_udp.o iperf3_profile-iperf_sctp.o iperf3_profile-iperf_util.o iperf3_profile-net.o iperf3_profile-tcp_info.o iperf3_profile-tcp_window_size.o iperf3_profile-timer.o iperf3_profile-units.o  ./.libs/libiperf.a -lm
make[2]: Leaving directory `/home/labuser/iperf/iperf-3.1.3/src'
make[1]: Leaving directory `/home/labuser/iperf/iperf-3.1.3/src'
Making all in examples
make[1]: Entering directory `/home/labuser/iperf/iperf-3.1.3/examples'
/opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I. -I../src  -I../src -static -g -static -Wall -MT mic-mic.o -MD -MP -MF .deps/mic-mic.Tpo -c -o mic-mic.o `test -f 'mic.c' || echo './'`mic.c
mv -f .deps/mic-mic.Tpo .deps/mic-mic.Po
/bin/bash ../libtool  --tag=CC   --mode=link /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -g -static -Wall -g  -o mic mic-mic.o ../src/libiperf.la -lm
libtool: link: /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -g -Wall -g -o mic mic-mic.o  ../src/.libs/libiperf.a -lm
/opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -DHAVE_CONFIG_H -I. -I../src  -I../src -static -g -static -Wall -MT mis-mis.o -MD -MP -MF .deps/mis-mis.Tpo -c -o mis-mis.o `test -f 'mis.c' || echo './'`mis.c
mv -f .deps/mis-mis.Tpo .deps/mis-mis.Po
/bin/bash ../libtool  --tag=CC   --mode=link /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -g -static -Wall -g  -o mis mis-mis.o ../src/libiperf.la -lm
libtool: link: /opt/aarch64/aarch64v8-marvell-linux-gnu-5.2.1_i686_20151110/bin/aarch64-linux-gnu-gcc -g -Wall -g -o mis mis-mis.o  ../src/.libs/libiperf.a -lm
make[1]: Leaving directory `/home/labuser/iperf/iperf-3.1.3/examples'
make[1]: Entering directory `/home/labuser/iperf/iperf-3.1.3'
make[1]: Nothing to be done for `all-am'.
make[1]: Leaving directory `/home/labuser/iperf/iperf-3.1.3'

lab-ubuntu64:~/iperf/iperf-3.1.3$ file src/iperf3
src/iperf3: ELF 64-bit LSB  executable, ARM aarch64, version 1 (SYSV), dynamically linked (uses shared libs), for GNU/Linux 3.7.0, BuildID[sha1]=2a0b18b39e152adf8889ebcd05febe588f3b7f06, not stripped


lab-ubuntu64:~/iperf/iperf-3.1.3$ ls src/iperf3 -l
-rwxrwxr-x 1 labuser labuser 166976 Aug  1 13:59 src/iperf3

lab-ubuntu64:~/iperf/iperf-3.1.3$ cp src/iperf3 /srv/for_armv8/iperf/



Although the file command shows iperf3 is dynamically lined, I can run it on A72 core without any extra setup of libraries.