Project

General

Profile

Emulation Debug

Create an Image

qemu-img create debian.img 8G
sudo mkfs.ext4 -O '^has_journal' -F debian.img
sudo mkdir debian
sudo mount -o loop debian.img debian
sudo debootstrap buster debian
sudo chroot debian apt update
sudo chroot debian apt install --no-install-recommends build-essential vim openssh-server less \
 pkg-config libnl-3-dev libnl-genl-3-dev libcap-dev tcpdump \
 trace-cmd flex bison libelf-dev libdw-dev binutils-dev libunwind-dev libssl-dev libslang2-dev liblzma-dev

sudo mkdir debian/root/.ssh/
ssh-add -L | sudo tee debian/root/.ssh/authorized_keys

sudo mkdir debian/host
sudo sh -c 'cat > debian/etc/fstab  << EOF
host            /host   9p      trans=virtio,version=9p2000.L 0 0
EOF'

sudo sh -c 'cat > debian/etc/rc.local << "EOF" 
#!/bin/sh -e

MAC_PART="$(ip link show enp0s1 | awk "/ether/ {print \$2}"| sed -e "s/.*://" -e "s/[\\n\\ ].*//"|awk "{print (\"0x\"\$1)*1 }")" 
IP_PART="$(echo $MAC_PART|awk "{ print \$1+50 }")" 
NODE_NR="$(echo $MAC_PART|awk "{ printf(\"%02d\", \$1) }")" 
ip addr add 192.168.251.${IP_PART}/24 dev enp0s1
ip link set up dev enp0s1
hostname "node"$NODE_NR
ip link set up dev lo
[ ! -x /host/test-init.sh ] || /host/test-init.sh
exit 0
EOF'
sudo chmod a+x debian/etc/rc.local

sudo sed -i 's/^root:[^:]*:/root::/' debian/etc/shadow

## optionally: allow ssh logins without passwords
# sudo sed -i 's/^#PermitRootLogin.*/PermitRootLogin yes/' debian/etc/ssh/sshd_config
# sudo sed -i 's/^#PermitEmptyPasswords.*/PermitEmptyPasswords yes/' debian/etc/ssh/sshd_config
# sudo sed -i 's/^UsePAM.*/UsePAM no/' debian/etc/ssh/sshd_config

## optionally: enable autologin for user root
#sudo mkdir debian/etc/systemd/system/serial-getty@ttyS0.service.d/
#sudo sh -c 'cat > debian/etc/systemd/system/serial-getty@ttyS0.service.d/autologin.conf  << EOF
#[Service]
#ExecStart=
#ExecStart=-/sbin/agetty --autologin root -s %I 115200,38400,9600 vt102
#EOF'

sudo sh -c 'echo '\''PATH="/host/batctl/:$PATH"'\'' >> debian/etc/profile'
sudo rm debian/var/cache/apt/archives/*.deb
sudo rm debian/var/lib/apt/lists/*
sudo e4defrag -v debian/
sudo umount debian
sudo zerofree -v debian.img

## optionally: convert image to qcow2
#sudo qemu-img convert -c -f raw -O qcow2 debian.img debian.qcow2
#sudo mv debian.qcow2 debian.img

Kernel compile

Any recent kernel can be used for the setup. The build result arch/x86/boot/bzImage has to be copied to the same directory as the debian.img

# make sure that libelf-dev is installed or module build will fail with something like "No rule to make target 'net/batman-adv/bat_algo.o'" 

git clone git://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git
cd linux-next

make allnoconfig
cat >> .config << EOF

# small configuration
CONFIG_SMP=y
CONFIG_EMBEDDED=n
# CONFIG_EXPERT is not set
CONFIG_MODULES=y
CONFIG_MODULE_UNLOAD=y
CONFIG_MODVERSIONS=y
CONFIG_MODULE_SRCVERSION_ALL=y
CONFIG_64BIT=y
CONFIG_X86_VSYSCALL_EMULATION=n
CONFIG_IA32_EMULATION=n
CONFIG_VOP_BUS=y
CONFIG_VOP=y
CONFIG_HW_RANDOM_VIRTIO=y
CONFIG_NET_9P_VIRTIO=y
CONFIG_VIRTIO_MENU=y
CONFIG_SCSI_VIRTIO=y
CONFIG_VIRTIO_BALLOON=y
CONFIG_VIRTIO_BLK=y
CONFIG_VIRTIO_CONSOLE=y
CONFIG_VIRTIO_INPUT=y
CONFIG_VIRTIO_NET=y
CONFIG_VIRTIO_PCI=y
CONFIG_VIRTIO_PCI_LEGACY=y
CONFIG_VIRTIO_BALLOON=y
CONFIG_VIRTIO_BLK_SCSI=y
CONFIG_VIRTIO_INPUT=y
CONFIG_VIRTIO_MMIO=y
CONFIG_VIRTIO_MMIO_CMDLINE_DEVICES=y
CONFIG_RPMSG_VIRTIO=y
CONFIG_VSOCKETS=y
CONFIG_VIRTIO_VSOCKETS=y
CONFIG_DRM=y
CONFIG_DRM_VIRTIO_GPU=y
CONFIG_CAIF=y
CONFIG_CAIF_VIRTIO=y
CONFIG_CRC16=y
CONFIG_LIBCRC32C=y
CONFIG_CRYPTO_SHA512=y
CONFIG_NET=y
CONFIG_INET=y
CONFIG_DEBUG_FS=y
CONFIG_IPV6=y
CONFIG_BRIDGE=y
CONFIG_VLAN_8021Q=y
CONFIG_WIRELESS=n
CONFIG_NET_9P=y
CONFIG_NETWORK_FILESYSTEMS=y
CONFIG_9P_FS=y
CONFIG_9P_FS_POSIX_ACL=y
CONFIG_9P_FS_SECURITY=y
CONFIG_BLOCK=y
CONFIG_BLK_DEV=y
CONFIG_EXT4_FS=y
CONFIG_EXT4_USE_FOR_EXT23=y
CONFIG_TTY=y
CONFIG_SERIAL_8250=y
CONFIG_SERIAL_8250_CONSOLE=y
CONFIG_HW_RANDOM=y
CONFIG_VHOST_RING=y
CONFIG_GENERIC_ALLOCATOR=y
CONFIG_SCSI_LOWLEVEL=y
CONFIG_SCSI=y
CONFIG_NETDEVICES=y
CONFIG_NET_CORE=y
CONFIG_DEVTMPFS=y
CONFIG_HYPERVISOR_GUEST=y
CONFIG_PARAVIRT=y
CONFIG_KVM_GUEST=y
CONFIG_BINFMT_ELF=y
CONFIG_BINFMT_SCRIPT=y
CONFIG_BINFMT_MISC=y
CONFIG_PCI=y
CONFIG_SYSVIPC=y
CONFIG_POSIX_MQUEUE=y
CONFIG_CROSS_MEMORY_ATTACH=y
CONFIG_UNIX=y
CONFIG_TMPFS=y
CONFIG_CGROUPS=y
CONFIG_BLK_CGROUP=y
CONFIG_CGROUP_CPUACCT=y
CONFIG_CGROUP_DEVICE=y
CONFIG_CGROUP_FREEZER=y
CONFIG_CGROUP_HUGETLB=y
CONFIG_CGROUP_NET_CLASSID=y
CONFIG_CGROUP_NET_PRIO=y
CONFIG_CGROUP_PERF=y
CONFIG_CGROUP_SCHED=y
CONFIG_DEVPTS_MULTIPLE_INSTANCES=y
CONFIG_INOTIFY_USER=y
CONFIG_FHANDLE=y
CONFIG_E1000=y
CONFIG_CPU_FREQ=y
CONFIG_CONFIG_X86_ACPI_CPUFREQ=y
CONFIG_CPU_FREQ_GOV_ONDEMAND=y
CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND=y
CONFIG_CFG80211=y
CONFIG_PARAVIRT_SPINLOCKS=y
CONFIG_DUMMY=y
CONFIG_PACKET=y
CONFIG_VETH=y
CONFIG_IP_MULTICAST=y
CONFIG_NET_IPGRE_DEMUX=y
CONFIG_NET_IP_TUNNEL=y
CONFIG_NET_IPGRE=y
CONFIG_NET_IPGRE_BROADCAST=y
# CONFIG_LEGACY_PTYS is not set

# makes boot a lot slower but required for shutdown
CONFIG_ACPI=y

#debug stuff
CONFIG_CC_STACKPROTECTOR_STRONG=y
CONFIG_LOCKUP_DETECTOR=y
CONFIG_DETECT_HUNG_TASK=y
CONFIG_SCHED_STACK_END_CHECK=y
CONFIG_DEBUG_RT_MUTEXES=y
CONFIG_DEBUG_SPINLOCK=y
CONFIG_DEBUG_MUTEXES=y
CONFIG_PROVE_LOCKING=y
CONFIG_LOCK_STAT=y
CONFIG_DEBUG_LOCKDEP=y
CONFIG_DEBUG_ATOMIC_SLEEP=y
CONFIG_DEBUG_LIST=y
CONFIG_DEBUG_PI_LIST=y
CONFIG_DEBUG_SG=y
CONFIG_DEBUG_NOTIFIERS=y
CONFIG_PROVE_RCU_REPEATEDLY=y
CONFIG_SPARSE_RCU_POINTER=y
CONFIG_DEBUG_STRICT_USER_COPY_CHECKS=y
CONFIG_X86_VERBOSE_BOOTUP=y
CONFIG_DEBUG_RODATA=y
CONFIG_DEBUG_RODATA_TEST=n
CONFIG_DEBUG_SET_MODULE_RONX=y
CONFIG_PAGE_EXTENSION=y
CONFIG_DEBUG_PAGEALLOC=y
CONFIG_DEBUG_OBJECTS=y
CONFIG_DEBUG_OBJECTS_FREE=y
CONFIG_DEBUG_OBJECTS_TIMERS=y
CONFIG_DEBUG_OBJECTS_WORK=y
CONFIG_DEBUG_OBJECTS_RCU_HEAD=y
CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER=y
CONFIG_DEBUG_KMEMLEAK=y
CONFIG_DEBUG_KMEMLEAK_EARLY_LOG_SIZE=8000
CONFIG_DEBUG_STACK_USAGE=y
CONFIG_DEBUG_STACKOVERFLOW=y
CONFIG_DEBUG_INFO=y
CONFIG_DEBUG_INFO_DWARF4=y
CONFIG_GDB_SCRIPTS=y
CONFIG_READABLE_ASM=y
CONFIG_STACK_VALIDATION=y
CONFIG_WQ_WATCHDOG=y
CONFIG_DEBUG_KOBJECT_RELEASE=y
CONFIG_DEBUG_WQ_FORCE_RR_CPU=y
CONFIG_OPTIMIZE_INLINING=y
CONFIG_ENABLE_MUST_CHECK=y
CONFIG_ENABLE_WARN_DEPRECATED=y
CONFIG_DEBUG_SECTION_MISMATCH=y
CONFIG_UNWINDER_ORC=y
CONFIG_FTRACE=y
CONFIG_FUNCTION_TRACER=y
CONFIG_FUNCTION_GRAPH_TRACER=y
CONFIG_FTRACE_SYSCALLS=y
CONFIG_TRACER_SNAPSHOT=y
CONFIG_TRACER_SNAPSHOT_PER_CPU_SWAP=y
CONFIG_STACK_TRACER=y
CONFIG_UPROBE_EVENTS=y
CONFIG_DYNAMIC_FTRACE=y
CONFIG_FUNCTION_PROFILER=y
CONFIG_HIST_TRIGGERS=y

# for GCC 5+
CONFIG_KASAN=y
CONFIG_KASAN_INLINE=y
CONFIG_UBSAN_SANITIZE_ALL=y
CONFIG_UBSAN=y
CONFIG_UBSAN_NULL=y
EOF
make olddefconfig

cat >> .config << EOF
# allow to use unsigned regdb with hwsim
CONFIG_EXPERT=y
CONFIG_CFG80211_CERTIFICATION_ONUS=y
# CONFIG_CFG80211_REQUIRE_SIGNED_REGDB is not set
EOF
make olddefconfig

make all -j$(nproc || echo 1)

Start of the simple environment

The two node environment must be started inside a screen session. The hub (bridge with 5 tap devices) has to be started first to have a simple network. A more complex network setup can be on the page Emulation

The ETH in hub.sh has to be changed to the real interface which provides internet-connectivity

cat > hub.sh << "EOF" 
#! /bin/sh
USER="$(whoami)" 
BRIDGE=br-qemu

sudo ip link add "${BRIDGE}" type bridge
for i in `seq 1 5`; do
        sudo ip tuntap add dev tap$i mode tap user "$USER" 
        sudo ip link set tap$i up
        sudo ip link set tap$i master "${BRIDGE}" 
done

sudo ip link set "${BRIDGE}" up
sudo ip addr replace 192.168.251.1/24 dev br-qemu
EOF

chmod +x hub.sh

The SHARED_PATH in run.sh has to be changed to a valid path which is used to share the precompiled batman-adv.ko and other tools

cat > run.sh << "EOF" 
#! /bin/sh

if [ -z "${STY}" ]; then
        echo "must be started inside a screen session" >&2
        exit 1
fi

SHARED_PATH=/home/sven/tmp/qemu-batman/

for i in $(seq 1 5); do
        qemu-img create -b debian.img -f qcow2 root.cow$i
        normalized_id="$(echo "$i"|awk '{ printf "%02d\n",$1 }')" 
        twodigit_id="$(echo $i|awk '{ printf "%02X", $1 }')" 
        screen qemu-system-x86_64 -enable-kvm -name "debian${i}" \
                -kernel linux-next/arch/x86/boot/bzImage -append "root=/dev/vda rw console=ttyS0 nokaslr" \
                -display none -no-user-config -nodefaults \
                -m 512 -device virtio-balloon \
                -cpu host -smp 2 -machine q35,accel=kvm,usb=off,dump-guest-core=off \
                -drive file=root.cow$i,if=virtio,cache=unsafe \
                -nic tap,ifname=tap$i,script=no,model=virtio-net-pci,mac=02:ba:de:af:fe:"${twodigit_id}" \
                -nic user,model=virtio-net-pci,mac=06:ba:de:af:fe:"${twodigit_id}" \
                -virtfs local,path="${SHARED_PATH}",security_model=none,mount_tag=host \
                -gdb tcp:127.0.0.1:$((23000+$i)) \
                -device virtio-rng-pci \
                -serial mon:stdio
        sleep 1
done
EOF

chmod +x run.sh 

The test-init.sh script can be used to automatically initialize the testsetup during boot:

cat > test-init.sh << "EOF" 
#! /bin/sh

set -e
export PATH="/host/batctl/:$PATH" 

# ip link add dummy0 type dummy
ip link set up dummy0

rmmod batman-adv || true
insmod /host/batman-adv/net/batman-adv/batman-adv.ko
batctl routing_algo BATMAN_IV
/host/batctl/batctl if add dummy0
/host/batctl/batctl it 5000
/host/batctl/batctl if add enp0s1
ip link set up dev enp0s1
ip link set up dev bat0
EOF

chmod +x test-init.sh

Everything can then be started up inside a screen session

screen
./hub.sh
./run.sh

Building the batman-adv module

The kernel module can be build outside the virtual environment and shared over the 9p mount. The path to the kernel sources have to be provided to the make process

make KERNELPATH=/home/batman/linux-next

The kernel module can also be compiled for better readability for the calltraces:

make EXTRA_CFLAGS="-fno-inline -Og -fno-optimize-sibling-calls" KERNELPATH=/home/sven/tmp/qemu-batman/linux-next V=1

View traffic via wireshark from virtual machine

On host system

mkfifo remote-dump
ssh root@192.168.251.51 'tcpdump -i enp3s0 -s 0 -U -n -w - "port not 22"' > remote-dump
wireshark -k -i remote-dump

Connect to gluon VXLAN

Gluon supports meshing over VXLAN. The gluon node just has to enable Mesh-on-WAN or Mesh-on-LAN and then it is possible to connect our virtual machine with the WAN/LAN port of the machine. Let us assume that the host running the Qemu instances is using interface enp8s0 for this connection and that the qemu instances are all connected to bridge br-qemu</code.

The first information we have to get from from the gluon node is the VXLAN id. It can be queried on the gluon node via:

lua -e 'print(tonumber(require("gluon.util").domain_seed_bytes("gluon-mesh-vxlan", 3), 16))'

We must then create a vxlan interface on top of our normal ethernet interface, make sure that the ethernet interface is using a EUI64 based IPv6 link local address and insert the new interface in our bridge

#! /bin/sh

BRIDGE=br-qemu
ETH=enp8s0
VXLAN=vx_mesh_lan
# calculated on gluon node
VXLAN_ID=12094920

xor2() {
        echo -n "${1:0:1}" 
        echo -n "${1:1:1}" | tr '0123456789abcdef' '23016745ab89efcd'
}

interface_linklocal() {
        local macaddr="$(cat /sys/class/net/"${ETH}"/address)" 
        local oldIFS="$IFS"; IFS=':'; set -- $macaddr; IFS="$oldIFS" 

        echo "fe80::$(xor2 "$1")$2:$3ff:fe$4:$5$6" 
}

sudo ip addr add "$(interface_linklocal)"/64 dev "$ETH" 
sudo ip link del "${VXLAN}" 
sudo ip -6 link add "${VXLAN}" type vxlan \
   id "${VXLAN_ID}" \
   dstport 4789 \
   local "$(interface_linklocal)" \
   group ff02::15c \
   dev "${ETH}" \
   udp6zerocsumtx udp6zerocsumrx \
   ttl 1

sudo ip link set "${VXLAN}" up master "${BRIDGE}" 

Using GDB

The instances are listening on 127.0.0.1 TCP port 23000 + instance_no. We will use in the following example instance 1. The gdb debugger can be started from the linux source directory and all lx-* helpers will automatically be loaded.

The debugging session with gdb can be started from the linux-next directory:

$ gdb -iex "set auto-load safe-path scripts/gdb/" -ex 'target remote 127.0.0.1:23001' -ex c  ./vmlinux

The module can now be loaded in the qemu instance. After that, we have to reload the symbol information via lx-symbol and can set any kind of breakpoints on the batman-adv module:

^C
Thread 1 received signal SIGINT, Interrupt.
default_idle () at arch/x86/kernel/process.c:581
581             trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
(gdb) lx-symbols /home/sven/tmp/qemu-batman/batman-adv/net/batman-adv/
loading vmlinux
scanning for modules in /home/sven/tmp/qemu-batman/batman-adv/net/batman-adv/
scanning for modules in /home/sven/tmp/qemu-batman/linux-next
loading @0xffffffffa0000000: /home/sven/tmp/qemu-batman/batman-adv/net/batman-adv//batman-adv.ko
(gdb) b batadv_iv_send_outstanding_bat_ogm_packet
Breakpoint 1 at 0xffffffffa0005d60: file /home/sven/tmp/qemu-batman/batman-adv/net/batman-adv/bat_iv_ogm.c, line 1692.
(gdb) c

It is also possible to evaluate data structures in the gdb commandline using small python code blocks. To get for example the name of all devices which batman-adv knows about and the name of the batman-adv interface they belong to:

python
import linux.lists
from linux.utils import CachedType

struct_batadv_hard_iface = CachedType('struct batadv_hard_iface').get_type().pointer()

for node in linux.lists.list_for_each_entry(gdb.parse_and_eval("batadv_hardif_list"), struct_batadv_hard_iface, 'list'):
    hardif = node['net_dev']['name'].string()
    softif = node['soft_iface']['name'].string() if node['soft_iface'] else "none" 
    gdb.write("hardif {} belongs to {}\n".format(hardif, softif))
end