From 8cd50230d9734c11caedddaca5f384a3fef07209 Mon Sep 17 00:00:00 2001 From: Harald Hoyer Date: Mon, 19 Apr 2021 16:34:05 +0200 Subject: [PATCH] ci: use disk ids for TEST 35 ISCSI MULTI Due to parallel probing of the linux kernel `/dev/sd*` can't be used to reliably address a hard disk. This can be seen by the many spurious failures of the dracut CI, where `mdadm` failed with error 524 or tests failed due to the success marker message written to the wrong disk. * don't rely on `/dev/sd*` but use disk ids and `/dev/disk/by-id/ata-disk_` * specify the exact qemu machine architecture `-M q35` needed for the disk ids. A later patch will move this to `run-qemu`, when all tests are converted * due to `-M q35` the interface names have changed from `ens2` -> `enp0s1` and `ens3` -> `enp0s2` --- test/TEST-35-ISCSI-MULTI/99-idesymlinks.rules | 8 -- test/TEST-35-ISCSI-MULTI/client-init.sh | 2 +- .../TEST-35-ISCSI-MULTI/create-client-root.sh | 11 +-- .../TEST-35-ISCSI-MULTI/create-server-root.sh | 16 +-- test/TEST-35-ISCSI-MULTI/server-init.sh | 28 +++--- test/TEST-35-ISCSI-MULTI/test.sh | 99 ++++++++++--------- 6 files changed, 76 insertions(+), 88 deletions(-) delete mode 100644 test/TEST-35-ISCSI-MULTI/99-idesymlinks.rules diff --git a/test/TEST-35-ISCSI-MULTI/99-idesymlinks.rules b/test/TEST-35-ISCSI-MULTI/99-idesymlinks.rules deleted file mode 100644 index d557790a..00000000 --- a/test/TEST-35-ISCSI-MULTI/99-idesymlinks.rules +++ /dev/null @@ -1,8 +0,0 @@ -ACTION=="add|change", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", KERNEL=="hda", SYMLINK+="sda" -ACTION=="add|change", SUBSYSTEM=="block", ENV{DEVTYPE}=="partition", KERNEL=="hda*", SYMLINK+="sda$env{MINOR}" -ACTION=="add|change", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", KERNEL=="hdb", SYMLINK+="sdb" -ACTION=="add|change", SUBSYSTEM=="block", ENV{DEVTYPE}=="partition", KERNEL=="hdb*", SYMLINK+="sdb$env{MINOR}" -ACTION=="add|change", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", KERNEL=="hdc", SYMLINK+="sdc" -ACTION=="add|change", SUBSYSTEM=="block", ENV{DEVTYPE}=="partition", KERNEL=="hdc*", SYMLINK+="sdc$env{MINOR}" -ACTION=="add|change", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", KERNEL=="hdd", SYMLINK+="sdd" -ACTION=="add|change", SUBSYSTEM=="block", ENV{DEVTYPE}=="partition", KERNEL=="hdd*", SYMLINK+="sdd$env{MINOR}" diff --git a/test/TEST-35-ISCSI-MULTI/client-init.sh b/test/TEST-35-ISCSI-MULTI/client-init.sh index bab846c2..965ddc5c 100755 --- a/test/TEST-35-ISCSI-MULTI/client-init.sh +++ b/test/TEST-35-ISCSI-MULTI/client-init.sh @@ -11,7 +11,7 @@ stty sane echo "made it to the rootfs! Powering down." while read -r dev _ fstype opts rest || [ -n "$dev" ]; do [ "$fstype" != "ext3" ] && continue - echo "iscsi-OK $dev $fstype $opts" | dd oflag=direct,dsync of=/dev/sda + echo "iscsi-OK $dev $fstype $opts" | dd oflag=direct,dsync of=/dev/disk/by-id/ata-disk_marker break done < /proc/mounts if getargbool 0 rd.shell; then diff --git a/test/TEST-35-ISCSI-MULTI/create-client-root.sh b/test/TEST-35-ISCSI-MULTI/create-client-root.sh index 04cfaeb7..92f70670 100755 --- a/test/TEST-35-ISCSI-MULTI/create-client-root.sh +++ b/test/TEST-35-ISCSI-MULTI/create-client-root.sh @@ -7,15 +7,12 @@ rm -f -- /etc/lvm/lvm.conf udevadm control --reload udevadm settle -echo "Size of /dev/sdc and /dev/sdd" -blockdev --getsize64 /dev/sdc /dev/sdd - -mkfs.ext3 -j -L singleroot -F /dev/sda \ +mkfs.ext3 -j -L singleroot -F /dev/disk/by-id/ata-disk_singleroot \ && mkdir -p /sysroot \ - && mount /dev/sda /sysroot \ + && mount /dev/disk/by-id/ata-disk_singleroot /sysroot \ && cp -a -t /sysroot /source/* \ && umount /sysroot \ - && mdadm --create /dev/md0 --run --auto=yes --level=stripe --raid-devices=2 /dev/sdc /dev/sdd \ + && mdadm --create /dev/md0 --run --auto=yes --level=stripe --raid-devices=2 /dev/disk/by-id/ata-disk_raid0-1 /dev/disk/by-id/ata-disk_raid0-2 \ && mdadm -W /dev/md0 || : \ && lvm pvcreate -ff -y /dev/md0 \ && lvm vgcreate dracut /dev/md0 \ @@ -26,6 +23,6 @@ mkfs.ext3 -j -L singleroot -F /dev/sda \ && cp -a -t /sysroot /source/* \ && umount /sysroot \ && lvm lvchange -a n /dev/dracut/root \ - && echo "dracut-root-block-created" | dd oflag=direct,dsync of=/dev/sdb + && echo "dracut-root-block-created" | dd oflag=direct,dsync of=/dev/disk/by-id/ata-disk_marker sync poweroff -f diff --git a/test/TEST-35-ISCSI-MULTI/create-server-root.sh b/test/TEST-35-ISCSI-MULTI/create-server-root.sh index 3b02da85..cf55509b 100755 --- a/test/TEST-35-ISCSI-MULTI/create-server-root.sh +++ b/test/TEST-35-ISCSI-MULTI/create-server-root.sh @@ -6,20 +6,14 @@ done rm -f -- /etc/lvm/lvm.conf udevadm control --reload udevadm settle -set -e -# save a partition at the beginning for future flagging purposes -sfdisk /dev/sda << EOF -,1M -, -EOF -udevadm settle -mkfs.ext3 -L dracut /dev/sda2 +ls -al /dev/disk/by-id + +mkfs.ext3 -L dracut /dev/disk/by-id/ata-disk_root mkdir -p /root -mount /dev/sda2 /root +mount /dev/disk/by-id/ata-disk_root /root cp -a -t /root /source/* mkdir -p /root/run umount /root -echo "dracut-root-block-created" | dd oflag=direct,dsync of=/dev/sda1 -sync +echo "dracut-root-block-created" | dd oflag=direct,dsync of=/dev/disk/by-id/ata-disk_marker poweroff -f diff --git a/test/TEST-35-ISCSI-MULTI/server-init.sh b/test/TEST-35-ISCSI-MULTI/server-init.sh index d5372013..420335c9 100755 --- a/test/TEST-35-ISCSI-MULTI/server-init.sh +++ b/test/TEST-35-ISCSI-MULTI/server-init.sh @@ -14,10 +14,6 @@ wait_for_if_link() { while [ $cnt -lt 600 ]; do li=$(ip -o link show dev "$1" 2> /dev/null) [ -n "$li" ] && return 0 - if [[ $2 ]]; then - li=$(ip -o link show dev "$2" 2> /dev/null) - [ -n "$li" ] && return 0 - fi sleep 0.1 cnt=$((cnt + 1)) done @@ -51,17 +47,19 @@ linkup() { wait_for_if_link "$1" 2> /dev/null && ip link set "$1" up 2> /dev/null && wait_for_if_up "$1" 2> /dev/null } -wait_for_if_link eth0 ens2 -wait_for_if_link eth1 ens3 +wait_for_if_link enp0s1 +wait_for_if_link enp0s2 ip addr add 127.0.0.1/8 dev lo ip link set lo up -ip link set dev eth0 name ens2 -ip addr add 192.168.50.1/24 dev ens2 -linkup ens2 -ip link set dev eth1 name ens3 -ip addr add 192.168.51.1/24 dev ens3 -linkup ens3 + +ip link set dev eth0 name enp0s1 +ip addr add 192.168.50.1/24 dev enp0s1 +linkup enp0s1 + +ip link set dev eth1 name enp0s2 +ip addr add 192.168.51.1/24 dev enp0s2 +linkup enp0s2 : > /var/lib/dhcpd/dhcpd.leases chmod 777 /var/lib/dhcpd/dhcpd.leases @@ -71,9 +69,9 @@ tgtd tgtadm --lld iscsi --mode target --op new --tid 1 --targetname iqn.2009-06.dracut:target0 tgtadm --lld iscsi --mode target --op new --tid 2 --targetname iqn.2009-06.dracut:target1 tgtadm --lld iscsi --mode target --op new --tid 3 --targetname iqn.2009-06.dracut:target2 -tgtadm --lld iscsi --mode logicalunit --op new --tid 1 --lun 1 -b /dev/sdb -tgtadm --lld iscsi --mode logicalunit --op new --tid 2 --lun 2 -b /dev/sdc -tgtadm --lld iscsi --mode logicalunit --op new --tid 3 --lun 3 -b /dev/sdd +tgtadm --lld iscsi --mode logicalunit --op new --tid 1 --lun 1 -b /dev/disk/by-id/ata-disk_singleroot +tgtadm --lld iscsi --mode logicalunit --op new --tid 2 --lun 2 -b /dev/disk/by-id/ata-disk_raid0-1 +tgtadm --lld iscsi --mode logicalunit --op new --tid 3 --lun 3 -b /dev/disk/by-id/ata-disk_raid0-2 tgtadm --lld iscsi --mode target --op bind --tid 1 -I 192.168.50.101 tgtadm --lld iscsi --mode target --op bind --tid 2 -I 192.168.51.101 tgtadm --lld iscsi --mode target --op bind --tid 3 -I 192.168.50.101 diff --git a/test/TEST-35-ISCSI-MULTI/test.sh b/test/TEST-35-ISCSI-MULTI/test.sh index 62dd41ec..b52cab15 100755 --- a/test/TEST-35-ISCSI-MULTI/test.sh +++ b/test/TEST-35-ISCSI-MULTI/test.sh @@ -23,16 +23,20 @@ run_server() { # Start server first echo "iSCSI TEST SETUP: Starting DHCP/iSCSI server" - "$testdir"/run-qemu \ - -drive format=raw,index=0,media=disk,file="$TESTDIR"/server.ext3 \ - -drive format=raw,index=1,media=disk,file="$TESTDIR"/root.ext3 \ - -drive format=raw,index=2,media=disk,file="$TESTDIR"/iscsidisk2.img \ - -drive format=raw,index=3,media=disk,file="$TESTDIR"/iscsidisk3.img \ + declare -a disk_args=() + declare -i disk_index=0 + qemu_add_drive_args disk_index disk_args "$TESTDIR"/server.img serverroot 1 + qemu_add_drive_args disk_index disk_args "$TESTDIR"/singleroot.img singleroot + qemu_add_drive_args disk_index disk_args "$TESTDIR"/raid0-1.img raid0-1 + qemu_add_drive_args disk_index disk_args "$TESTDIR"/raid0-2.img raid0-2 + + "$testdir"/run-qemu -M q35 \ + "${disk_args[@]}" \ -serial "${SERIAL:-"file:$TESTDIR/server.log"}" \ -net nic,macaddr=52:54:00:12:34:56,model=e1000 \ -net nic,macaddr=52:54:00:12:34:57,model=e1000 \ -net socket,listen=127.0.0.1:12331 \ - -append "panic=1 systemd.crash_reboot root=/dev/sda2 rootfstype=ext3 rw console=ttyS0,115200n81 selinux=0 $SERVER_DEBUG" \ + -append "panic=1 systemd.crash_reboot root=/dev/disk/by-id/ata-disk_serverroot rootfstype=ext3 rw console=ttyS0,115200n81 selinux=0 $SERVER_DEBUG" \ -initrd "$TESTDIR"/initramfs.server \ -pidfile "$TESTDIR"/server.pid -daemonize || return 1 chmod 644 "$TESTDIR"/server.pid || return 1 @@ -58,16 +62,19 @@ run_client() { shift echo "CLIENT TEST START: $test_name" - dd if=/dev/zero of="$TESTDIR"/client.img bs=1M count=1 + dd if=/dev/zero of="$TESTDIR"/marker.img bs=1MiB count=1 + declare -a disk_args=() + declare -i disk_index=0 + qemu_add_drive_args disk_index disk_args "$TESTDIR"/marker.img marker - "$testdir"/run-qemu \ - -drive format=raw,index=0,media=disk,file="$TESTDIR"/client.img \ + "$testdir"/run-qemu -M q35 \ + "${disk_args[@]}" \ -net nic,macaddr=52:54:00:12:34:00,model=e1000 \ -net nic,macaddr=52:54:00:12:34:01,model=e1000 \ -net socket,connect=127.0.0.1:12331 \ -append "panic=1 systemd.crash_reboot rw rd.auto rd.retry=50 console=ttyS0,115200n81 selinux=0 rd.debug=0 rd.shell=0 $DEBUGFAIL $*" \ -initrd "$TESTDIR"/initramfs.testing - if ! grep -U --binary-files=binary -F -m 1 -q iscsi-OK "$TESTDIR"/client.img; then + if ! grep -U --binary-files=binary -F -m 1 -q iscsi-OK "$TESTDIR"/marker.img; then echo "CLIENT TEST END: $test_name [FAILED - BAD EXIT]" return 1 fi @@ -80,8 +87,8 @@ do_test_run() { initiator=$(iscsi-iname) run_client "netroot=iscsi target1 target2" \ "root=LABEL=sysroot" \ - "ip=192.168.50.101:::255.255.255.0::ens2:off" \ - "ip=192.168.51.101:::255.255.255.0::ens3:off" \ + "ip=192.168.50.101:::255.255.255.0::enp0s1:off" \ + "ip=192.168.51.101:::255.255.255.0::enp0s2:off" \ "netroot=iscsi:192.168.51.1::::iqn.2009-06.dracut:target1" \ "netroot=iscsi:192.168.50.1::::iqn.2009-06.dracut:target2" \ "rd.iscsi.initiator=$initiator" \ @@ -89,8 +96,8 @@ do_test_run() { run_client "netroot=iscsi target1 target2 rd.iscsi.waitnet=0" \ "root=LABEL=sysroot" \ - "ip=192.168.50.101:::255.255.255.0::ens2:off" \ - "ip=192.168.51.101:::255.255.255.0::ens3:off" \ + "ip=192.168.50.101:::255.255.255.0::enp0s1:off" \ + "ip=192.168.51.101:::255.255.255.0::enp0s2:off" \ "netroot=iscsi:192.168.51.1::::iqn.2009-06.dracut:target1" \ "netroot=iscsi:192.168.50.1::::iqn.2009-06.dracut:target2" \ "rd.iscsi.firmware" \ @@ -100,8 +107,8 @@ do_test_run() { run_client "netroot=iscsi target1 target2 rd.iscsi.waitnet=0 rd.iscsi.testroute=0" \ "root=LABEL=sysroot" \ - "ip=192.168.50.101:::255.255.255.0::ens2:off" \ - "ip=192.168.51.101:::255.255.255.0::ens3:off" \ + "ip=192.168.50.101:::255.255.255.0::enp0s1:off" \ + "ip=192.168.51.101:::255.255.255.0::enp0s2:off" \ "netroot=iscsi:192.168.51.1::::iqn.2009-06.dracut:target1" \ "netroot=iscsi:192.168.50.1::::iqn.2009-06.dracut:target2" \ "rd.iscsi.firmware" \ @@ -111,8 +118,8 @@ do_test_run() { run_client "netroot=iscsi target1 target2 rd.iscsi.waitnet=0 rd.iscsi.testroute=0 default GW" \ "root=LABEL=sysroot" \ - "ip=192.168.50.101::192.168.50.1:255.255.255.0::ens2:off" \ - "ip=192.168.51.101::192.168.51.1:255.255.255.0::ens3:off" \ + "ip=192.168.50.101::192.168.50.1:255.255.255.0::enp0s1:off" \ + "ip=192.168.51.101::192.168.51.1:255.255.255.0::enp0s2:off" \ "netroot=iscsi:192.168.51.1::::iqn.2009-06.dracut:target1" \ "netroot=iscsi:192.168.50.1::::iqn.2009-06.dracut:target2" \ "rd.iscsi.firmware" \ @@ -144,14 +151,6 @@ test_setup() { return 1 fi - # Create the blank file to use as a root filesystem - rm -f "$TESTDIR"/root.ext3 - dd if=/dev/zero of="$TESTDIR"/root.ext3 bs=4096 count=$((200 * 256)) - rm -f "$TESTDIR"/iscsidisk2.img - dd if=/dev/zero of="$TESTDIR"/iscsidisk2.img bs=4096 count=$((100 * 256)) - rm -f "$TESTDIR"/iscsidisk3.img - dd if=/dev/zero of="$TESTDIR"/iscsidisk3.img bs=4096 count=$((100 * 256)) - kernel=$KVERSION # Create what will eventually be our root filesystem onto an overlay rm -rf -- "$TESTDIR"/overlay @@ -197,7 +196,6 @@ test_setup() { inst_multiple sfdisk mkfs.ext3 poweroff cp umount setsid dd sync blockdev inst_hook initqueue 01 ./create-client-root.sh inst_hook initqueue/finished 01 ./finished-false.sh - inst_simple ./99-idesymlinks.rules /etc/udev/rules.d/99-idesymlinks.rules ) # create an initramfs that will create the target root filesystem. @@ -210,27 +208,29 @@ test_setup() { -f "$TESTDIR"/initramfs.makeroot "$KVERSION" || return 1 rm -rf -- "$TESTDIR"/overlay - # Need this so kvm-qemu will boot (needs non-/dev/zero local disk) - if ! dd if=/dev/zero of="$TESTDIR"/client.img bs=1M count=1; then - echo "Unable to make client sdb image" 1>&2 - return 1 - fi + dd if=/dev/zero of="$TESTDIR"/marker.img bs=1MiB count=1 + dd if=/dev/zero of="$TESTDIR"/singleroot.img bs=1MiB count=200 + dd if=/dev/zero of="$TESTDIR"/raid0-1.img bs=1MiB count=100 + dd if=/dev/zero of="$TESTDIR"/raid0-2.img bs=1MiB count=100 + + declare -a disk_args=() + declare -i disk_index=0 + qemu_add_drive_args disk_index disk_args "$TESTDIR"/marker.img marker + qemu_add_drive_args disk_index disk_args "$TESTDIR"/singleroot.img singleroot + qemu_add_drive_args disk_index disk_args "$TESTDIR"/raid0-1.img raid0-1 + qemu_add_drive_args disk_index disk_args "$TESTDIR"/raid0-2.img raid0-2 + # Invoke KVM and/or QEMU to actually create the target filesystem. - "$testdir"/run-qemu \ - -drive format=raw,index=0,media=disk,file="$TESTDIR"/root.ext3 \ - -drive format=raw,index=1,media=disk,file="$TESTDIR"/client.img \ - -drive format=raw,index=2,media=disk,file="$TESTDIR"/iscsidisk2.img \ - -drive format=raw,index=3,media=disk,file="$TESTDIR"/iscsidisk3.img \ + "$testdir"/run-qemu -M q35 \ + "${disk_args[@]}" \ -append "root=/dev/fakeroot rw rootfstype=ext3 quiet console=ttyS0,115200n81 selinux=0" \ -initrd "$TESTDIR"/initramfs.makeroot || return 1 - grep -U --binary-files=binary -F -m 1 -q dracut-root-block-created "$TESTDIR"/client.img || return 1 - rm -- "$TESTDIR"/client.img + grep -U --binary-files=binary -F -m 1 -q dracut-root-block-created "$TESTDIR"/marker.img || return 1 + rm -- "$TESTDIR"/marker.img # Make server root echo "MAKE SERVER ROOT" - dd if=/dev/zero of="$TESTDIR"/server.ext3 bs=1M count=60 - export kernel=$KVERSION rm -rf -- "$TESTDIR"/overlay ( @@ -279,7 +279,6 @@ test_setup() { inst_multiple sfdisk mkfs.ext3 poweroff cp umount sync dd inst_hook initqueue 01 ./create-server-root.sh inst_hook initqueue/finished 01 ./finished-false.sh - inst_simple ./99-idesymlinks.rules /etc/udev/rules.d/99-idesymlinks.rules ) # create an initramfs that will create the target root filesystem. @@ -292,12 +291,21 @@ test_setup() { --no-hostonly-cmdline -N \ -f "$TESTDIR"/initramfs.makeroot "$KVERSION" || return 1 + dd if=/dev/zero of="$TESTDIR"/marker.img bs=1MiB count=1 + dd if=/dev/zero of="$TESTDIR"/server.img bs=1MiB count=60 + declare -a disk_args=() + # shellcheck disable=SC2034 + declare -i disk_index=0 + qemu_add_drive_args disk_index disk_args "$TESTDIR"/marker.img marker + qemu_add_drive_args disk_index disk_args "$TESTDIR"/server.img root + # Invoke KVM and/or QEMU to actually create the target filesystem. - "$testdir"/run-qemu \ - -drive format=raw,index=0,media=disk,file="$TESTDIR"/server.ext3 \ + "$testdir"/run-qemu -M q35 \ + "${disk_args[@]}" \ -append "root=/dev/dracut/root rw rootfstype=ext3 quiet console=ttyS0,115200n81 selinux=0" \ -initrd "$TESTDIR"/initramfs.makeroot || return 1 - grep -U --binary-files=binary -F -m 1 -q dracut-root-block-created "$TESTDIR"/server.ext3 || return 1 + grep -U --binary-files=binary -F -m 1 -q dracut-root-block-created "$TESTDIR"/marker.img || return 1 + rm -- "$TESTDIR"/marker.img rm -rf -- "$TESTDIR"/overlay # Make an overlay with needed tools for the test harness @@ -309,7 +317,6 @@ test_setup() { inst_multiple poweroff shutdown inst_hook shutdown-emergency 000 ./hard-off.sh inst_hook emergency 000 ./hard-off.sh - inst_simple ./99-idesymlinks.rules /etc/udev/rules.d/99-idesymlinks.rules ) # Make server's dracut image