Hi, I'm having trouble after passing through a NVMe drive (used to passthrough a SATA drive instead) to my W10 VM.
Asus Z270-A Prime
Nvidia 1030
Intel i7-6700K
NVMe: Samsung Evo 970 Plus 500 GB
Some debugging info:
VT-D is on
The NVMe drive has a bare-metal installed version of W10 Pro (same as the SATA drive prior)
I followed SpadeInvaderOne's instructions to set the function 0x1 properly for the GPU.
Here's the template:
<?xml version='1.0' encoding='UTF-8'?>
<domain type='kvm'>
<name>Windows 10 Gaming</name>
<uuid>4395501d-a8b5-1d47-2da6-8e76fad67cd1</uuid>
<metadata>
<vmtemplate xmlns="unraid" name="Windows 10" icon="windows.png" os="windows10"/>
</metadata>
<memory unit='KiB'>16777216</memory>
<currentMemory unit='KiB'>16777216</currentMemory>
<memoryBacking>
<nosharepages/>
</memoryBacking>
<vcpu placement='static'>6</vcpu>
<cputune>
<vcpupin vcpu='0' cpuset='1'/>
<vcpupin vcpu='1' cpuset='5'/>
<vcpupin vcpu='2' cpuset='2'/>
<vcpupin vcpu='3' cpuset='6'/>
<vcpupin vcpu='4' cpuset='3'/>
<vcpupin vcpu='5' cpuset='7'/>
</cputune>
<os>
<type arch='x86_64' machine='pc-i440fx-5.1'>hvm</type>
<loader readonly='yes' type='pflash'>/usr/share/qemu/ovmf-x64/OVMF_CODE-pure-efi.fd</loader>
<nvram>/etc/libvirt/qemu/nvram/4395501d-a8b5-1d47-2da6-8e76fad67cd1_VARS-pure-efi.fd</nvram>
<boot dev='hd'/>
</os>
<features>
<acpi/>
<apic/>
<hyperv>
<relaxed state='on'/>
<vapic state='on'/>
<spinlocks state='on' retries='8191'/>
<vendor_id state='on' value='none'/>
</hyperv>
</features>
<cpu mode='host-passthrough' check='none' migratable='on'>
<topology sockets='1' dies='1' cores='3' threads='2'/>
<cache mode='passthrough'/>
</cpu>
<clock offset='localtime'>
<timer name='hypervclock' present='yes'/>
<timer name='hpet' present='no'/>
</clock>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>restart</on_crash>
<devices>
<emulator>/usr/local/sbin/qemu</emulator>
<controller type='pci' index='0' model='pci-root'/>
<controller type='virtio-serial' index='0'>
<address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
</controller>
<controller type='usb' index='0' model='qemu-xhci' ports='15'>
<address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0'/>
</controller>
<interface type='bridge'>
<mac address='52:54:00:1a:f3:1b'/>
<source bridge='br0'/>
<model type='virtio'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
</interface>
<serial type='pty'>
<target type='isa-serial' port='0'>
<model name='isa-serial'/>
</target>
</serial>
<console type='pty'>
<target type='serial' port='0'/>
</console>
<channel type='unix'>
<target type='virtio' name='org.qemu.guest_agent.0'/>
<address type='virtio-serial' controller='0' bus='0' port='1'/>
</channel>
<input type='tablet' bus='usb'>
<address type='usb' bus='0' port='2'/>
</input>
<input type='mouse' bus='ps2'/>
<input type='keyboard' bus='ps2'/>
<hostdev mode='subsystem' type='pci' managed='yes'>
<driver name='vfio'/>
<source>
<address domain='0x0000' bus='0x02' slot='0x00' function='0x0'/>
</source>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
</hostdev>
<hostdev mode='subsystem' type='pci' managed='yes'>
<driver name='vfio'/>
<source>
<address domain='0x0000' bus='0x02' slot='0x00' function='0x1'/>
</source>
<address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0' multifunction='on'/>
</hostdev>
<hostdev mode='subsystem' type='pci' managed='yes'>
<driver name='vfio'/>
<source>
<address domain='0x0000' bus='0x06' slot='0x00' function='0x0'/>
</source>
<address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x1'/>
</hostdev>
<hostdev mode='subsystem' type='pci' managed='yes'>
<driver name='vfio'/>
<source>
<address domain='0x0000' bus='0x07' slot='0x00' function='0x0'/>
</source>
<address type='pci' domain='0x0000' bus='0x00' slot='0x08' function='0x0'/>
</hostdev>
<hostdev mode='subsystem' type='usb' managed='no'>
<source>
<vendor id='0x046d'/>
<product id='0xc52b'/>
</source>
<address type='usb' bus='0' port='1'/>
</hostdev>
<memballoon model='none'/>
</devices>
</domain>
Here's the error log:
ErrorWarningSystemArrayLogin
-blockdev '{"driver":"file","filename":"/usr/share/qemu/ovmf-x64/OVMF_CODE-pure-efi.fd","node-name":"libvirt-pflash0-storage","auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-pflash0-format","read-only":true,"driver":"raw","file":"libvirt-pflash0-storage"}' \
-blockdev '{"driver":"file","filename":"/etc/libvirt/qemu/nvram/4395501d-a8b5-1d47-2da6-8e76fad67cd1_VARS-pure-efi.fd","node-name":"libvirt-pflash1-storage","auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-pflash1-format","read-only":false,"driver":"raw","file":"libvirt-pflash1-storage"}' \
-machine pc-i440fx-5.1,accel=kvm,usb=off,dump-guest-core=off,mem-merge=off,pflash0=libvirt-pflash0-format,pflash1=libvirt-pflash1-format \
-cpu host,migratable=on,hv-time,hv-relaxed,hv-vapic,hv-spinlocks=0x1fff,hv-vendor-id=none,host-cache-info=on,l3-cache=off \
-m 16384 \
-overcommit mem-lock=off \
-smp 6,sockets=1,dies=1,cores=3,threads=2 \
-uuid 4395501d-a8b5-1d47-2da6-8e76fad67cd1 \
-display none \
-no-user-config \
-nodefaults \
-chardev socket,id=charmonitor,fd=32,server,nowait \
-mon chardev=charmonitor,id=monitor,mode=control \
-rtc base=localtime \
-no-hpet \
-no-shutdown \
-boot strict=on \
-device qemu-xhci,p2=15,p3=15,id=usb,bus=pci.0,addr=0x7 \
-device virtio-serial-pci,id=virtio-serial0,bus=pci.0,addr=0x4 \
-netdev tap,fd=34,id=hostnet0,vhost=on,vhostfd=35 \
-device virtio-net-pci,netdev=hostnet0,id=net0,mac=52:54:00:1a:f3:1b,bus=pci.0,addr=0x2 \
-chardev pty,id=charserial0 \
-device isa-serial,chardev=charserial0,id=serial0 \
-chardev socket,id=charchannel0,fd=36,server,nowait \
-device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=org.qemu.guest_agent.0 \
-device usb-tablet,id=input0,bus=usb.0,port=2 \
-device vfio-pci,host=0000:02:00.0,id=hostdev0,bus=pci.0,addr=0x3 \
-device vfio-pci,host=0000:02:00.1,id=hostdev1,bus=pci.0,multifunction=on,addr=0x5 \
-device vfio-pci,host=0000:06:00.0,id=hostdev2,bus=pci.0,addr=0x5.0x1 \
-device vfio-pci,host=0000:07:00.0,id=hostdev3,bus=pci.0,addr=0x8 \
-device usb-host,hostbus=1,hostaddr=4,id=hostdev4,bus=usb.0,port=1 \
-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \
-msg timestamp=on
2022-02-23 23:27:56.152+0000: Domain id=5 is tainted: high-privileges
2022-02-23 23:27:56.152+0000: Domain id=5 is tainted: host-cpu
char device redirected to /dev/pts/0 (label charserial0)
2022-02-23T23:27:57.688818Z qemu-system-x86_64: -device vfio-pci,host=0000:02:00.0,id=hostdev0,bus=pci.0,addr=0x3: Failed to mmap 0000:02:00.0 BAR 3. Performance may be slow
2022-02-23T23:28:01.135519Z qemu-system-x86_64: vfio_err_notifier_handler(0000:06:00.0) Unrecoverable error detected. Please collect any data possible and then kill the guest
IOMMU:
IOMMU group 0: [8086:191f] 00:00.0 Host bridge: Intel Corporation Xeon E3-1200 v5/E3-1500 v5/6th Gen Core Processor Host Bridge/DRAM Registers (rev 07)
IOMMU group 1: [8086:1901] 00:01.0 PCI bridge: Intel Corporation 6th-10th Gen Core Processor PCIe Controller (x16) (rev 07)
IOMMU group 2: [8086:1905] 00:01.1 PCI bridge: Intel Corporation Xeon E3-1200 v5/E3-1500 v5/6th Gen Core Processor PCIe Controller (x8) (rev 07)
IOMMU group 3: [8086:a2af] 00:14.0 USB controller: Intel Corporation 200 Series/Z370 Chipset Family USB 3.0 xHCI Controller
Bus 001 Device 001: ID 1d6b:0002 Linux Foundation 2.0 root hub
Bus 001 Device 002: ID 30de:6544 Kingston DataTraveler 2.0
Bus 001 Device 003: ID 0764:0501 Cyber Power System, Inc. CP1500 AVR UPS
Bus 001 Device 004: ID 046d:c52b Logitech, Inc. Unifying Receiver
Bus 002 Device 001: ID 1d6b:0003 Linux Foundation 3.0 root hub
IOMMU group 4: [8086:a2ba] 00:16.0 Communication controller: Intel Corporation 200 Series PCH CSME HECI #1
IOMMU group 5: [8086:a282] 00:17.0 SATA controller: Intel Corporation 200 Series PCH SATA controller [AHCI mode]
[2:0:0:0] disk ATA WDC WD140EDGZ-11 0A85 /dev/sdj 14.0TB
[3:0:0:0] disk ATA WDC WD60EFRX-68M 0A82 /dev/sdk 6.00TB
[4:0:0:0] disk ATA WDC WD140EDGZ-11 0A85 /dev/sdl 14.0TB
[5:0:0:0] disk ATA WDC WD140EDGZ-11 0A85 /dev/sdm 14.0TB
IOMMU group 6: [8086:a2e7] 00:1b.0 PCI bridge: Intel Corporation 200 Series PCH PCI Express Root Port #17 (rev f0)
IOMMU group 7: [8086:a2eb] 00:1b.4 PCI bridge: Intel Corporation 200 Series PCH PCI Express Root Port #21 (rev f0)
IOMMU group 8: [8086:a290] 00:1c.0 PCI bridge: Intel Corporation 200 Series PCH PCI Express Root Port #1 (rev f0)
IOMMU group 9: [8086:a294] 00:1c.4 PCI bridge: Intel Corporation 200 Series PCH PCI Express Root Port #5 (rev f0)
IOMMU group 10: [8086:a298] 00:1d.0 PCI bridge: Intel Corporation 200 Series PCH PCI Express Root Port #9 (rev f0)
IOMMU group 11: [8086:a2c5] 00:1f.0 ISA bridge: Intel Corporation 200 Series PCH LPC Controller (Z270)
[8086:a2a1] 00:1f.2 Memory controller: Intel Corporation 200 Series/Z370 Chipset Family Power Management Controller
[8086:a2f0] 00:1f.3 Audio device: Intel Corporation 200 Series PCH HD Audio
[8086:a2a3] 00:1f.4 SMBus: Intel Corporation 200 Series/Z370 Chipset Family SMBus Controller
IOMMU group 12: [8086:15b8] 00:1f.6 Ethernet controller: Intel Corporation Ethernet Connection (2) I219-V
IOMMU group 13: [1000:0087] 01:00.0 Serial Attached SCSI controller: Broadcom / LSI SAS2308 PCI-Express Fusion-MPT SAS-2 (rev 05)
[1:0:0:0] disk ATA WDC WD140EDFZ-11 0A81 /dev/sdb 14.0TB
[1:0:1:0] disk ATA WDC WD140EDFZ-11 0A81 /dev/sdc 14.0TB
[1:0:2:0] disk ATA WDC WD140EDFZ-11 0A81 /dev/sdd 14.0TB
[1:0:3:0] disk ATA WDC WD120EDAZ-11 0A81 /dev/sde 12.0TB
[1:0:4:0] disk ATA WDC WD140EDFZ-11 0A81 /dev/sdf 14.0TB
[1:0:5:0] disk ATA WDC WD120EFAX-68 0A81 /dev/sdg 12.0TB
[1:0:6:0] disk ATA WDC WD120EMFZ-11 0A81 /dev/sdh 12.0TB
[1:0:7:0] disk ATA WDC WD140EDFZ-11 0A81 /dev/sdi 14.0TB
IOMMU group 14: [10de:1d01] 02:00.0 VGA compatible controller: NVIDIA Corporation GP108 [GeForce GT 1030] (rev a1)
[10de:0fb8] 02:00.1 Audio device: NVIDIA Corporation GP108 High Definition Audio Controller (rev a1)
IOMMU group 15: [1cc1:8201] 03:00.0 Non-Volatile memory controller: ADATA Technology Co., Ltd. XPG SX8200 Pro PCIe Gen3x4 M.2 2280 Solid State Drive (rev 03)
[N:0:1:1] disk ADATA SX8200PNP__1 /dev/nvme0n1 2.04TB
IOMMU group 16: [1cc1:8201] 04:00.0 Non-Volatile memory controller: ADATA Technology Co., Ltd. XPG SX8200 Pro PCIe Gen3x4 M.2 2280 Solid State Drive (rev 03)
[N:1:1:1] disk ADATA SX8200PNP__1 /dev/nvme1n1 2.04TB
IOMMU group 17: [1b21:2142] 06:00.0 USB controller: ASMedia Technology Inc. ASM2142 USB 3.1 Host Controller
This controller is bound to vfio, connected USB devices are not visible.
IOMMU group 18: [144d:a808] 07:00.0 Non-Volatile memory controller: Samsung Electronics Co Ltd NVMe SSD Controller SM981/PM981/PM983
This controller is bound to vfio, connected drives are not visible.
Any help would be greatly appreciated!