GTX 970 passthrough to Windows VM is crashing my server


Recommended Posts

As stated in the title. It works with VNC but crashes as soon as I passthrough the graphics card. Here are the logs from the VM, first with it working with VNC then with it crashing with passthrough.

LC_ALL=C \
PATH=/bin:/sbin:/usr/bin:/usr/sbin \
HOME=/ \
QEMU_AUDIO_DRV=none \
/usr/local/sbin/qemu \
-name 'guest=Windows 10,debug-threads=on' \
-S \
-object 'secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-2-Windows 10/master-key.aes' \
-machine pc-i440fx-3.1,accel=kvm,usb=off,dump-guest-core=off,mem-merge=off \
-cpu host,hv_time,hv_relaxed,hv_vapic,hv_spinlocks=0x1fff,hv_vendor_id=none \
-drive file=/usr/share/qemu/ovmf-x64/OVMF_CODE-pure-efi.fd,if=pflash,format=raw,unit=0,readonly=on \
-drive file=/etc/libvirt/qemu/nvram/40837305-fe22-b321-e212-4a24bdd90d0a_VARS-pure-efi.fd,if=pflash,format=raw,unit=1 \
-m 12288 \
-realtime mlock=off \
-smp 12,sockets=1,cores=12,threads=1 \
-uuid 40837305-fe22-b321-e212-4a24bdd90d0a \
-no-user-config \
-nodefaults \
-chardev socket,id=charmonitor,fd=26,server,nowait \
-mon chardev=charmonitor,id=monitor,mode=control \
-rtc base=localtime \
-no-hpet \
-no-shutdown \
-boot strict=on \
-device ich9-usb-ehci1,id=usb,bus=pci.0,addr=0x7.0x7 \
-device ich9-usb-uhci1,masterbus=usb.0,firstport=0,bus=pci.0,multifunction=on,addr=0x7 \
-device ich9-usb-uhci2,masterbus=usb.0,firstport=2,bus=pci.0,addr=0x7.0x1 \
-device ich9-usb-uhci3,masterbus=usb.0,firstport=4,bus=pci.0,addr=0x7.0x2 \
-device virtio-serial-pci,id=virtio-serial0,bus=pci.0,addr=0x4 \
-device usb-hub,id=hub0,bus=usb.0,port=1 \
-drive 'file=/mnt/user/domains/Windows 10/vdisk1.img,format=raw,if=none,id=drive-virtio-disk2,cache=writeback' \
-device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x5,drive=drive-virtio-disk2,id=virtio-disk2,bootindex=1,write-cache=on \
-drive file=/dev/nvme0n1,format=raw,if=none,id=drive-virtio-disk3,cache=writeback \
-device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x6,drive=drive-virtio-disk3,id=virtio-disk3,write-cache=on \
-netdev tap,fd=28,id=hostnet0,vhost=on,vhostfd=29 \
-device virtio-net-pci,netdev=hostnet0,id=net0,mac=52:54:00:4b:5a:8c,bus=pci.0,addr=0x3 \
-chardev pty,id=charserial0 \
-device isa-serial,chardev=charserial0,id=serial0 \
-chardev socket,id=charchannel0,fd=31,server,nowait \
-device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=org.qemu.guest_agent.0 \
-device usb-tablet,id=input0,bus=usb.0,port=1.6 \
-vnc 0.0.0.0:0,websocket=5700 \
-k en-us \
-device qxl-vga,id=video0,ram_size=67108864,vram_size=67108864,vram64_size_mb=0,vgamem_mb=16,max_outputs=1,bus=pci.0,addr=0x2 \
-device usb-host,hostbus=3,hostaddr=2,id=hostdev0,bus=usb.0,port=2 \
-device usb-host,hostbus=5,hostaddr=8,id=hostdev1,bus=usb.0,port=3 \
-device usb-host,hostbus=5,hostaddr=3,id=hostdev2,bus=usb.0,port=4 \
-device usb-host,hostbus=5,hostaddr=4,id=hostdev3,bus=usb.0,port=5 \
-device usb-host,hostbus=5,hostaddr=6,id=hostdev4,bus=usb.0,port=6 \
-device usb-host,hostbus=1,hostaddr=2,id=hostdev5,bus=usb.0,port=1.1 \
-device usb-host,hostbus=1,hostaddr=4,id=hostdev6,bus=usb.0,port=1.2 \
-device usb-host,hostbus=1,hostaddr=3,id=hostdev7,bus=usb.0,port=1.3 \
-device usb-host,hostbus=1,hostaddr=6,id=hostdev8,bus=usb.0,port=1.4 \
-device usb-host,hostbus=5,hostaddr=5,id=hostdev9,bus=usb.0,port=1.5 \
-device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x8 \
-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \
-msg timestamp=on
2019-06-28 03:50:14.653+0000: Domain id=2 is tainted: high-privileges
2019-06-28 03:50:14.653+0000: Domain id=2 is tainted: host-cpu
char device redirected to /dev/pts/0 (label charserial0)
2019-06-28T03:58:06.751344Z qemu-system-x86_64: terminating on signal 15 from pid 8348 (/usr/sbin/libvirtd)
2019-06-28 03:58:07.385+0000: shutting down, reason=shutdown
2019-06-28 03:59:19.644+0000: starting up libvirt version: 5.1.0, qemu version: 3.1.0, kernel: 4.19.55-Unraid, hostname: Tower
LC_ALL=C \
PATH=/bin:/sbin:/usr/bin:/usr/sbin \
HOME=/ \
QEMU_AUDIO_DRV=none \
/usr/local/sbin/qemu \
-name 'guest=Windows 10,debug-threads=on' \
-S \
-object 'secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-3-Windows 10/master-key.aes' \
-machine pc-i440fx-3.1,accel=kvm,usb=off,dump-guest-core=off,mem-merge=off \
-cpu host,hv_time,hv_relaxed,hv_vapic,hv_spinlocks=0x1fff,hv_vendor_id=none \
-drive file=/usr/share/qemu/ovmf-x64/OVMF_CODE-pure-efi.fd,if=pflash,format=raw,unit=0,readonly=on \
-drive file=/etc/libvirt/qemu/nvram/40837305-fe22-b321-e212-4a24bdd90d0a_VARS-pure-efi.fd,if=pflash,format=raw,unit=1 \
-m 12288 \
-realtime mlock=off \
-smp 12,sockets=1,cores=12,threads=1 \
-uuid 40837305-fe22-b321-e212-4a24bdd90d0a \
-display none \
-no-user-config \
-nodefaults \
-chardev socket,id=charmonitor,fd=26,server,nowait \
-mon chardev=charmonitor,id=monitor,mode=control \
-rtc base=localtime \
-no-hpet \
-no-shutdown \
-boot strict=on \
-device ich9-usb-ehci1,id=usb,bus=pci.0,addr=0x7.0x7 \
-device ich9-usb-uhci1,masterbus=usb.0,firstport=0,bus=pci.0,multifunction=on,addr=0x7 \
-device ich9-usb-uhci2,masterbus=usb.0,firstport=2,bus=pci.0,addr=0x7.0x1 \
-device ich9-usb-uhci3,masterbus=usb.0,firstport=4,bus=pci.0,addr=0x7.0x2 \
-device virtio-serial-pci,id=virtio-serial0,bus=pci.0,addr=0x4 \
-device usb-hub,id=hub0,bus=usb.0,port=1 \
-drive 'file=/mnt/user/domains/Windows 10/vdisk1.img,format=raw,if=none,id=drive-virtio-disk2,cache=writeback' \
-device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x3,drive=drive-virtio-disk2,id=virtio-disk2,bootindex=1,write-cache=on \
-drive file=/dev/nvme0n1,format=raw,if=none,id=drive-virtio-disk3,cache=writeback \
-device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x5,drive=drive-virtio-disk3,id=virtio-disk3,write-cache=on \
-netdev tap,fd=28,id=hostnet0,vhost=on,vhostfd=29 \
-device virtio-net-pci,netdev=hostnet0,id=net0,mac=52:54:00:4b:5a:8c,bus=pci.0,addr=0x2 \
-chardev pty,id=charserial0 \
-device isa-serial,chardev=charserial0,id=serial0 \
-chardev socket,id=charchannel0,fd=31,server,nowait \
-device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=org.qemu.guest_agent.0 \
-device usb-tablet,id=input0,bus=usb.0,port=1.6 \
-device vfio-pci,host=08:00.0,id=hostdev0,bus=pci.0,addr=0x6 \
-device vfio-pci,host=08:00.1,id=hostdev1,bus=pci.0,addr=0x8 \
-device usb-host,hostbus=3,hostaddr=2,id=hostdev2,bus=usb.0,port=2 \
-device usb-host,hostbus=5,hostaddr=8,id=hostdev3,bus=usb.0,port=3 \
-device usb-host,hostbus=5,hostaddr=3,id=hostdev4,bus=usb.0,port=4 \
-device usb-host,hostbus=5,hostaddr=4,id=hostdev5,bus=usb.0,port=5 \
-device usb-host,hostbus=5,hostaddr=6,id=hostdev6,bus=usb.0,port=6 \
-device usb-host,hostbus=1,hostaddr=2,id=hostdev7,bus=usb.0,port=1.1 \
-device usb-host,hostbus=1,hostaddr=4,id=hostdev8,bus=usb.0,port=1.2 \
-device usb-host,hostbus=1,hostaddr=3,id=hostdev9,bus=usb.0,port=1.3 \
-device usb-host,hostbus=1,hostaddr=6,id=hostdev10,bus=usb.0,port=1.4 \
-device usb-host,hostbus=5,hostaddr=5,id=hostdev11,bus=usb.0,port=1.5 \
-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \
-msg timestamp=on
2019-06-28 03:59:19.644+0000: Domain id=3 is tainted: high-privileges
2019-06-28 03:59:19.644+0000: Domain id=3 is tainted: host-cpu
char device redirected to /dev/pts/0 (label charserial0)
2019-06-28T04:00:19.150521Z qemu-system-x86_64: vhost_region_add_section:Section rounded to c0000000 prior to previous c0000030
2019-06-28T04:00:19.150754Z qemu-system-x86_64: vhost_region_add_section:Section rounded to c0000000 prior to previous c0000030
2019-06-28T04:00:19.151019Z qemu-system-x86_64: vhost_region_add_section:Section rounded to c0000000 prior to previous c0000030
2019-06-28T04:00:19.151186Z qemu-system-x86_64: vhost_region_add_section:Section rounded to c0000000 prior to previous c0000030
2019-06-28T04:00:19.151344Z qemu-system-x86_64: vhost_region_add_section:Section rounded to c0000000 prior to previous c0000030
2019-06-28T04:00:19.151506Z qemu-system-x86_64: vhost_region_add_section:Section rounded to c0000000 prior to previous c0000030
2019-06-28T04:00:19.151842Z qemu-system-x86_64: vhost_region_add_section:Section rounded to c0000000 prior to previous c0000030
2019-06-28T04:00:19.152021Z qemu-system-x86_64: vhost_region_add_section:Section rounded to c0000000 prior to previous c0000030
2019-06-28T04:00:19.152561Z qemu-system-x86_64: vhost_region_add_section:Section rounded to c0000000 prior to previous c0000030
2019-06-28T04:00:19.156008Z qemu-system-x86_64: vhost_region_add_section:Section rounded to c0000000 prior to previous c0000030
2019-06-28T04:00:19.156273Z qemu-system-x86_64: vhost_region_add_section:Section rounded to c0000000 prior to previous c0000030
2019-06-28T04:00:19.156441Z qemu-system-x86_64: vhost_region_add_section:Section rounded to c0000000 prior to previous c0000030
2019-06-28T04:00:19.156606Z qemu-system-x86_64: vhost_region_add_section:Section rounded to c0000000 prior to previous c0000030
2019-06-28T04:00:19.156778Z qemu-system-x86_64: vhost_region_add_section:Section rounded to c0000000 prior to previous c0000030
2019-06-28T04:00:19.157201Z qemu-system-x86_64: vhost_region_add_section:Section rounded to c0000000 prior to previous c0000030
2019-06-28T04:00:19.157380Z qemu-system-x86_64: vhost_region_add_section:Section rounded to c0000000 prior to previous c0000030
2019-06-28T04:00:19.157942Z qemu-system-x86_64: vhost_region_add_section:Section rounded to c0000000 prior to previous c0000030
2019-06-28T04:00:19.161290Z qemu-system-x86_64: vhost_region_add_section:Section rounded to c0000000 prior to previous c0000030
2019-06-28T04:00:19.163039Z qemu-system-x86_64: vhost_region_add_section:Section rounded to c0000000 prior to previous c0000030
2019-06-28T04:00:19.163217Z qemu-system-x86_64: vhost_region_add_section:Section rounded to c0000000 prior to previous c0000030
2019-06-28T04:00:19.163375Z qemu-system-x86_64: vhost_region_add_section:Section rounded to c0000000 prior to previous c0000030
2019-06-28T04:00:19.163540Z qemu-system-x86_64: vhost_region_add_section:Section rounded to c0000000 prior to previous c0000030
2019-06-28T04:00:19.164013Z qemu-system-x86_64: vhost_region_add_section:Section rounded to c0000000 prior to previous c0000030
2019-06-28T04:00:19.164322Z qemu-system-x86_64: vhost_region_add_section:Section rounded to c0000000 prior to previous c0000030
2019-06-28T04:00:19.165199Z qemu-system-x86_64: vhost_region_add_section:Section rounded to c0000000 prior to previous c0000030
2019-06-28T04:00:19.168919Z qemu-system-x86_64: vhost_region_add_section:Section rounded to c0000000 prior to previous c0000030

XML with the graphics card is here:
 

<?xml version='1.0' encoding='UTF-8'?>
<domain type='kvm'>
  <name>Windows 10</name>
  <uuid>40837305-fe22-b321-e212-4a24bdd90d0a</uuid>
  <metadata>
    <vmtemplate xmlns="unraid" name="Windows 10" icon="windows.png" os="windows10"/>
  </metadata>
  <memory unit='KiB'>12582912</memory>
  <currentMemory unit='KiB'>12582912</currentMemory>
  <memoryBacking>
    <nosharepages/>
  </memoryBacking>
  <vcpu placement='static'>12</vcpu>
  <cputune>
    <vcpupin vcpu='0' cpuset='4'/>
    <vcpupin vcpu='1' cpuset='5'/>
    <vcpupin vcpu='2' cpuset='6'/>
    <vcpupin vcpu='3' cpuset='7'/>
    <vcpupin vcpu='4' cpuset='8'/>
    <vcpupin vcpu='5' cpuset='9'/>
    <vcpupin vcpu='6' cpuset='10'/>
    <vcpupin vcpu='7' cpuset='11'/>
    <vcpupin vcpu='8' cpuset='12'/>
    <vcpupin vcpu='9' cpuset='13'/>
    <vcpupin vcpu='10' cpuset='14'/>
    <vcpupin vcpu='11' cpuset='15'/>
  </cputune>
  <os>
    <type arch='x86_64' machine='pc-i440fx-3.1'>hvm</type>
    <loader readonly='yes' type='pflash'>/usr/share/qemu/ovmf-x64/OVMF_CODE-pure-efi.fd</loader>
    <nvram>/etc/libvirt/qemu/nvram/40837305-fe22-b321-e212-4a24bdd90d0a_VARS-pure-efi.fd</nvram>
  </os>
  <features>
    <acpi/>
    <apic/>
    <hyperv>
      <relaxed state='on'/>
      <vapic state='on'/>
      <spinlocks state='on' retries='8191'/>
      <vendor_id state='on' value='none'/>
    </hyperv>
  </features>
  <cpu mode='host-passthrough' check='none'>
    <topology sockets='1' cores='12' threads='1'/>
  </cpu>
  <clock offset='localtime'>
    <timer name='hypervclock' present='yes'/>
    <timer name='hpet' present='no'/>
  </clock>
  <on_poweroff>destroy</on_poweroff>
  <on_reboot>restart</on_reboot>
  <on_crash>restart</on_crash>
  <devices>
    <emulator>/usr/local/sbin/qemu</emulator>
    <disk type='file' device='disk'>
      <driver name='qemu' type='raw' cache='writeback'/>
      <source file='/mnt/user/domains/Windows 10/vdisk1.img'/>
      <target dev='hdc' bus='virtio'/>
      <boot order='1'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
    </disk>
    <disk type='block' device='disk'>
      <driver name='qemu' type='raw' cache='writeback'/>
      <source dev='/dev/nvme0n1'/>
      <target dev='hdd' bus='virtio'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
    </disk>
    <controller type='pci' index='0' model='pci-root'/>
    <controller type='virtio-serial' index='0'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
    </controller>
    <controller type='usb' index='0' model='ich9-ehci1'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x7'/>
    </controller>
    <controller type='usb' index='0' model='ich9-uhci1'>
      <master startport='0'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0' multifunction='on'/>
    </controller>
    <controller type='usb' index='0' model='ich9-uhci2'>
      <master startport='2'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x1'/>
    </controller>
    <controller type='usb' index='0' model='ich9-uhci3'>
      <master startport='4'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x2'/>
    </controller>
    <interface type='bridge'>
      <mac address='52:54:00:4b:5a:8c'/>
      <source bridge='br0'/>
      <model type='virtio'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
    </interface>
    <serial type='pty'>
      <target type='isa-serial' port='0'>
        <model name='isa-serial'/>
      </target>
    </serial>
    <console type='pty'>
      <target type='serial' port='0'/>
    </console>
    <channel type='unix'>
      <target type='virtio' name='org.qemu.guest_agent.0'/>
      <address type='virtio-serial' controller='0' bus='0' port='1'/>
    </channel>
    <input type='tablet' bus='usb'>
      <address type='usb' bus='0' port='1.6'/>
    </input>
    <input type='mouse' bus='ps2'/>
    <input type='keyboard' bus='ps2'/>
    <hostdev mode='subsystem' type='pci' managed='yes'>
      <driver name='vfio'/>
      <source>
        <address domain='0x0000' bus='0x08' slot='0x00' function='0x0'/>
      </source>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x06' function='0x0'/>
    </hostdev>
    <hostdev mode='subsystem' type='pci' managed='yes'>
      <driver name='vfio'/>
      <source>
        <address domain='0x0000' bus='0x08' slot='0x00' function='0x1'/>
      </source>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x08' function='0x0'/>
    </hostdev>
    <hostdev mode='subsystem' type='usb' managed='no'>
      <source>
        <vendor id='0x045e'/>
        <product id='0x02e6'/>
      </source>
      <address type='usb' bus='0' port='2'/>
    </hostdev>
    <hostdev mode='subsystem' type='usb' managed='no'>
      <source>
        <vendor id='0x045e'/>
        <product id='0x0719'/>
      </source>
      <address type='usb' bus='0' port='3'/>
    </hostdev>
    <hostdev mode='subsystem' type='usb' managed='no'>
      <source>
        <vendor id='0x046d'/>
        <product id='0x0826'/>
      </source>
      <address type='usb' bus='0' port='4'/>
    </hostdev>
    <hostdev mode='subsystem' type='usb' managed='no'>
      <source>
        <vendor id='0x046d'/>
        <product id='0xc226'/>
      </source>
      <address type='usb' bus='0' port='5'/>
    </hostdev>
    <hostdev mode='subsystem' type='usb' managed='no'>
      <source>
        <vendor id='0x046d'/>
        <product id='0xc227'/>
      </source>
      <address type='usb' bus='0' port='6'/>
    </hostdev>
    <hostdev mode='subsystem' type='usb' managed='no'>
      <source>
        <vendor id='0x0a12'/>
        <product id='0x0001'/>
      </source>
      <address type='usb' bus='0' port='1.1'/>
    </hostdev>
    <hostdev mode='subsystem' type='usb' managed='no'>
      <source>
        <vendor id='0x1532'/>
        <product id='0x006c'/>
      </source>
      <address type='usb' bus='0' port='1.2'/>
    </hostdev>
    <hostdev mode='subsystem' type='usb' managed='no'>
      <source>
        <vendor id='0x24ae'/>
        <product id='0x1005'/>
      </source>
      <address type='usb' bus='0' port='1.3'/>
    </hostdev>
    <hostdev mode='subsystem' type='usb' managed='no'>
      <source>
        <vendor id='0x264a'/>
        <product id='0x1fa5'/>
      </source>
      <address type='usb' bus='0' port='1.4'/>
    </hostdev>
    <hostdev mode='subsystem' type='usb' managed='no'>
      <source>
        <vendor id='0x28de'/>
        <product id='0x1142'/>
      </source>
      <address type='usb' bus='0' port='1.5'/>
    </hostdev>
    <hub type='usb'>
      <address type='usb' bus='0' port='1'/>
    </hub>
    <memballoon model='none'/>
  </devices>
</domain>

I have also attached my diagnostics.

tower-diagnostics-20190628-0419.zip

Link to comment
  1. In the BIOS, did you set it to use whichever slot your 1030 is on to boot? If not, do it first and see if your VM boot up.
  2. If that still fails, watch the Spaceinvader One video below and follow his guide to dump the vbios and use it to see if it helps. The vbios helps with stability.
  3. If that still fails, create a new template but switch to Q35 (instead of i440fx). Some cards would only work with Q35. You MUST create a new template and not just switch it in the GUI because switching from i440fx to Q35 is too complex for the GUI to handle.
    Remember to add the bit of code at the bottom of my post to your xml right above </domain>. Otherwise your GPU will run at PCIe x1.
  4. If all the above fail then there's nothing else I know that would help.

 

Spaceinvader One's video

 

Code to add to xml for Q35:

  <qemu:commandline>
    <qemu:arg value='-global'/>
    <qemu:arg value='pcie-root-port.speed=8'/>
    <qemu:arg value='-global'/>
    <qemu:arg value='pcie-root-port.width=16'/>
  </qemu:commandline>

 

 

Link to comment

After 2 days of fighting this problem I solved it.

 

I really don't understand how it fixed it, but here are the steps that did:

1. Downloaded the lastest drivers from nvidia ( 430.68 ) using VNC

2. Told Windows to reset itself from recovery using VNC

3.  Shut down and swapped to passthrough

4. Booted and installed the latest drivers.

 

Now it's working.

 

I think there might have been a problem with the drivers it downloaded from MS, but I really don't have a clue.

Link to comment

Join the conversation

You can post now and register later. If you have an account, sign in now to post with your account.
Note: Your post will require moderator approval before it will be visible.

Guest
Reply to this topic...

×   Pasted as rich text.   Restore formatting

  Only 75 emoji are allowed.

×   Your link has been automatically embedded.   Display as a link instead

×   Your previous content has been restored.   Clear editor

×   You cannot paste images directly. Upload or insert images from URL.